Compare commits

...
Author SHA1 Message Date
jschoubben 858b06a94e Take mesh-sdk at c8e3509, which counts an item's text as a placement of a config value (issue 1658)
mesh/repo-check head 858b06a94e69 merged into trunk 12752911b644 — fail: head 858b06a94e69 does not merge cleanly into trunk 12752911b644: in conflict go.m…
mesh/delivery-group group fix/1656-xterm-config rejected: a member's own check failed
mesh/merge-gate group fix/1656-xterm-config: composed again for the group's new heads
mesh/delivery superseded: a newer head of the same pull request
2026-10-21 12:55:08 +02:00
mesh-tools cada7e3c83 Merge pull request 'Take the mesh-sdk that knows shares on an s3-bucket requirement, and prove the controller composes and checks it (issue 1593)' (#416) from feat/files-through-the-store into main 2026-10-21 10:42:04 +00:00
mesh-tools ff42f21839 Merge pull request 'Refuse a build whose module directory, archive or bundle output leads out of the module through a link, so a repository cannot make build-agent pack files from the build node (issue 1609)' (#415) from fix/1609-archive-symlink into main 2026-10-21 10:16:52 +00:00
jschoubben 64c1996a5e Prove the controller composes and checks shares on an s3-bucket requirement (issue 1593)
mesh/merge-gate head 64c1996a5e9d merged into trunk ff42f2183900 — pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus…
mesh/repo-check head 64c1996a5e9d merged into trunk ff42f2183900 — pass: its merge-check.sh passed
mesh/delivery delivered
2026-10-21 11:53:24 +02:00
jschoubben 8a1ef29e36 Take mesh-sdk at 465beb4, which carries shares on an s3-bucket requirement (issue 1593) 2026-10-21 11:51:38 +02:00
mesh-tools 0caa8c6844 Refuse a bundle whose output directory leads out of the module through a link, so a repository cannot make the build write into or pack files it does not hold (issue 1609)
mesh/merge-gate head 0caa8c6844d9 merged into trunk 0205b8f7080a — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 0caa8c6844d9 merged into trunk 0205b8f7080a — pass: its merge-check.sh passed
mesh/delivery delivered
2026-10-21 09:46:00 +00:00
mesh-tools fcca6ba9e0 Refuse an archive or a module directory that leads out of the repository through a link in the tree, so a repository cannot make the build pack files it does not hold (issue 1609) 2026-10-21 09:22:14 +00:00
mesh-tools 0205b8f708 Merge pull request 'Hold a merge for any failure to record it, so the bus hands it over again and no merge is lost (issue 1583)' (#414) from fix/1583-merge-handed-over into main 2026-10-21 09:05:55 +00:00
mesh-tools c8539bf35a Merge pull request 'Do not ask the operator to stop a delivery that is already stopped; say what ends it (issue 1581)' (#413) from fix/1581-stopped-alert into main 2026-10-21 08:52:14 +00:00
mesh-tools 5c94759cc0 Hold a merge for any failure to record it, so the bus handing it over again never loses it (issue 1583)
mesh/merge-gate head 5c94759cc0c2 merged into trunk c8539bf35a8d — pass: builds mesh-controller → novox; no bus step; every machine composes with the chang…
mesh/repo-check head 5c94759cc0c2 merged into trunk c8539bf35a8d — pass: its merge-check.sh passed
mesh/delivery delivered
2026-10-21 08:43:09 +00:00
mesh-tools cddf150871 Spell the dash escape with one backslash, as the reviewed source has it (issue 1581)
mesh/merge-gate head cddf15087178 merged into trunk 0fee5404d95b — pass: builds mesh-controller → novox; no bus step; every machine composes with the chang…
mesh/repo-check head cddf15087178 merged into trunk 0fee5404d95b — pass: its merge-check.sh passed
mesh/delivery delivered
2026-10-21 08:30:27 +00:00
mesh-tools ef29adbc1a Spell the dash in stoppedNeeds as an escape, as the reviewed source has it (issue 1581) 2026-10-21 08:30:12 +00:00
mesh-tools e962279f1d Do not ask the operator to stop a delivery that is already stopped; say what ends it (issue 1581) 2026-10-21 08:29:57 +00:00
21 changed files with 1066 additions and 46 deletions
+28 -12
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"slices"
@@ -170,6 +171,21 @@ func (f mergeFacts) plans() bool {
return len(f.from) > 0 || len(f.packaging) > 0 || len(f.later) > 0 || f.owed
}
// heldMerge marks a failure to act on a merge, whatever it was, as "not now": the announcement is held on the bus and
// handed over again, so the merge joins the next batch, never taken (issue 1583, after 921). The bus acknowledges it
// only when hearMerge answers without ErrTryAgain, and a merge heard once is not announced again: the bus answering
// "jetstream not enabled", a bucket missing for a moment or a store read cut short each lost the merge when only an
// outage of a known kind (notNow) was held. The delay between hands-over is bounded (link.MergeRedeliverAfter).
func heldMerge(err error) error {
if err == nil {
return nil
}
if errors.Is(err, link.ErrTryAgain) {
return err
}
return fmt.Errorf("%w: the merge is not kept yet: %w", link.ErrTryAgain, err)
}
// hearMerge puts a merge the forge announced into the batch that answers it, at the moment it was heard,
// and cuts what is due.
func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.Time) error {
@@ -180,7 +196,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
inv := f.open.inventory
repository := m.Owner + "/" + m.Repo
if kept, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil {
return notNow(err)
return heldMerge(err)
} else if known {
fmt.Printf("%s merged into %s (%.8s) was heard before; %s answers it\n", repository, m.Base, m.Commit,
orWords(kept.Plan, "a walk of its own commit, once the walks before it end"))
@@ -188,15 +204,15 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return notNow(err)
return heldMerge(err)
}
read, err := readForPlanning(ctx, inv)
if err != nil {
return notNow(err)
return heldMerge(err)
}
facts, err := factsOfMerge(ctx, inv, m, now, entries, read)
if err != nil {
return notNow(err)
return heldMerge(err)
}
merged, touches := facts.merged, facts.touches
if !facts.plans() {
@@ -210,7 +226,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
}
if already == 0 {
if done, err := f.firstBuilds(ctx, m, entries, read, now); err != nil || done {
return err
return heldMerge(err)
}
}
fmt.Print(nothingReadsIt(repository, m))
@@ -220,16 +236,16 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
own := ownPath(touches)
waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries))
if err != nil {
return notNow(err)
return heldMerge(err)
}
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return notNow(err)
return heldMerge(err)
}
defer release()
// Again under the hold: a controller handing over may have kept it since.
if _, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil || known {
return notNow(err)
return heldMerge(err)
}
var moves []string
for _, e := range touches {
@@ -248,11 +264,11 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
// word** (decided during the build, 2026-10-10): batched together, the catalogue's deliveries would start
// with the controller's and skip their turn. Each kind has a batch of its own.
if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own, deliverySeatHeld(entries)); err != nil {
return notNow(err)
return heldMerge(err)
} else if ok {
heard.Plan = p.ID
if _, err := inv.AddMerge(ctx, heard); err != nil {
return notNow(err)
return heldMerge(err)
}
if p.Batch() {
if err := keepBatch(ctx, inv, &p, now, ""); err != nil {
@@ -268,11 +284,11 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
batch, why, err := batchToJoin(ctx, inv, heard, entries, read, now, own, waits)
if err != nil {
return notNow(err)
return heldMerge(err)
}
heard.Plan = batch.ID
if added, err := inv.AddMerge(ctx, heard); err != nil || !added {
return notNow(err)
return heldMerge(err)
}
holdFor(&batch, m.Head, waits, holding, own, now)
if err := keepBatch(ctx, inv, &batch, now, ""); err != nil {
@@ -0,0 +1,75 @@
//go:build integration
package main
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/chain"
"github.com/novox/mesh-controller/internal/link"
)
// withoutJetStream is a bucket whose reads fail the way a bus answers while its streams are away.
type withoutJetStream struct {
chain.Backend
away atomic.Bool
}
func (w *withoutJetStream) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
if w.away.Load() {
return chain.Entry{}, false, nats.ErrJetStreamNotEnabled
}
return w.Backend.Get(ctx, key)
}
func (w *withoutJetStream) Keys(ctx context.Context) ([]string, error) {
if w.away.Load() {
return nil, nats.ErrJetStreamNotEnabled
}
return w.Backend.Keys(ctx)
}
func (w *withoutJetStream) Watch(ctx context.Context) (<-chan chain.Entry, error) {
if w.away.Load() {
return nil, nats.ErrJetStreamNotEnabled
}
return w.Backend.Watch(ctx)
}
func (w *withoutJetStream) WatchFiltered(ctx context.Context, filter string) (<-chan chain.Entry, error) {
if w.away.Load() {
return nil, nats.ErrJetStreamNotEnabled
}
return w.Backend.WatchFiltered(ctx, filter)
}
// **A merge heard while the bus answers "jetstream not enabled" is held, recorded nowhere, and recorded when handed
// over again** (issue 1583, mesh-catalog #404): hearMerge answers ErrTryAgain, so the bus does not acknowledge the
// announcement, and the merge joins a batch when the bus is back.
func TestAMergeHeardWhileTheBusHasNoJetStreamIsHeldAndRecordedWhenHandedOverAgain(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksWithPaths(t)
ctx := t.Context()
assignments := &withoutJetStream{Backend: chain.NewMemory()}
open.inventory.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), assignments, func(context.Context) error { return nil }))
merge := catalogueMerge("cat2bbbbbbbb", "app", t0)
assignments.away.Store(true)
err := (following{open: open}).hearMerge(ctx, merge, t0.Add(time.Second))
if !errors.Is(err, link.ErrTryAgain) {
t.Fatalf("a merge heard while the bus had no jetstream answered %v, which the bus takes as done", err)
}
assignments.away.Store(false)
hear(t, open, merge, t0.Add(10*time.Second))
if _, bs := walks(t, open); len(bs) != 1 {
t.Fatalf("the merge handed over again made %d batch(es)", len(bs))
}
}
@@ -0,0 +1,33 @@
package main
import (
"errors"
"testing"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/link"
)
// **A merge the controller cannot act on is held, whatever the error** (issue 1583, after 921): the bus acknowledges an
// announcement only when hearMerge answers without ErrTryAgain, and a merge heard once is not announced again. The bus
// answering "jetstream not enabled" (mesh-catalog #404, 2026-10-21) is no outage of a kind notNow knows, and took the merge.
func TestAMergeThatFailsOnAnyErrorIsHeldAndNotTaken(t *testing.T) {
for name, cause := range map[string]error{
"jetstream not enabled": nats.ErrJetStreamNotEnabled,
"a bucket missing": errors.New("the bucket mesh-controller_builds is not on the bus"),
"no responders": nats.ErrNoResponders,
} {
held := heldMerge(cause)
if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) {
t.Errorf("%s answers %v, which the bus takes as done", name, held)
}
}
if heldMerge(nil) != nil {
t.Error("a merge that was heard answers an error")
}
already := link.ErrTryAgain
if heldMerge(already) != already {
t.Error("a merge already held is wrapped again")
}
}
+15
View File
@@ -951,6 +951,10 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
needs, actions = "release it, or stop it, "+FromMeshMCPServer, []conditions.Action{release, stop}
case "ready", "checked":
needs = "merge its pull request, or close it."
case "stopped":
// Already stopped: mesh-delivery.stop refuses it, so neither offer nor ask it (novox/hq issue 1581).
// What ends it is the stalled line's own word, such as a later merge that delivers its modules.
needs = stoppedNeeds(l)
default:
needs, actions = "stop it "+FromMeshMCPServer, []conditions.Action{stop}
}
@@ -960,6 +964,17 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
}
// stoppedNeeds is what ends a delivery that is already stopped, from the stalled line's healer words ("none: a
// merge that delivers its modules ends it — the operator's"): the part that says what ends it.
func stoppedNeeds(l stalledLine) string {
ends := strings.TrimSpace(strings.TrimPrefix(l.H2, "none:"))
ends, _, _ = strings.Cut(ends, " — ")
if ends == "" {
ends = "a later merge that delivers its modules ends it"
}
return "nothing to stop, it is stopped already: " + ends + "."
}
// unansweredWords are the plain words of a pull request's head whose merge check was asked and has not answered
// within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release
// or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look
+21
View File
@@ -498,3 +498,24 @@ func TestAnUnansweredMergeCheckOnAReadyOrRejectedDeliveryOffersARecheck(t *testi
t.Fatalf("a line with no delivery held says to %q", o.Needs)
}
}
// **A delivery already stopped is not asked to be stopped** (novox/hq issue 1581): mesh-delivery.stop refuses a
// stopped delivery, so the condition offers no Stop and says what ends it, in the stalled line's own words. Any
// other held state keeps its Stop.
func TestAStoppedDeliveryIsNotAskedToBeStopped(t *testing.T) {
l := stalledLine{ID: "novox/mesh-catalog@2f6086120c46", State: "stopped", For: "12h19m36s",
H2: "none: a merge that delivers its modules ends it — the operator's",
Says: "a merged change never reached the machines"}
_, _, _, needs, actions := stalledWords(l, conditions.Observation{Resolver: conditions.ResolverOperator})
if len(actions) != 0 {
t.Fatalf("a stopped delivery offers %v, want no action", actions)
}
if strings.Contains(needs, "stop it") || !strings.Contains(needs, "a merge that delivers its modules ends it") {
t.Fatalf("needs = %q, want what ends it and no ask to stop", needs)
}
l.State = "failing"
_, _, _, needs, actions = stalledWords(l, conditions.Observation{Resolver: conditions.ResolverOperator})
if len(actions) != 1 || actions[0].Label != "Stop" || !strings.HasPrefix(needs, "stop it") {
t.Fatalf("another held state changed: needs %q, actions %v", needs, actions)
}
}
+104
View File
@@ -0,0 +1,104 @@
package main
import (
"bytes"
"crypto/ecdh"
"crypto/rand"
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/overlay"
)
// Issue 1593 (novox/hq ADR 0468 §8): `shares` on a requirement of s3-bucket, taken in with mesh-sdk 465beb4. The fixture's
// shape is mesh-sdk's go/catalogue/bucket_shares_test.go (issue 1591) and go/compose/resolve_test.go theObjectsMesh.
const filesRequiringShares = `{"module":"files","version":"1","requires":[
{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["put"]}]}]}`
func aKey(t *testing.T) string {
t.Helper()
k, err := ecdh.X25519().GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes())
}
// 1593/AC1 unit: composing the provider's machine, the file it receives for the consumer carries the shares.
func TestTheProvidersMachineReceivesTheSharesOfAnS3BucketRequirement(t *testing.T) {
var overlayModule catalogue.Manifest
raw, err := json.Marshal(overlay.Manifest())
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(raw, &overlayModule); err != nil {
t.Fatal(err)
}
consumer, err := catalogue.ParseManifest([]byte(filesRequiringShares))
if err != nil {
t.Fatal(err)
}
provider := catalogue.Manifest{Module: "objects", Version: "1",
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}
in := compose.Input{
Catalogue: map[string]catalogue.Manifest{overlay.Name: overlayModule, "objects": provider, "files": consumer},
Issued: []compose.Issued{{Provision: "s3-bucket", Consumer: "laptop", ConsumerModule: "files",
Provider: "anchor", ForProvider: "sealed-to-anchor"}},
}
for i, name := range []string{"anchor", "laptop"} {
assigned := []string{overlay.Name, "objects"}
if name == "laptop" {
assigned = []string{overlay.Name, "files"}
}
in.Nodes = append(in.Nodes, compose.Node{Name: name, SealingKey: name + "-key", Assigned: assigned,
Place: compose.Place{Key: aKey(t), Endpoint: name + ".example:51820", Site: "here", Hub: name == "anchor",
Address: []string{"10.77.0.1", "10.77.0.2"}[i]},
Capabilities: map[string]bool{"container-runtime": true, "wireguard": true, "systemd": true}})
}
rendered, err := compose.Render(in, "anchor", compose.Foreseeing)
if err != nil {
t.Fatal(err)
}
if len(rendered.Rendering.Grants) != 1 {
t.Fatalf("grants are %+v", rendered.Rendering.Grants)
}
got, err := json.Marshal(rendered.Rendering.Grants[0].Values["shares"])
if err != nil {
t.Fatal(err)
}
if want := `[{"methods":["put"],"to":"seat:node-files"}]`; string(got) != want {
t.Fatalf("the provider's grant for the consumer carries shares %s, want %s", got, want)
}
}
// 1593/AC2 unit: the module check accepts a manifest with shares and refuses an unknown method, citing ADR 0468 §8.
func TestModuleCheckAcceptsSharesAndRefusesAnUnknownMethod(t *testing.T) {
dir := t.TempDir()
write := func(name, body string) string {
p := filepath.Join(dir, name+".json")
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
return p
}
var out bytes.Buffer
if err := moduleCheckFor([]string{write("files", filesRequiringShares)}, 6, &out); err != nil {
t.Fatalf("a manifest with shares was refused: %v\n%s", err, out.String())
}
out.Reset()
bad := write("bad", `{"module":"bad","version":"1","requires":[
{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["delete"]}]}]}`)
err := moduleCheckFor([]string{bad}, 6, &out)
if err == nil || !strings.Contains(out.String(), "ADR 0468 §8") {
t.Fatalf("an unknown method passed or the refusal does not cite the ADR: %v\n%s", err, out.String())
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020085642-ab0c16ddad26
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861
github.com/jackc/pgx/v5 v5.10.0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
+2 -6
View File
@@ -1,11 +1,7 @@
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4 h1:yYDxFRJoH59kiCoEulD+FBwctyf9CrtKIxlEAsVL39Q=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9 h1:qHpN/fpTDqijfMVoepxUT6mPkfcydb4LiPKsIokv6iM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020085642-ab0c16ddad26 h1:oT1l3ZdjyF82MK14QMLKzoH2+iOFH/+UEYdlK3T2f7k=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020085642-ab0c16ddad26/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861 h1:BsqolZMMsC5jLP8X88BwShqFssLgtREcgBIy6OfPSI8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+149
View File
@@ -0,0 +1,149 @@
package builder
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// A link as a parent of an archive's `from` must not lead the build out of the module: the repository
// is somebody else's, and a build that followed it would pack, register and serve any directory the
// build account can read (issue 1609).
func TestAnArchiveWhoseFromGoesThroughALinkOutOfTheModuleIsRefused(t *testing.T) {
outside := t.TempDir()
if err := os.MkdirAll(filepath.Join(outside, "x"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(outside, "x", "secret"), []byte("s"), 0o644); err != nil {
t.Fatal(err)
}
tree := t.TempDir()
if err := os.Symlink(outside, filepath.Join(tree, "link")); err != nil {
t.Fatal(err)
}
r := &recorded{}
a := catalogue.Artifact{Name: "all", Kind: catalogue.ArtifactArchive, From: "link/x"}
_, err := one(context.Background(), nil, r, "m", tree, t.TempDir(), "c", "", a, nil, nil, "", Npmrc{}, nil,
nil, func(string, string, ...any) {})
if err == nil {
t.Fatal("an archive was packed through a link out of the module")
}
realOutside, _ := filepath.EvalSymlinks(filepath.Join(outside, "x"))
realTree, _ := filepath.EvalSymlinks(tree)
for _, want := range []string{realOutside, realTree, "link/x"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal %q does not name %q", err, want)
}
}
if len(r.archives) != 0 {
t.Errorf("a refused archive left %d published", len(r.archives))
}
}
// A link that stays inside the module is still followed, and its archive keeps its digest.
func TestAnArchiveWhoseFromGoesThroughALinkInsideTheModuleIsPacked(t *testing.T) {
tree := t.TempDir()
if err := os.MkdirAll(filepath.Join(tree, "real", "x"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(tree, "real", "x", "f"), []byte("f"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink("real", filepath.Join(tree, "link")); err != nil {
t.Fatal(err)
}
r := &recorded{}
a := catalogue.Artifact{Name: "all", Kind: catalogue.ArtifactArchive, From: "link/x"}
if _, err := one(context.Background(), nil, r, "m", tree, t.TempDir(), "c", "", a, nil, nil, "", Npmrc{}, nil,
nil, func(string, string, ...any) {}); err != nil {
t.Fatal(err)
}
}
// A module directory that is itself a link out of the fetched tree is refused, naming both places.
func TestAModuleDirectoryThatIsALinkOutOfTheRepositoryIsRefused(t *testing.T) {
outside := t.TempDir()
if err := os.WriteFile(filepath.Join(outside, ManifestName), []byte("{}"), 0o644); err != nil {
t.Fatal(err)
}
tree := t.TempDir()
if err := os.MkdirAll(filepath.Join(tree, "modules"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(tree, "modules", "evil")); err != nil {
t.Fatal(err)
}
_, err := inside(tree, "modules/evil")
if err == nil {
t.Fatal("a module directory behind a link out of the repository was accepted")
}
realOutside, _ := filepath.EvalSymlinks(outside)
realTree, _ := filepath.EvalSymlinks(tree)
for _, want := range []string{realOutside, realTree, "modules/evil"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal %q does not name %q", err, want)
}
}
// A link that stays inside the repository, and a module directory not there, are left as they were.
if err := os.MkdirAll(filepath.Join(tree, "real"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Symlink("real", filepath.Join(tree, "modules", "fine")); err != nil {
t.Fatal(err)
}
if _, err := inside(tree, "modules/fine"); err != nil {
t.Errorf("a link inside the repository was refused: %v", err)
}
if _, err := inside(tree, "modules/none"); err != nil {
t.Errorf("an absent module directory was refused here: %v", err)
}
}
// A repository that commits `.mesh-build` as a link must not make a bundle's build write into, or pack,
// a directory outside the module (issue 1609): the compile, the bundling and the pack each refuse it.
func TestABundleWhoseOutputDirectoryIsALinkOutOfTheModuleIsRefused(t *testing.T) {
outside := t.TempDir()
if err := os.WriteFile(filepath.Join(outside, "secret"), []byte("s"), 0o644); err != nil {
t.Fatal(err)
}
tree := t.TempDir()
if err := os.Symlink(outside, filepath.Join(tree, ".mesh-build")); err != nil {
t.Fatal(err)
}
chain, err := ToolchainFor("go")
if err != nil {
t.Fatal(err)
}
a := catalogue.Artifact{Name: "b", Kind: catalogue.ArtifactBundle, Language: "go", From: "cmd/b"}
ran := false
run := func(context.Context, string, string, ...string) (string, error) { ran = true; return "", nil }
realOutside, _ := filepath.EvalSymlinks(outside)
realTree, _ := filepath.EvalSymlinks(tree)
check := func(what string, err error) {
t.Helper()
if err == nil {
t.Fatalf("%s went through a link out of the module", what)
}
for _, want := range []string{realOutside, realTree} {
if !strings.Contains(err.Error(), want) {
t.Errorf("%s: the refusal %q does not name %q", what, err, want)
}
}
}
_, err = compile(context.Background(), run, tree, chain, "base", a)
check("compile", err)
_, err = bundled(context.Background(), run, tree, chain, "base", a, nil)
check("bundled", err)
check("the pack", outsideTheModule(tree, filepath.Join(tree, Out(a.Name))))
if ran {
t.Error("a command ran for a refused bundle")
}
if entries, _ := os.ReadDir(outside); len(entries) != 1 {
t.Errorf("a refused bundle wrote into the directory outside: %d entries", len(entries))
}
}
+81 -1
View File
@@ -943,6 +943,23 @@ func inside(tree, path string) (string, error) {
return "", fmt.Errorf(
"%q leaves the repository, and a build reads only its own tree", path)
}
// A link in the fetched tree must not lead the module out of it either (issue 1609): compare where
// both really are. A module directory that does not exist is left to the read of its manifest.
realTree, err := filepath.EvalSymlinks(tree)
if err != nil {
return "", err
}
realWithin, err := filepath.EvalSymlinks(within)
if errors.Is(err, os.ErrNotExist) {
return within, nil
}
if err != nil {
return "", err
}
if rel, err := filepath.Rel(realTree, realWithin); err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("%q leads out of the repository through a link: it resolves to %s, and the repository is %s",
path, realWithin, realTree)
}
return within, nil
}
@@ -1278,6 +1295,9 @@ func one(ctx context.Context, run Runner, publish Publisher,
}
}
say("bundle", "compiled, packing")
if err := outsideTheModule(tree, compiled); err != nil {
return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
}
body, err := pack(compiled)
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
@@ -1326,7 +1346,10 @@ func one(ctx context.Context, run Runner, publish Publisher,
}
} else {
src.ownHas(cleanEntry(a.From) + "/**")
body, err = pack(filepath.Join(tree, a.From))
var root string
if root, err = archiveRoot(tree, a); err == nil {
body, err = pack(root)
}
}
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
@@ -1343,6 +1366,57 @@ func one(ctx context.Context, run Runner, publish Publisher,
module, a.Name, a.Kind)
}
// archiveRoot is the directory an archive artifact is packed from, refused when a link in the fetched
// tree leads out of the module. The repository is somebody else's: pack follows a link in a parent
// component of `from`, so without this a committed link makes the build pack, register and serve any
// directory the build account can read (issue 1609). Both are compared where they really are.
func archiveRoot(tree string, a catalogue.Artifact) (string, error) {
root := filepath.Join(tree, a.From)
realTree, err := filepath.EvalSymlinks(tree)
if err != nil {
return "", err
}
realRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return "", err
}
if rel, err := filepath.Rel(realTree, realRoot); err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("the archive %s is taken from %q, which lies outside the module: it resolves to %s, and the module is %s",
a.Name, a.From, realRoot, realTree)
}
return root, nil
}
// outsideTheModule refuses a path that, once every link in it is followed, is not inside the module's
// directory, naming where it really is and where the module is. For a path not there yet (an output
// directory about to be made) the deepest part that exists is the one followed. A repository commits
// its own links, so `.mesh-build` as a link would otherwise have a bundle's build write into, and pack,
// any directory the build account can read (issue 1609).
func outsideTheModule(tree, path string) error {
realTree, err := filepath.EvalSymlinks(tree)
if err != nil {
return err
}
real, rest := path, ""
for {
resolved, err := filepath.EvalSymlinks(real)
if err == nil {
real = filepath.Join(resolved, rest)
break
}
if !errors.Is(err, os.ErrNotExist) || filepath.Dir(real) == real {
return err
}
rest = filepath.Join(filepath.Base(real), rest)
real = filepath.Dir(real)
}
if rel, err := filepath.Rel(realTree, real); err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return fmt.Errorf("%s leads out of the module through a link: it resolves to %s, and the module is %s",
path, real, realTree)
}
return nil
}
// pack tars and gzips a directory.
//
// **Deterministically**: entries sorted, and no timestamps, uid, gid or original names carried
@@ -1646,6 +1720,9 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
// a package in a third (ADR 0040). Compiling them all into one place would have them overwrite
// each other and then be packed together, so each bundle compiles and packs alone.
out := Out(a.Name)
if err := outsideTheModule(tree, filepath.Join(tree, out)); err != nil {
return "", err
}
// **The output directory exists before the compiler is told about it.** `tsc --outDir` makes
// one; `go build -o` writes a file into a directory and does not create it, failing with a
@@ -1972,6 +2049,9 @@ func bundled(ctx context.Context, run Runner, tree string, chain Toolchain, base
a catalogue.Artifact, launchers map[string]string) (string, error) {
const within = "/app/modules/module"
out, final := Out(a.Name), Out(a.Name)+bundledSuffix
if err := outsideTheModule(tree, filepath.Join(tree, final)); err != nil {
return "", err
}
if err := os.RemoveAll(filepath.Join(tree, final)); err != nil {
return "", err
}
+1 -1
View File
@@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself.
mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile
mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock
mesh-sdk ab0c16ddad260289006a40c5dc6fd4973e0b5e0b conformance/events/module-event.json
mesh-sdk c8e35097e8611c174c7937d9ade86ed292314e9a conformance/events/module-event.json
To move them, from this repository's root, with the two repositories checked out beside it:
@@ -0,0 +1,124 @@
package catalogue
import (
"encoding/json"
"fmt"
"regexp"
"sort"
)
// BucketShare is one entry of `shares` on a module's requirement of `s3-bucket` (novox/hq ADR 0468 §8): who else may
// put or get in the consumer's bucket, and with which methods. Not the NFS shares of shares.go (ADR 0263).
//
// "requires": [{"name": "s3-bucket", "shares": [{"to": "seat:node-files", "methods": ["put"]},
// {"to": "print-and-scan", "methods": ["get"]}]}]
//
// `to` is a module name, or `seat:<seat name>`. `methods` is a non-empty subset of get and put, without a repeat.
// Shares are allowed on `s3-bucket` only, and no two entries name the same `to`.
type BucketShare struct {
To string `json:"to"`
Methods []string `json:"methods"`
}
// BucketRequirement is the requirement `shares` may be written on.
const BucketRequirement = "s3-bucket"
// RequireSharesKey is where a stored manifest keeps the shares of its requirements: beside `requires`, which stays a
// plain list of names, for the reason OptionalRequiresKey is beside it. An older controller sets the unknown key
// aside instead of failing the catalogue.
const RequireSharesKey = "requires-shares"
// SharesValue is the key of the value a consumer gives the provider of s3-bucket in its grant: the list of
// {to, methods} sorted by `to`. A consumer's own `contributes` may not name it.
const SharesValue = "shares"
var shareTarget = regexp.MustCompile(`^(seat:)?[a-z0-9][a-z0-9-]*$`)
// checkBucketShares refuses a malformed `shares` of one requirement, naming the entry and ADR 0468 §8.
func checkBucketShares(requirement string, shares []BucketShare) error {
if len(shares) == 0 {
return nil
}
if requirement != BucketRequirement {
return fmt.Errorf("requires %s: `shares` is allowed on %s only (novox/hq ADR 0468 §8)", requirement, BucketRequirement)
}
seen := map[string]bool{}
for i, s := range shares {
at := fmt.Sprintf("requires %s: shares[%d] (to %q)", requirement, i, s.To)
if !shareTarget.MatchString(s.To) {
return fmt.Errorf("%s: `to` is a module name or seat:<seat name> (novox/hq ADR 0468 §8)", at)
}
if seen[s.To] {
return fmt.Errorf("%s: a second entry for the same `to` (novox/hq ADR 0468 §8)", at)
}
seen[s.To] = true
if len(s.Methods) == 0 {
return fmt.Errorf("%s: `methods` is empty; name get, put or both (novox/hq ADR 0468 §8)", at)
}
used := map[string]bool{}
for _, method := range s.Methods {
if method != "get" && method != "put" {
return fmt.Errorf("%s: method %q is not get or put (novox/hq ADR 0468 §8)", at, method)
}
if used[method] {
return fmt.Errorf("%s: method %q twice (novox/hq ADR 0468 §8)", at, method)
}
used[method] = true
}
}
return nil
}
// readRequireShares reads the stored form's side key.
func readRequireShares(raw json.RawMessage) (map[string][]BucketShare, error) {
var out map[string][]BucketShare
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("%s: an object of requirement to a list of {to, methods}: %w", RequireSharesKey, err)
}
for requirement, shares := range out {
if err := checkBucketShares(requirement, shares); err != nil {
return nil, err
}
}
return out, nil
}
// sortedShares is a requirement's shares as they are handed to the provider: sorted by `to` (a file whose lines move
// looks changed), each entry's methods sorted, and a copy so the manifest is not reordered.
func sortedShares(shares []BucketShare) []BucketShare {
out := make([]BucketShare, 0, len(shares))
for _, s := range shares {
methods := append([]string(nil), s.Methods...)
sort.Strings(methods)
out = append(out, BucketShare{To: s.To, Methods: methods})
}
sort.Slice(out, func(i, j int) bool { return out[i].To < out[j].To })
return out
}
// withShares adds this module's shares of the requirement to the values it gives the provider. A module whose own
// `contributes` already names `shares` is refused: the field is the only way to say it (ADR 0468 §8).
func withShares(m Manifest, requirement string, values map[string]any) (map[string]any, error) {
shares := m.RequireShares[requirement]
if _, named := values[SharesValue]; named && requirement == BucketRequirement {
return nil, fmt.Errorf("%s contributing to %s names `shares`; write it on its requirement instead (novox/hq ADR 0468 §8)",
m.Module, requirement)
}
if len(shares) == 0 {
return values, nil
}
out := make(map[string]any, len(values)+1)
for k, v := range values {
out[k] = v
}
listed := []any{}
for _, s := range sortedShares(shares) {
methods := []any{}
for _, method := range s.Methods {
methods = append(methods, method)
}
listed = append(listed, map[string]any{"to": s.To, "methods": methods})
}
out[SharesValue] = listed
return out, nil
}
@@ -1783,6 +1783,10 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
if err != nil {
return nil, err
}
// What the module shares of its bucket reaches the provider with the rest (novox/hq ADR 0468 §8).
if values, err = withShares(m, to, values); err != nil {
return nil, err
}
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
}
// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
@@ -2169,11 +2173,19 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti
mine = append(mine, g.Values)
}
}
var asker Manifest
for _, m := range r.Modules {
if m.Module == module {
asker = m
}
}
if len(mine) == 1 {
// Its shares are already in, put there where its contribution was composed (contributions).
return mine[0], true, nil
}
if len(mine) > 1 {
return map[string]any{}, true, nil
values, err := withShares(asker, requirement, map[string]any{})
return values, err == nil, err
}
// It contributes no payload — but a require-only consumer of a parameterless provision (one whose
// `serves` names no consumer-supplied key: `redis-cache`, `amqp`) still ASKS for it and must be
@@ -2186,7 +2198,8 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti
}
for _, req := range m.Requires {
if req == requirement {
return map[string]any{}, true, nil
values, err := withShares(m, requirement, map[string]any{})
return values, err == nil, err
}
}
}
+46 -2
View File
@@ -494,6 +494,9 @@ type Manifest struct {
// by UnmarshalJSON from the long form of a `requires` entry, {name, optional}, or from the stored
// form's `optional-requires` list; never written by hand.
OptionalRequires map[string]bool `json:"-"`
// RequireShares is who else may use the bucket a module requires, by requirement (BucketShare, novox/hq ADR 0468
// §8): filled from the long form's `shares`, or from the stored form's `requires-shares`; never written by hand.
RequireShares map[string][]BucketShare `json:"-"`
// Emits are the events this module publishes, named **locally**: `order.placed`, not a subject
// and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the
@@ -1538,8 +1541,9 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
delete(keys, "contributes")
}
var optionalRequires map[string]bool
var requireShares map[string][]BucketShare
if requires, ok := keys["requires"]; ok && string(requires) != "null" {
names, optional, err := readRequires(requires)
names, optional, shares, err := readRequires(requires)
if err != nil {
return err
}
@@ -1549,6 +1553,23 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
}
keys["requires"] = plainNames
optionalRequires = optional
requireShares = shares
}
// The stored form's shares (RequireSharesKey).
if stored, ok := keys[RequireSharesKey]; ok {
if string(stored) != "null" {
stood, err := readRequireShares(stored)
if err != nil {
return err
}
for requirement, shares := range stood {
if requireShares == nil {
requireShares = map[string][]BucketShare{}
}
requireShares[requirement] = shares
}
}
delete(keys, RequireSharesKey)
}
// The stored form: `requires` plain, and which are optional beside it (OptionalRequiresKey).
if stored, ok := keys[OptionalRequiresKey]; ok {
@@ -1599,6 +1620,7 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
m.unknown = unknown
m.forms = forms
m.OptionalRequires = optionalRequires
m.RequireShares = requireShares
if len(plain) > 0 {
m.Secrets = plain
}
@@ -1611,6 +1633,21 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
if len(contributesMany) > 0 {
m.ContributesMany = contributesMany
}
if err := m.checkSharesNotContributed(); err != nil {
return err
}
return nil
}
// checkSharesNotContributed refuses a module whose own `contributes` to s3-bucket names `shares` (ADR 0468 §8).
func (m Manifest) checkSharesNotContributed() error {
if len(m.RequireShares[BucketRequirement]) == 0 {
return nil
}
if _, named := m.Contributes[BucketRequirement][SharesValue]; named {
return fmt.Errorf("contributes.%s names `shares` and so does its requirement; keep one, the requirement's (novox/hq ADR 0468 §8)",
BucketRequirement)
}
return nil
}
@@ -1655,7 +1692,7 @@ func (m Manifest) marshalFields() ([]byte, error) {
if err != nil {
return nil, err
}
if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 && len(m.OptionalRequires) == 0 {
if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 && len(m.OptionalRequires) == 0 && len(m.RequireShares) == 0 {
return raw, nil
}
var keys map[string]json.RawMessage
@@ -1671,6 +1708,13 @@ func (m Manifest) marshalFields() ([]byte, error) {
}
keys[OptionalRequiresKey] = listed
}
if len(m.RequireShares) > 0 {
listed, err := json.Marshal(m.RequireShares)
if err != nil {
return nil, err
}
keys[RequireSharesKey] = listed
}
if len(m.ContributesMany) > 0 {
mergedContributes := map[string]any{}
for to, values := range m.Contributes {
@@ -32,17 +32,20 @@ const OnlyWhenBound = "only-when-bound"
type requirementForm struct {
Name string `json:"name"`
Optional bool `json:"optional,omitempty"`
// Shares is who else may use the consumer's bucket (BucketShare, novox/hq ADR 0468 §8), on s3-bucket only.
Shares []BucketShare `json:"shares,omitempty"`
}
// readRequires reads `requires` in both of its forms — a bare name, or {name, optional} — into the
// names and which of them are optional.
func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
func readRequires(raw json.RawMessage) ([]string, map[string]bool, map[string][]BucketShare, error) {
var entries []json.RawMessage
if err := json.Unmarshal(raw, &entries); err != nil {
return nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
return nil, nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
}
var names []string
optional := map[string]bool{}
var shares map[string][]BucketShare
for i, e := range entries {
var plain string
if err := json.Unmarshal(e, &plain); err == nil {
@@ -53,12 +56,21 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
dec := json.NewDecoder(bytes.NewReader(e))
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
return nil, nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
}
if strings.TrimSpace(full.Name) == "" {
return nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
return nil, nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
}
names = append(names, full.Name)
if err := checkBucketShares(full.Name, full.Shares); err != nil {
return nil, nil, nil, fmt.Errorf("requires[%d]: %w", i, err)
}
if len(full.Shares) > 0 {
if shares == nil {
shares = map[string][]BucketShare{}
}
shares[full.Name] = full.Shares
}
if full.Optional {
optional[full.Name] = true
}
@@ -66,7 +78,7 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
if len(optional) == 0 {
optional = nil
}
return names, optional, nil
return names, optional, shares, nil
}
// OptionalRequiresKey is where a stored manifest keeps which of its requirements are optional: beside
@@ -324,7 +324,7 @@ func settingKeysUsedBy(m Manifest) map[string]bool {
return used
}
// settingKeysAskedOutsideOwners is every key a module's files, units, contributions and served facts
// settingKeysAskedOutsideOwners is every key a module's files, units, items, contributions and served facts
// ask for: what a directory's owner may follow (OwnerSettingProblems).
func settingKeysAskedOutsideOwners(m Manifest) map[string]bool {
used := map[string]bool{}
@@ -345,6 +345,10 @@ func settingKeysAskedOutsideOwners(m Manifest) map[string]bool {
}
}
}
// An item's text, filled from its template (WithTemplatesFrom), places a key as a file's does (issue 1658).
for _, it := range m.Items {
note(it.Content)
}
inValues := func(values map[string]any) {
for _, v := range values {
if s, ok := v.(string); ok {
@@ -20,6 +20,16 @@ type BuildRecord struct {
At time.Time `json:"at,omitzero"`
SourceFingerprint string `json:"source-fingerprint,omitempty"`
SourceInputs *SourceInputs `json:"source-inputs,omitempty"`
// Made is what the build made (Build.Made, records.go:110); the digest is the tail of an artifact's Reference.
Made []Artifact `json:"made,omitempty"`
}
// Artifact is one thing a build made: mesh-controller internal/chain/records.go Artifact, json keys name, kind and
// reference, nothing else (novox/hq ADR 0478 clause 7).
type Artifact struct {
Name string `json:"name"`
Kind string `json:"kind"`
Reference string `json:"reference"`
}
// BuildsBucket is the bucket the builds are kept in: mesh-controller internal/broker/controller_buckets.go:101,
+95 -14
View File
@@ -377,15 +377,46 @@ func (c Candidate) check() error {
// Manifest, the digest of the module's manifest, the key of the manifests bucket. A value written before has none of
// the three, and a reader of today's shape ignores them.
type Module struct {
V int `json:"v"`
FP string `json:"fp,omitempty"`
None string `json:"none,omitempty"`
FP2 string `json:"fp2,omitempty"`
None2 string `json:"none2,omitempty"`
Manifest string `json:"manifest,omitempty"`
Src uint64 `json:"src,omitempty"`
Repository string `json:"repository"`
Commit string `json:"commit"`
V int `json:"v"`
FP string `json:"fp,omitempty"`
None string `json:"none,omitempty"`
FP2 string `json:"fp2,omitempty"`
None2 string `json:"none2,omitempty"`
Manifest string `json:"manifest,omitempty"`
// Inputs2 are the new-form inputs the merge gave the module, kept so the indexer computes FP2 again when a build
// module's toolchain versions appear, with no merge (novox/hq ADR 0465 clause 2). It needs Manifest.
Inputs2 *KeptInputs2 `json:"inputs2,omitempty"`
Src uint64 `json:"src,omitempty"`
Repository string `json:"repository"`
Commit string `json:"commit"`
// Path is the module's directory in its repository, trimmed of slashes, and RootPath (".") when its manifest is the
// repository's root (novox/hq ADR 0478 clause 8, issue 1615): the build tools need it. Empty means not known: a value
// written before the indexer learned paths has none.
Path string `json:"path,omitempty"`
// Toolchains are the toolchains FP2 was computed with, by language (go, typescript): the version installed and the
// build module's recipe version the build nodes agreed on then (novox/hq ADR 0477 clause 2, ADR 0478 clause 3). A builder
// builds the fingerprint only with these. Empty when FP2 is empty or the module has no toolchain; a value written
// before has none (issue 1615).
Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"`
}
// RootPath is Module.Path of a module whose manifest is its repository's root.
const RootPath = "."
// KeptInputs2 are SourceInputs2 without the module's name (the key has it) and without toolchains (those come from
// the build modules' toolchain keys when the fingerprint is computed): what a Module keeps of a merge.
type KeptInputs2 struct {
Tree string `json:"tree"`
Bases []string `json:"bases,omitempty"`
Builder string `json:"builder,omitempty"`
Contexts map[string]string `json:"contexts,omitempty"`
Locks map[string]string `json:"locks,omitempty"`
}
// With is the full inputs of module m: the kept ones and the given toolchains.
func (k KeptInputs2) With(module string, toolchains map[string]Toolchain2) SourceInputs2 {
return SourceInputs2{Module: module, Tree: k.Tree, Bases: k.Bases, Builder: k.Builder, Contexts: k.Contexts,
Locks: k.Locks, Toolchains: toolchains}
}
func (m Module) check() error {
@@ -399,6 +430,14 @@ func (m Module) check() error {
case m.FP2 != "" && m.None2 != "":
return fmt.Errorf("delivery: a module has both fp2 and none2")
}
for language, t := range m.Toolchains {
if t.Version == "" || t.Recipe == "" {
return fmt.Errorf("delivery: the module's toolchain of %q needs its version and its recipe version", language)
}
}
if m.Inputs2 != nil && m.Manifest == "" {
return fmt.Errorf("delivery: a module keeps new-form inputs but has no manifest digest")
}
return checkNewForm(m.FP2, m.Manifest)
}
@@ -421,9 +460,14 @@ type Forge struct {
Merge string `json:"merge,omitempty"`
Trees map[string]string `json:"trees,omitempty"`
Root string `json:"root,omitempty"`
Act string `json:"act,omitempty"`
Words string `json:"words,omitempty"`
Open []OpenPull `json:"open,omitempty"`
// Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message,
// of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path
// relative to it (novox/hq ADR 0465 clause 1). Older writers send neither.
Manifests map[string]string `json:"manifests,omitempty"`
Locks map[string]map[string]string `json:"locks,omitempty"`
Act string `json:"act,omitempty"`
Words string `json:"words,omitempty"`
Open []OpenPull `json:"open,omitempty"`
}
func (f Forge) check() error {
@@ -455,6 +499,13 @@ func (f Forge) check() error {
if len(f.Dirs) > 0 && !f.DirsKnown {
return fmt.Errorf("delivery: a pull message with module directories must say it knows them (dirs-known)")
}
dirs := map[string]bool{}
for _, d := range f.Dirs {
dirs[d] = true
}
if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil {
return err
}
case ForgeMerged:
if len(f.Dirs) > 0 || f.DirsKnown {
return fmt.Errorf("delivery: a merged message carries no module directories")
@@ -479,10 +530,16 @@ func (f Forge) check() error {
return fmt.Errorf("delivery: a merged message's tree of the directory %q is empty", dir)
}
}
if err := f.checkInputs(func(d string) bool { _, ok := f.Trees[d]; return ok }, "its trees"); err != nil {
return err
}
case ForgeRefused:
if len(f.Dirs) > 0 || f.DirsKnown {
return fmt.Errorf("delivery: a refused message carries no module directories")
}
if len(f.Manifests) > 0 || len(f.Locks) > 0 {
return fmt.Errorf("delivery: a refused message carries no manifests or lock hashes")
}
if f.Act == "" {
return fmt.Errorf("delivery: a refused message needs the act it answers")
}
@@ -493,11 +550,35 @@ func (f Forge) check() error {
return nil
}
// checkInputs refuses a manifest or a lock hash of a directory that known does not know (of is what it knows them
// by), an empty manifest and an empty lock path or hash, naming the directory.
func (f Forge) checkInputs(known func(dir string) bool, of string) error {
for dir, m := range f.Manifests {
if !known(dir) {
return fmt.Errorf("delivery: the manifest of the directory %q is not of %s", dir, of)
}
if m == "" {
return fmt.Errorf("delivery: the manifest of the directory %q is empty", dir)
}
}
for dir, locks := range f.Locks {
if !known(dir) {
return fmt.Errorf("delivery: the lock hashes of the directory %q are not of %s", dir, of)
}
for path, hash := range locks {
if path == "" || hash == "" {
return fmt.Errorf("delivery: the lock %q of the directory %q has an empty path or hash", path, dir)
}
}
}
return nil
}
// checkOpen refuses an open message that names a repository, a branch, a head or anything of the other kinds, or
// an open pull request that is not `<owner>/<name>` and a number.
func (f Forge) checkOpen() error {
if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown ||
f.Merge != "" || len(f.Trees) > 0 || f.Root != "" || f.Act != "" || f.Words != "" {
f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || f.Root != "" || f.Act != "" || f.Words != "" {
return fmt.Errorf("delivery: an open message carries only the open pull requests it names")
}
for _, p := range f.Open {
@@ -603,7 +684,7 @@ func SameSource(a, b Source) bool { return reflect.DeepEqual(a, b) }
func SameCandidate(a, b Candidate) bool { return reflect.DeepEqual(a, b) }
// SameModule says whether two modules are the same value.
func SameModule(a, b Module) bool { return a == b }
func SameModule(a, b Module) bool { return reflect.DeepEqual(a, b) }
// SameForge says whether two forge messages are the same value.
func SameForge(a, b Forge) bool { return reflect.DeepEqual(a, b) }
+116
View File
@@ -0,0 +1,116 @@
package delivery
import (
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
)
// StepsState is the name of the state a build module keeps its long steps in; its bucket is `<module>_steps`, built
// by StepsBucket (novox/hq ADR 0469 clause 2; issue 1471). A tool call is cut at 30 s, so a fetch, a test level or a
// build answers at once with a step id and writes its progress and its end to the key `<node>.<step id>`.
const StepsState = "steps"
// StepsBucket is the bucket of a build module's steps.
func StepsBucket(module string) string { return module + "_" + StepsState }
// The states of a step. A step ends in passed, failed or refused; its key is written for each of them, and that
// write is the event the step's reader waits on.
const (
StepRunning = "running"
StepPassed = "passed"
StepFailed = "failed"
StepRefused = "refused"
)
var stepIDOK = regexp.MustCompile(`^[a-z0-9]+$`)
func checkStepID(id string) error {
if !stepIDOK.MatchString(id) {
return fmt.Errorf("delivery: the step id %q is not lower-case letters and digits", id)
}
return nil
}
// StepKey is the key of a node's step: `<node>.<id>`. It refuses a node that is not a name and an id that is not
// lower-case letters and digits (one key token, never a dot).
func StepKey(node, id string) (string, error) {
if err := checkName("node", node); err != nil {
return "", err
}
if err := checkStepID(id); err != nil {
return "", err
}
return node + "." + id, nil
}
// ParseStepKey is the inverse of StepKey.
func ParseStepKey(key string) (node, id string, err error) {
node, id, ok := strings.Cut(key, ".")
if !ok {
return "", "", fmt.Errorf("delivery: %q is not a step key, <node>.<id>", key)
}
if _, err := StepKey(node, id); err != nil {
return "", "", fmt.Errorf("delivery: %q is not a step key: %w", key, err)
}
return node, id, nil
}
// Step is the value of a steps key: which tool runs on which node, how far it is, when it started and ended (UTC, as
// the toolchains' At), why it was refused or failed, and what it returned.
type Step struct {
V int `json:"v"`
ID string `json:"id"`
Tool string `json:"tool"`
Node string `json:"node"`
State string `json:"state"`
Started time.Time `json:"started"`
Ended time.Time `json:"ended,omitzero"`
Reason string `json:"reason,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
}
func (s Step) check() error {
if err := checkStepID(s.ID); err != nil {
return err
}
if err := checkName("node", s.Node); err != nil {
return err
}
if s.Tool == "" {
return fmt.Errorf("delivery: a step needs its tool")
}
switch s.State {
case StepRunning, StepPassed, StepFailed, StepRefused:
default:
return fmt.Errorf("delivery: the step state %q is not running, passed, failed or refused", s.State)
}
if len(s.Result) > 0 && !json.Valid(s.Result) {
return fmt.Errorf("delivery: the step's result is not JSON")
}
return nil
}
// EncodeStep is s as JSON at Version; it refuses an id that is not one key token, a node that is not a name, an empty
// tool, a state outside the four and a result that is not JSON.
func EncodeStep(s Step) ([]byte, error) {
if err := s.check(); err != nil {
return nil, err
}
s.V = Version
return json.Marshal(s)
}
// DecodeStep reads a steps value; it refuses an unknown version (ErrUnknownVersion) and what EncodeStep refuses.
func DecodeStep(data []byte) (Step, error) {
var s Step
if err := decode(data, &s); err != nil {
return Step{}, err
}
if err := s.check(); err != nil {
return Step{}, err
}
return s, nil
}
@@ -0,0 +1,127 @@
package delivery
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
)
// ToolchainsState is the name of the state a build module keeps its toolchains in; its bucket is
// `<module>_toolchains`, built as the SDK names a module's state, by ToolchainsBucket (novox/hq ADR 0465 clause 2).
const ToolchainsState = "toolchains"
// ToolchainsBucket is the bucket of a build module's toolchains.
func ToolchainsBucket(module string) string { return module + "_" + ToolchainsState }
// ToolchainNoReport is the reason AgreedToolchain gives when no node has reported a language's toolchain.
const ToolchainNoReport = "toolchain not yet reported"
func checkToolchainLanguage(language string) error {
switch language {
case "go", "typescript":
return nil
}
return fmt.Errorf("delivery: the toolchain language %q is not go or typescript", language)
}
// ToolchainKey is the key of a node's toolchain of one language: `<node>.<language>`. It refuses a node that is not a
// name and a language other than go or typescript.
func ToolchainKey(node, language string) (string, error) {
if err := checkName("node", node); err != nil {
return "", err
}
if err := checkToolchainLanguage(language); err != nil {
return "", err
}
return node + "." + language, nil
}
// ParseToolchainKey is the inverse of ToolchainKey.
func ParseToolchainKey(key string) (node, language string, err error) {
node, language, ok := strings.Cut(key, ".")
if !ok {
return "", "", fmt.Errorf("delivery: %q is not a toolchain key, <node>.<language>", key)
}
if _, err := ToolchainKey(node, language); err != nil {
return "", "", fmt.Errorf("delivery: %q is not a toolchain key: %w", key, err)
}
return node, language, nil
}
// Toolchain is the value of a toolchains key: the version of the language's toolchain installed on the node and the
// build module's recipe version, and when the module saw them.
type Toolchain struct {
V int `json:"v"`
Language string `json:"language"`
Version string `json:"version"`
Recipe string `json:"recipe"`
At time.Time `json:"at"`
}
func (t Toolchain) check() error {
if err := checkToolchainLanguage(t.Language); err != nil {
return err
}
if t.Version == "" || t.Recipe == "" {
return fmt.Errorf("delivery: a %s toolchain needs its version and its recipe", t.Language)
}
return nil
}
// EncodeToolchain is t as JSON at Version; it refuses a language other than go or typescript and an empty version or
// recipe.
func EncodeToolchain(t Toolchain) ([]byte, error) {
if err := t.check(); err != nil {
return nil, err
}
t.V = Version
return json.Marshal(t)
}
// DecodeToolchain reads a toolchains value; it refuses an unknown version (ErrUnknownVersion) and what EncodeToolchain
// refuses.
func DecodeToolchain(data []byte) (Toolchain, error) {
var t Toolchain
if err := decode(data, &t); err != nil {
return Toolchain{}, err
}
if err := t.check(); err != nil {
return Toolchain{}, err
}
return t, nil
}
// NodeToolchain is one node's entry of a language, as read from the toolchains bucket.
type NodeToolchain struct {
Node string
Toolchain
}
// AgreedToolchain is the one Toolchain2 every node's entry of a language agrees on, or none and why: "toolchain not
// yet reported" with no entries, and "builders disagree" naming each differing version and recipe with the nodes
// reporting it when two differ (novox/hq ADR 0405 clause 2). The answer does not depend on the entries' order. The
// entries must all be of one language.
func AgreedToolchain(entries []NodeToolchain) (tc Toolchain2, none string) {
if len(entries) == 0 {
return Toolchain2{}, ToolchainNoReport
}
groups := map[Toolchain2][]string{}
for _, e := range entries {
k := Toolchain2{Language: e.Language, Version: e.Version, Recipe: e.Recipe}
groups[k] = append(groups[k], e.Node)
}
if len(groups) == 1 {
for k := range groups {
return k, ""
}
}
parts := make([]string, 0, len(groups))
for k, nodes := range groups {
sort.Strings(nodes)
parts = append(parts, fmt.Sprintf("%s recipe %s on %s", k.Version, k.Recipe, strings.Join(nodes, ", ")))
}
sort.Strings(parts)
return Toolchain2{}, "builders disagree: " + strings.Join(parts, "; ")
}
+1 -1
View File
@@ -1,4 +1,4 @@
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020085642-ab0c16ddad26
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861
## explicit; go 1.26.0
git.novox.be/novox/mesh-sdk/go/asks
git.novox.be/novox/mesh-sdk/go/catalogue