Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c77d17fb42 | ||
|
|
8311ecb849 | ||
|
|
ba8b8f28a3 | ||
|
|
8af6e72669 | ||
|
|
62cddf4efc | ||
|
|
19c204273b | ||
|
|
a20bdf7952 |
+19
-2
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -42,14 +43,30 @@ type AskSteps struct {
|
||||
// RunAsk runs ask's job with steps and answers a delivery.BuildAnswer or a delivery.CheckAnswer.
|
||||
func RunAsk(ctx context.Context, steps AskSteps, ask delivery.StepAsk) (any, error) {
|
||||
if ask.Build != nil {
|
||||
return steps.build(ctx, *ask.Build)
|
||||
job := *ask.Build
|
||||
job.Fetch = withPackages(job.Fetch, ask.Packages)
|
||||
return steps.build(ctx, job)
|
||||
}
|
||||
if ask.Check != nil {
|
||||
return steps.check(ctx, *ask.Check)
|
||||
job := *ask.Check
|
||||
job.Fetch = withPackages(job.Fetch, ask.Packages)
|
||||
return steps.check(ctx, job)
|
||||
}
|
||||
return nil, Refuse("the ask %s holds no job", ask.ID)
|
||||
}
|
||||
|
||||
// withPackages is the fetch's arguments with the ask's package versions (novox/hq ADR 0573 §2), which the build module
|
||||
// writes into the workspace it fetches; a copy, so the ask is left as it was. An ask with none, or a copy alone that
|
||||
// fetches nothing, is left as it is.
|
||||
func withPackages(fetch map[string]any, packages delivery.PackageVersions) map[string]any {
|
||||
if len(packages) == 0 || fetch == nil {
|
||||
return fetch
|
||||
}
|
||||
out := maps.Clone(fetch)
|
||||
out["packages"] = maps.Clone(packages)
|
||||
return out
|
||||
}
|
||||
|
||||
func raw(v any) (json.RawMessage, error) {
|
||||
if r, ok := v.(json.RawMessage); ok {
|
||||
return r, nil
|
||||
|
||||
@@ -74,15 +74,21 @@ func TestRunAskBuilds(t *testing.T) {
|
||||
steps := r.steps()
|
||||
var released []string
|
||||
steps.Release = func(ws string) error { released = append(released, ws); return nil }
|
||||
out, err := RunAsk(context.Background(), steps, ask(delivery.BuildJob{Module: "m", Path: "modules/m", Publish: true,
|
||||
// Issue 1585: the ask's package versions reach the fetch, and the ask's own fetch arguments stay as they were.
|
||||
a := ask(delivery.BuildJob{Module: "m", Path: "modules/m", Publish: true,
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "base", From: "registry/b@sha256:00"}, {Artifact: "bundle"}, {Artifact: "lib"}}}))
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "base", From: "registry/b@sha256:00"}, {Artifact: "bundle"}, {Artifact: "lib"}}})
|
||||
a.Packages = delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": runCommit}
|
||||
out, err := RunAsk(context.Background(), steps, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, put := a.Build.Fetch["packages"]; put {
|
||||
t.Error("the ask's fetch arguments were changed")
|
||||
}
|
||||
got := strings.Join(r.calls, "\n")
|
||||
want := `copy {"artifact":"base","from":"registry/b@sha256:00","module":"m"}
|
||||
fetch {"commit":"` + runCommit + `","repository":"novox/x"}
|
||||
fetch {"commit":"` + runCommit + `","packages":{"git.novox.be/novox/mesh-sdk/go":"` + runCommit + `"},"repository":"novox/x"}
|
||||
build {"artifact":"bundle","module":"modules/m","publish":true,"workspace":"/w/1"}
|
||||
build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"}`
|
||||
if got != want {
|
||||
@@ -91,9 +97,9 @@ build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"}`
|
||||
if len(released) != 1 || released[0] != "/w/1" {
|
||||
t.Errorf("released %v, want the one workspace", released)
|
||||
}
|
||||
a := out.(delivery.BuildAnswer)
|
||||
if len(a.Artifacts) != 3 || a.Artifacts[1].Reference != "artifact-store://bundle" {
|
||||
t.Fatalf("answer %+v", a)
|
||||
answer := out.(delivery.BuildAnswer)
|
||||
if len(answer.Artifacts) != 3 || answer.Artifacts[1].Reference != "artifact-store://bundle" {
|
||||
t.Fatalf("answer %+v", answer)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,9 +2,12 @@ package buildstep
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -27,6 +30,10 @@ type FetchRequest struct {
|
||||
Base string `json:"base,omitempty"`
|
||||
Dir string `json:"dir,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
// Packages are the version of each mesh package the build or the check builds against (delivery.StepAsk.Packages,
|
||||
// novox/hq ADR 0573 §2, §4): the build module writes each into the workspace's go.mod and go.sum, whatever version
|
||||
// the repository names, checksum-verified, and names them in its log.
|
||||
Packages delivery.PackageVersions `json:"packages,omitempty"`
|
||||
}
|
||||
|
||||
// FetchRequestProperties are the JSON schema properties of FetchRequest's dir and paths, for a fetch tool's Input.
|
||||
@@ -35,6 +42,8 @@ func FetchRequestProperties() map[string]any {
|
||||
"dir": map[string]any{"type": "string", "description": "the module's directory in the repository (\".\" for its root), with paths"},
|
||||
"paths": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "the paths the module names, relative to dir, from the indexer's kept inputs: the build sees only these, the manifest and its lock files"},
|
||||
"packages": map[string]any{"type": "object", "additionalProperties": map[string]any{"type": "string"},
|
||||
"description": "optional: the version (a full commit) of each mesh package to build against, by its module path, whatever version go.mod names (novox/hq ADR 0573 §2); the builder and the checker fill it"},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +58,12 @@ func (r FetchRequest) Args() map[string]any {
|
||||
if len(r.Paths) > 0 {
|
||||
args["paths"] = append([]string(nil), r.Paths...)
|
||||
}
|
||||
if len(r.Packages) > 0 {
|
||||
args["packages"] = map[string]any{}
|
||||
for p, v := range r.Packages {
|
||||
args["packages"].(map[string]any)[p] = v
|
||||
}
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
@@ -78,9 +93,42 @@ func ParseFetchRequest(args map[string]any) (FetchRequest, error) {
|
||||
default:
|
||||
return FetchRequest{}, fmt.Errorf("paths is a list of strings")
|
||||
}
|
||||
packages, err := PackagesArg(args["packages"])
|
||||
if err != nil {
|
||||
return FetchRequest{}, err
|
||||
}
|
||||
r.Packages = packages
|
||||
return r, r.Check()
|
||||
}
|
||||
|
||||
// PackagesArg reads a tool call's packages argument, an object of strings (as JSON gives it) or the map an ask gives;
|
||||
// nil for none.
|
||||
func PackagesArg(v any) (delivery.PackageVersions, error) {
|
||||
out := delivery.PackageVersions{}
|
||||
switch v := v.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case delivery.PackageVersions:
|
||||
maps.Copy(out, v)
|
||||
case map[string]string:
|
||||
maps.Copy(out, v)
|
||||
case map[string]any:
|
||||
for p, version := range v {
|
||||
s, ok := version.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("packages is an object of versions (strings), by package")
|
||||
}
|
||||
out[p] = s
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("packages is an object of versions (strings), by package")
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return out, out.Check()
|
||||
}
|
||||
|
||||
// Check is what ParseFetchRequest refuses.
|
||||
func (r FetchRequest) Check() error {
|
||||
if !requestRepositoryOK.MatchString(r.Repository) || strings.Contains(r.Repository, "..") {
|
||||
|
||||
@@ -10,11 +10,15 @@ import (
|
||||
// from the indexer's kept inputs and the build module reads them from the call, never from the manifest (ADR 0495
|
||||
// clause 1). The request survives the trip through a tool call's arguments.
|
||||
func TestAFetchRequestCarriesTheNamedPathsThroughTheToolCall(t *testing.T) {
|
||||
// And the versions of the mesh packages to build against (novox/hq ADR 0573 §2; issue 1585 S4).
|
||||
req := FetchRequest{Repository: "novox/mesh-sdk", Commit: strings.Repeat("a", 40), Ref: "refs/pull/7/head",
|
||||
Dir: ".", Paths: []string{"package.json", "src"}}
|
||||
Dir: ".", Paths: []string{"package.json", "src"}, Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": strings.Repeat("d", 40)}}
|
||||
args := req.Args()
|
||||
// A tool call's arguments arrive as JSON: lists of any.
|
||||
// A tool call's arguments arrive as JSON: lists of any, objects of any.
|
||||
args["paths"] = []any{"package.json", "src"}
|
||||
if p, ok := args["packages"].(map[string]any); !ok || p["git.novox.be/novox/mesh-sdk/go"] != strings.Repeat("d", 40) {
|
||||
t.Fatalf("the packages are not in the call's arguments: %v", args["packages"])
|
||||
}
|
||||
got, err := ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -42,6 +46,9 @@ func TestAFetchRequestCarriesTheNamedPathsThroughTheToolCall(t *testing.T) {
|
||||
{map[string]any{"dir": "../a", "paths": []any{"src"}}, "leaves the repository"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{3}}, "paths is a list of strings"},
|
||||
{map[string]any{"dir": "modules/a"}, "names a module directory and no path"},
|
||||
{map[string]any{"packages": map[string]any{"git.novox.be/novox/mesh-sdk/go": "v0.1.7"}}, "no full commit"},
|
||||
{map[string]any{"packages": map[string]any{"a b": strings.Repeat("d", 40)}}, "no module path"},
|
||||
{map[string]any{"packages": []any{"x"}}, "packages is an object"},
|
||||
} {
|
||||
args := map[string]any{}
|
||||
for k, v := range ok {
|
||||
|
||||
@@ -25,6 +25,8 @@ type ModuleTree struct {
|
||||
// Paths are, of a module whose every artifact names its paths, each named path's git object id at the commit; Tree
|
||||
// is then their delivery.NamedTree with Manifest (novox/hq ADR 0499 clause 2). Absent otherwise.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
// Packages are the mesh packages the directory's go.sum names (delivery.MeshPackagesOf, novox/hq ADR 0573 §2).
|
||||
Packages []string `json:"packages,omitempty"`
|
||||
}
|
||||
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
@@ -67,6 +69,9 @@ func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, co
|
||||
mt.Locks = map[string]string{}
|
||||
}
|
||||
mt.Locks[lock] = sha256Hex(raw)
|
||||
if lock == "go.sum" {
|
||||
mt.Packages = delivery.MeshPackagesOf(raw) // the mesh packages it uses (novox/hq ADR 0573 §2)
|
||||
}
|
||||
}
|
||||
}
|
||||
trees[mdir] = mt
|
||||
|
||||
@@ -33,7 +33,8 @@ func TestModuleTreesAnswersTreeManifestAndLocksPerModuleDirectory(t *testing.T)
|
||||
writeTreeFile(t, root, "module.json", `{"root":1}`)
|
||||
writeTreeFile(t, root, "go.sum", "rootsum")
|
||||
writeTreeFile(t, root, "modules/a/module.json", `{"a":1}`)
|
||||
writeTreeFile(t, root, "modules/a/go.sum", "asum")
|
||||
asum := "git.novox.be/novox/mesh-sdk/go v0.1.7 h1:x=\ngit.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:y=\ngithub.com/nats-io/nats.go v1.0.0 h1:z=\n"
|
||||
writeTreeFile(t, root, "modules/a/go.sum", asum)
|
||||
writeTreeFile(t, root, "modules/a/package-lock.json", "alock")
|
||||
writeTreeFile(t, root, "modules/b/module.json", `{"b":1}`)
|
||||
merge := strings.Repeat("c", 40)
|
||||
@@ -55,8 +56,9 @@ func TestModuleTreesAnswersTreeManifestAndLocksPerModuleDirectory(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]ModuleTree{
|
||||
".": {Tree: "tree-of-" + merge + "^{tree}", Manifest: hexOf(`{"root":1}`), Locks: map[string]string{"go.sum": hexOf("rootsum")}},
|
||||
"modules/a": {Tree: "tree-of-" + merge + ":modules/a", Manifest: hexOf(`{"a":1}`), Locks: map[string]string{"go.sum": hexOf("asum"), "package-lock.json": hexOf("alock")}},
|
||||
".": {Tree: "tree-of-" + merge + "^{tree}", Manifest: hexOf(`{"root":1}`), Locks: map[string]string{"go.sum": hexOf("rootsum")}},
|
||||
"modules/a": {Tree: "tree-of-" + merge + ":modules/a", Manifest: hexOf(`{"a":1}`), Locks: map[string]string{"go.sum": hexOf(asum), "package-lock.json": hexOf("alock")},
|
||||
Packages: []string{"git.novox.be/novox/mesh-sdk/go"}}, // issue 1585: the mesh packages its go.sum names
|
||||
"modules/b": {Tree: "tree-of-" + merge + ":modules/b", Manifest: hexOf(`{"b":1}`)},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
|
||||
@@ -31,6 +31,7 @@ func parsed(t *testing.T, m Manifest) error {
|
||||
}
|
||||
|
||||
func TestAnUnsetPreferenceTakesItsDefault(t *testing.T) {
|
||||
t.Run("the node over the mesh over the default", theNodeOverTheMeshOverTheDefault)
|
||||
m := notifier()
|
||||
for _, layers := range [][]Layer{nil, {{From: MeshWideLayer, Values: map[string]any{}}}} {
|
||||
file := map[string]any{}
|
||||
@@ -49,7 +50,7 @@ func TestAnUnsetPreferenceTakesItsDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNodeOverTheMeshOverTheDefault(t *testing.T) {
|
||||
func theNodeOverTheMeshOverTheDefault(t *testing.T) {
|
||||
m := notifier()
|
||||
layers := []Layer{
|
||||
{From: MeshWideLayer, Values: map[string]any{"width": float64(300)}},
|
||||
|
||||
@@ -42,7 +42,6 @@ const (
|
||||
StreamMostDeliveries = 10
|
||||
)
|
||||
|
||||
|
||||
// MeshStreamNames are the mesh's own streams by name. Written here as well as in the broker, which this
|
||||
// package cannot import; a test keeps them agreeing.
|
||||
var MeshStreamNames = []string{"CONTROL", "NODES", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED", "CHECKS_READY", "CHECKS_OTHER"}
|
||||
|
||||
@@ -545,10 +545,16 @@ func PlansHeld(ctx context.Context, r Reader, mesh Mesh, s Scope) (plans map[str
|
||||
// plansHeldOf is the plans of node over in, every module in held held, and each module a refusal is of held too until
|
||||
// the plans compose (culprit's rule over the Planning mode: its stub first, then its bare stub).
|
||||
func plansHeldOf(in compose.Input, node string, held map[string]string) (map[string]catalogue.Plan, error) {
|
||||
return heldOf(in, node, held, func(in compose.Input) (map[string]catalogue.Plan, error) { return plansIn(in, node, compose.Planning) })
|
||||
}
|
||||
|
||||
// heldOf composes compose1 over in with every module in held held, and each module a refusal is of held too until it
|
||||
// composes (culprit's rule: its stub first, then its bare stub); what compose1 answers of a held module is left out.
|
||||
// Plans and data items are composed so (PlansHeld, DataItemsHeld): one module never ends the read (novox/hq ADR 0578 §3).
|
||||
func heldOf[V any](in compose.Input, node string, held map[string]string, compose1 func(compose.Input) (map[string]V, error)) (map[string]V, error) {
|
||||
for _, m := range sortedKeys(held) {
|
||||
in = stubbed(in, node, m, "")
|
||||
}
|
||||
compose1 := func(in compose.Input) (map[string]catalogue.Plan, error) { return plansIn(in, node, compose.Planning) }
|
||||
// Each try holds one module more or leaves out all of one held module's offer: bounded by twice the modules placed.
|
||||
for tries := 0; tries <= 2*len(assignedOn(in, node))+1; tries++ {
|
||||
plans, err := compose1(in)
|
||||
@@ -585,7 +591,7 @@ func plansHeldOf(in compose.Input, node string, held map[string]string) (map[str
|
||||
}
|
||||
in = stubbed(in, node, module, leave)
|
||||
}
|
||||
return nil, fmt.Errorf("frombus: the plans of %s do not compose with %d module(s) held: %s", node, len(held),
|
||||
return nil, fmt.Errorf("frombus: what %s composes does not compose with %d module(s) held: %s", node, len(held),
|
||||
strings.Join(sortedKeys(held), ", "))
|
||||
}
|
||||
|
||||
@@ -1057,9 +1063,34 @@ func DataItemsHeld(ctx context.Context, r Reader, mesh Mesh, s Scope) (items map
|
||||
if err != nil {
|
||||
return nil, nil, held, err
|
||||
}
|
||||
items, unplaced, err = compose.DataItems(in, s.Node)
|
||||
for m := range held {
|
||||
delete(items, m)
|
||||
}
|
||||
// Held module by module, as PlansHeld holds them: a refused stub or one module's composition refusal (an item's
|
||||
// template) holds that module and never ends the read.
|
||||
items, err = heldOf(in, s.Node, held, func(in compose.Input) (map[string][]catalogue.PlacedDataItem, error) {
|
||||
var err error
|
||||
items, unplaced, err = compose.DataItems(withoutHeldData(in, s.Node), s.Node)
|
||||
return items, err
|
||||
})
|
||||
return items, unplaced, held, err
|
||||
}
|
||||
|
||||
// withoutHeldData is in with no data item of a module held on node: none of a held module's items is measured, so its
|
||||
// data never refuses the rest (its stub keeps its data for the backups the plans compose).
|
||||
func withoutHeldData(in compose.Input, node string) compose.Input {
|
||||
out := in
|
||||
out.Nodes = append([]compose.Node(nil), in.Nodes...)
|
||||
for i := range out.Nodes {
|
||||
n := &out.Nodes[i]
|
||||
if n.Name != node || len(n.Held) == 0 {
|
||||
continue
|
||||
}
|
||||
kept := maps.Clone(n.Kept)
|
||||
for m := range n.Held {
|
||||
if man, ok := kept[m]; ok {
|
||||
man.Data = nil
|
||||
kept[m] = man
|
||||
}
|
||||
}
|
||||
n.Kept = kept
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/fromruntime"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -29,16 +31,25 @@ func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) {
|
||||
// refused is the indexer's verdict on bad, a problem its judge found (novox/hq ADR 0578 §1): Plans then holds
|
||||
// backup too, and never ends the read.
|
||||
refused string
|
||||
// dataHeld says the data items' read holds backup too: a refusal it meets (a refused manifest, an unusable build,
|
||||
// an item composition refuses), not one only a plan meets.
|
||||
dataHeld bool
|
||||
// dataOnly checks the data items' read alone: the other reads blame restic, whose templates collect the item.
|
||||
dataOnly bool
|
||||
}{
|
||||
{name: "every module composes"},
|
||||
{name: "a built manifest that cannot be used",
|
||||
bad: `{"module":"backup","version":"1","build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`,
|
||||
built: `{"module":"backup","version":"1","requires":"amqp"}`, why: "backup"},
|
||||
built: `{"module":"backup","version":"1","requires":"amqp"}`, why: "backup", dataHeld: true},
|
||||
{name: "a plan that refuses",
|
||||
bad: `{"module":"backup","version":"1","resources":[{"id":"conf","type":"file","path":"/etc/backup.conf",` +
|
||||
`"mode":"0600","content":"key=${secret:nope}"}]}`, why: "${secret:nope}"},
|
||||
{name: "an indexer's manifest the mesh refuses", bad: unusableItem, why: `"shows" is none of them`,
|
||||
refused: `"shows" is none of them`},
|
||||
refused: `"shows" is none of them`, dataHeld: true},
|
||||
// ADR 0578 §3: an accepted manifest whose data item its composition refuses is held by the culprit rule, in every read.
|
||||
{name: "a data item its composition refuses",
|
||||
bad: `{"module":"backup","version":"1","data":{"own":[{"id":"d","path":"${dir:nope}","class":"cache"}]}}`,
|
||||
why: "nope", dataHeld: true, dataOnly: true},
|
||||
{name: "an indexer's manifest the mesh refuses, placed elsewhere", bad: unusableItem, elsewhere: true,
|
||||
refused: `"shows" is none of them`},
|
||||
} {
|
||||
@@ -65,6 +76,13 @@ func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) {
|
||||
b.Builds["backup "+fp] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: builtDigest,
|
||||
Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://backup/files/blobs/x", Digest: "sha256:x"}}}
|
||||
}
|
||||
if row.dataOnly {
|
||||
items, _, iheld, err := frombus.DataItemsHeld(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node})
|
||||
if why, ok := iheld["backup"]; err != nil || !ok || !strings.Contains(why, row.why) || len(items["restic"]) == 0 {
|
||||
t.Errorf("%s: data items %v held %v: %v", row.name, items, iheld, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
plans, err := frombus.Plans(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node})
|
||||
switch {
|
||||
case row.refused != "" && (err != nil || !planned(plans, "restic") || planned(plans, "backup")):
|
||||
@@ -111,6 +129,43 @@ func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) {
|
||||
if _, planned := plans["backup"]; planned {
|
||||
t.Errorf("%s: the held module has a plan", row.name)
|
||||
}
|
||||
// The data items are read by the same rule (ADR 0578 §3: disk-load's read): the bad module held, the read whole.
|
||||
items, _, iheld, err := frombus.DataItemsHeld(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node})
|
||||
// A refusal only a plan meets (a file's secret) holds nothing here: data items render no such file.
|
||||
mustHold := row.dataHeld
|
||||
if why, ok := iheld["backup"]; err != nil || ok != mustHold || mustHold && !strings.Contains(why, row.why) || items["backup"] != nil {
|
||||
t.Errorf("%s: data items %v held %v: %v", row.name, items, iheld, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0580 clause 1: the watched reader reads a state it composes from out of the copy, a build by key each time it is
|
||||
// asked, and a manifest once per digest.
|
||||
byKey := map[string]int{}
|
||||
ws := stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
|
||||
byKey[state+" watched"]++
|
||||
return handle(stdio.StateChange{Key: "placement." + Node, Op: stdio.OpPut, Value: []byte(`{}`), Current: true})
|
||||
}, func(state, key string) ([]byte, bool, error) { byKey[state]++; return []byte(key), true, nil })
|
||||
w := fromruntime.Watched{WatchedStates: ws}
|
||||
for range 2 {
|
||||
_, _, _ = w.Get(frombus.BuildsRead, "restic."+fp)
|
||||
_, _, _ = w.Get(frombus.ManifestsRead, "sha256-d")
|
||||
if _, found, _ := w.Get(frombus.PlacementRead, "placement."+Node); !found {
|
||||
t.Error("the watched reader does not answer the placement from its copy")
|
||||
}
|
||||
}
|
||||
if byKey[frombus.BuildsRead] != 2 || byKey[frombus.ManifestsRead] != 1 || byKey[frombus.PlacementRead] != 0 ||
|
||||
byKey[frombus.PlacementRead+" watched"] != 1 {
|
||||
t.Errorf("the watched reader read %v, want a build by key each time, a manifest once, the placement watched once", byKey)
|
||||
}
|
||||
fetched := 0
|
||||
keep := fromruntime.KeptFetch(func(context.Context, string) ([]byte, error) { fetched++; return []byte("tar"), nil })
|
||||
for range 2 {
|
||||
if body, err := keep(context.Background(), "http://store/restic/files/blobs/sha256:a"); err != nil || string(body) != "tar" {
|
||||
t.Fatalf("the kept fetch answered %q, %v", body, err)
|
||||
}
|
||||
}
|
||||
if fetched != 1 {
|
||||
t.Errorf("a templates archive was fetched %d times, want once", fetched)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
@@ -28,9 +29,60 @@ func (States) Get(state, key string) ([]byte, bool, error) {
|
||||
// Keys are a state's keys.
|
||||
func (States) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// Reader is frombus's Reader over this bundle's read states, fetching archives over HTTP.
|
||||
// Reader is frombus's Reader over this bundle's read states key by key, fetching archives over HTTP. A holder that
|
||||
// composes again and again reads through Watched instead (novox/hq ADR 0580).
|
||||
func Reader() frombus.Reader { return frombus.StatesReader(States{}, HTTPFetch) }
|
||||
|
||||
// Watched is frombus's States over a watched copy (novox/hq ADR 0580 clause 1): the states a composition reads whole
|
||||
// (placement, haves, modules, resolved values) from the copy, a build by key when asked, a manifest once per digest.
|
||||
type Watched struct{ *stdio.WatchedStates }
|
||||
|
||||
// Get is a key's value: a build read now, a manifest kept once read, any other state's from the copy.
|
||||
func (w Watched) Get(state, key string) ([]byte, bool, error) {
|
||||
switch state {
|
||||
case frombus.BuildsRead:
|
||||
return w.Direct(state, key)
|
||||
case frombus.ManifestsRead:
|
||||
return w.Once(state, key)
|
||||
}
|
||||
return w.WatchedStates.Get(state, key)
|
||||
}
|
||||
|
||||
// WatchedReader is frombus's Reader over w, fetching each templates archive over HTTP once (KeptFetch): a composition
|
||||
// made again on every change reads it again otherwise.
|
||||
func WatchedReader(w *stdio.WatchedStates) frombus.Reader {
|
||||
return frombus.StatesReader(Watched{w}, KeptFetch(HTTPFetch))
|
||||
}
|
||||
|
||||
// archivesAtMost bounds the archives KeptFetch keeps; past it they are emptied, which costs only a fetch again.
|
||||
const archivesAtMost = 64
|
||||
|
||||
// KeptFetch is fetch kept once per URL: an archive's URL names its blob by digest, so its body never changes under it
|
||||
// (frombus checks the digest on every read). A failed fetch is not kept.
|
||||
func KeptFetch(fetch frombus.Fetch) frombus.Fetch {
|
||||
var mu sync.Mutex
|
||||
kept := map[string][]byte{}
|
||||
return func(ctx context.Context, url string) ([]byte, error) {
|
||||
mu.Lock()
|
||||
body, ok := kept[url]
|
||||
mu.Unlock()
|
||||
if ok {
|
||||
return body, nil
|
||||
}
|
||||
body, err := fetch(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mu.Lock()
|
||||
if len(kept) >= archivesAtMost {
|
||||
clear(kept)
|
||||
}
|
||||
kept[url] = body
|
||||
mu.Unlock()
|
||||
return body, nil
|
||||
}
|
||||
}
|
||||
|
||||
// HTTPFetch reads an archive over HTTP, at most 256 MiB.
|
||||
func HTTPFetch(ctx context.Context, url string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
|
||||
@@ -11,6 +11,10 @@ import (
|
||||
// SourcePrefix2 opens every source fingerprint of the new form.
|
||||
const SourcePrefix2 = "src2:"
|
||||
|
||||
// NoPackageVersion opens the reason a module has no fingerprint while a mesh package it uses has no published version
|
||||
// the indexer holds (novox/hq ADR 0573 §2); the packages follow, by module path.
|
||||
const NoPackageVersion = "no published version of the package "
|
||||
|
||||
// Toolchain2 is what an artifact's toolchain enters the new fingerprint as (novox/hq ADR 0405 clause 7): the
|
||||
// artifact's language, the installed toolchain's version and the build module's recipe version, in place of the
|
||||
// toolchain image's digest. A toolchain update so changes the fingerprints of that language's modules only.
|
||||
@@ -37,15 +41,19 @@ type SourceInputs2 struct {
|
||||
Contexts map[string]string
|
||||
Locks map[string]string
|
||||
Toolchains map[string]Toolchain2
|
||||
Unpinned string
|
||||
// Packages are the version of each mesh package the module uses, by package (novox/hq ADR 0573 §2): a package with
|
||||
// no version gives no fingerprint. None leaves the fingerprint as it was before packages entered it.
|
||||
Packages map[string]string
|
||||
Unpinned string
|
||||
}
|
||||
|
||||
// SourceFingerprint2 is the module's source fingerprint of the new form, `src2:` and the hex sha256 of its inputs'
|
||||
// lines, or none and the reason when the source does not pin the build: Unpinned when set, "no tree" when Tree is
|
||||
// empty, "no lock" when an artifact resolves packages and has no lock. It is a pure function of its inputs: the order
|
||||
// of maps and a base given twice change nothing. The lines are "module <m>", "tree <t>" and "builder <fp>", then
|
||||
// "base <fp>" (once each), "context <artifact> <tree>", "lock <artifact> <hash>" and "toolchain <artifact> <language>
|
||||
// <version> <recipe>", sorted, all joined by a newline.
|
||||
// "base <fp>" (once each), "context <artifact> <tree>", "lock <artifact> <hash>", "toolchain <artifact> <language>
|
||||
// <version> <recipe>" and "package <package> <version>", sorted, all joined by a newline. A module that uses no mesh
|
||||
// package has no package line, so its fingerprint is the one it had before packages entered (novox/hq ADR 0573 §2).
|
||||
func SourceFingerprint2(in SourceInputs2) (fp, none string) {
|
||||
if in.Unpinned != "" {
|
||||
return "", in.Unpinned
|
||||
@@ -58,6 +66,16 @@ func SourceFingerprint2(in SourceInputs2) (fp, none string) {
|
||||
return "", "no lock"
|
||||
}
|
||||
}
|
||||
var unpublished []string
|
||||
for p, v := range in.Packages {
|
||||
if v == "" {
|
||||
unpublished = append(unpublished, p)
|
||||
}
|
||||
}
|
||||
if len(unpublished) > 0 {
|
||||
sort.Strings(unpublished)
|
||||
return "", NoPackageVersion + strings.Join(unpublished, ", ")
|
||||
}
|
||||
lines := []string{"module " + in.Module, "tree " + in.Tree, "builder " + in.Builder}
|
||||
bases := map[string]bool{}
|
||||
for _, b := range in.Bases {
|
||||
@@ -75,6 +93,9 @@ func SourceFingerprint2(in SourceInputs2) (fp, none string) {
|
||||
for a, t := range in.Toolchains {
|
||||
lines = append(lines, "toolchain "+a+" "+t.Language+" "+t.Version+" "+t.Recipe)
|
||||
}
|
||||
for p, v := range in.Packages {
|
||||
lines = append(lines, "package "+p+" "+v)
|
||||
}
|
||||
sort.Strings(lines[3:])
|
||||
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||
return SourcePrefix2 + hex.EncodeToString(sum[:]), ""
|
||||
|
||||
@@ -62,6 +62,9 @@ func TestFingerprint2Changes(t *testing.T) {
|
||||
"a context": func(in *SourceInputs2) { in.Contexts["mail"] = "t-changed" },
|
||||
"a context added": func(in *SourceInputs2) { in.Contexts["extra"] = "t-root" },
|
||||
"a toolchain added": func(in *SourceInputs2) { in.Toolchains["extra"] = Toolchain2{"go", "1.26.0", "3"} },
|
||||
// Issue 1585 (ADR 0573 §2): a package's new published version, and a package used at all.
|
||||
"a package added": func(in *SourceInputs2) { in.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "c1"} },
|
||||
"a package version": func(in *SourceInputs2) { in.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "c2"} },
|
||||
} {
|
||||
in := inputs2()
|
||||
f(&in)
|
||||
@@ -111,6 +114,17 @@ func TestFingerprint2None(t *testing.T) {
|
||||
if fp, none := SourceFingerprint2(in); fp != "" || none != "no tree" {
|
||||
t.Errorf("no tree: fp = %q, none = %q", fp, none)
|
||||
}
|
||||
// Issue 1585: a package with no published version gives none, naming it; an empty set of packages changes nothing.
|
||||
in = inputs2()
|
||||
in.Packages = map[string]string{"git.novox.be/novox/b/go": "", "git.novox.be/novox/a/go": "", "git.novox.be/novox/c/go": "c1"}
|
||||
if fp, none := SourceFingerprint2(in); fp != "" || none != NoPackageVersion+"git.novox.be/novox/a/go, git.novox.be/novox/b/go" {
|
||||
t.Errorf("no package version: fp = %q, none = %q", fp, none)
|
||||
}
|
||||
in = inputs2()
|
||||
in.Packages = map[string]string{}
|
||||
if fp, _ := SourceFingerprint2(in); fp != mustFP2(t, inputs2()) {
|
||||
t.Errorf("no package changes the fingerprint: %s", fp)
|
||||
}
|
||||
in = inputs2()
|
||||
in.Tree, in.Unpinned = "", "a context is a Go program's build source"
|
||||
if _, none := SourceFingerprint2(in); none != in.Unpinned {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -163,4 +164,35 @@ func TestModuleKeepsInputs2(t *testing.T) {
|
||||
if _, err := EncodeModule(m); err == nil {
|
||||
t.Error("Inputs2 without a manifest digest was encoded")
|
||||
}
|
||||
|
||||
// Issue 1585 (ADR 0573 §2): the mesh packages a go.sum names, those of other repositories, and the latest published
|
||||
// version of each: the trunk commit of the deepest module of its repository whose directory holds it.
|
||||
sum := "git.novox.be/novox/mesh-sdk/go v0.1.7 h1:a=\ngit.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:b=\n" +
|
||||
"git.novox.be/novox/mesh-controller/modules/route-proxy/contract v0.1.0 h1:c=\n" +
|
||||
"github.com/nats-io/nats.go v1.0.0 h1:d=\ngit.novox.be/novox/mesh-host v0.0.0-1 h1:e=\n"
|
||||
pkgs := MeshPackagesOf([]byte(sum))
|
||||
if want := []string{"git.novox.be/novox/mesh-controller/modules/route-proxy/contract", "git.novox.be/novox/mesh-host", "git.novox.be/novox/mesh-sdk/go"}; !reflect.DeepEqual(pkgs, want) {
|
||||
t.Errorf("the packages of a go.sum: %v, want %v", pkgs, want)
|
||||
}
|
||||
if got := OtherPackages(pkgs, "novox/mesh-controller"); len(got) != 2 || got[0] != "git.novox.be/novox/mesh-host" {
|
||||
t.Errorf("the packages of other repositories: %v", got)
|
||||
}
|
||||
held := map[string]Module{
|
||||
"mesh-sdk": {Repository: "novox/mesh-sdk", Path: RootPath, Commit: "c-sdk"},
|
||||
"route-proxy": {Repository: "novox/mesh-controller", Path: "modules/route-proxy", Commit: "c-rp"},
|
||||
"controller": {Repository: "novox/mesh-controller", Path: RootPath, Commit: "c-ctl"},
|
||||
"route-proxy2": {Repository: "novox/mesh-controller", Path: "modules/route", Commit: "c-wrong"},
|
||||
"no-path": {Repository: "novox/mesh-host", Commit: "c-host"},
|
||||
}
|
||||
versions, missing := PackageVersionsOf(pkgs, held)
|
||||
if want := (PackageVersions{"git.novox.be/novox/mesh-sdk/go": "c-sdk", "git.novox.be/novox/mesh-controller/modules/route-proxy/contract": "c-rp"}); !reflect.DeepEqual(versions, want) {
|
||||
t.Errorf("versions %v, want %v", versions, want)
|
||||
}
|
||||
if len(missing) != 1 || missing[0] != "git.novox.be/novox/mesh-host" {
|
||||
t.Errorf("missing %v: a module that names no directory holds no package", missing)
|
||||
}
|
||||
kept.Packages = []string{"git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-host"}
|
||||
if fp, none := SourceFingerprint2(kept.WithPackages("m", tc, versions)); fp != "" || !strings.HasPrefix(none, NoPackageVersion) {
|
||||
t.Errorf("a package with no version: %q %q", fp, none)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// **Every module builds against the latest published version of each mesh package it uses** (novox/hq ADR 0573 §2;
|
||||
// issue 1585). A mesh package is a Go module of a repository of the mesh's forge (ForgeModulePrefix): mesh-sdk's Go
|
||||
// module, and a seat's contract its holder's repository publishes (ADR 0573 §1). Its version is the trunk merge the
|
||||
// indexer holds for the module whose directory holds the package (PackageVersionsOf): a merge that changes it changes
|
||||
// the fingerprint of every module that uses it, and the build writes that version into its own go.mod and go.sum
|
||||
// (StepAsk.Packages). Which packages a module uses is read from its lock file by MeshPackagesOf, the same way by the
|
||||
// forge bridge and by the build modules (buildstep.ModuleTrees), so nobody names a version by hand.
|
||||
|
||||
// ForgeModulePrefix is the module path prefix of the Go modules the mesh's forge serves: a Go module whose path starts
|
||||
// with it is a mesh package, git.novox.be/novox/<repository>[/<directory of the module in it>].
|
||||
const ForgeModulePrefix = "git.novox.be/novox/"
|
||||
|
||||
// MeshPackagesOf are the mesh packages a Go module's go.sum names (the module path of each line that starts with
|
||||
// ForgeModulePrefix), sorted and each once. go.sum names every module of the build's graph, so a package used only
|
||||
// through another is named too. Nil for none. A package of the module's own repository comes with its own tree, and
|
||||
// its reader leaves it out (OtherPackages).
|
||||
func MeshPackagesOf(goSum []byte) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, line := range strings.Split(string(goSum), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 2 || !strings.HasPrefix(f[0], ForgeModulePrefix) {
|
||||
continue
|
||||
}
|
||||
if _, _, ok := PackageRepository(f[0]); !ok {
|
||||
continue
|
||||
}
|
||||
seen[f[0]] = true
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return nil
|
||||
}
|
||||
return slices.Sorted(maps.Keys(seen))
|
||||
}
|
||||
|
||||
// OtherPackages are the packages not published from repository, the module's own, in their order; nil for none.
|
||||
func OtherPackages(packages []string, repository string) []string {
|
||||
var out []string
|
||||
for _, p := range packages {
|
||||
if repo, _, ok := PackageRepository(p); ok && !SameRepository(repo, repository) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PackageRepository is the repository (<owner>/<name>) a mesh package is published from and the package's directory in
|
||||
// it ("" for its root); false for a path that is not a forge module's.
|
||||
func PackageRepository(pkg string) (repository, dir string, ok bool) {
|
||||
rest, found := strings.CutPrefix(pkg, ForgeModulePrefix)
|
||||
if !found || rest == "" {
|
||||
return "", "", false
|
||||
}
|
||||
name, dir, _ := strings.Cut(rest, "/")
|
||||
if name == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return "novox/" + name, dir, true
|
||||
}
|
||||
|
||||
// PackageVersionsOf is the latest published version of each package from the modules the indexer holds (by key): the
|
||||
// trunk merge commit of the module of the package's repository whose directory holds the package's directory (the
|
||||
// deepest, a module at the repository's root holding every directory). A package no held module holds, or whose
|
||||
// module names no commit or no directory, is in missing, sorted; versions is nil when packages is empty.
|
||||
func PackageVersionsOf(packages []string, modules map[string]Module) (versions PackageVersions, missing []string) {
|
||||
for _, pkg := range packages {
|
||||
repository, dir, ok := PackageRepository(pkg)
|
||||
commit, depth := "", -1
|
||||
if ok {
|
||||
for _, key := range slices.Sorted(maps.Keys(modules)) {
|
||||
m := modules[key]
|
||||
if !SameRepository(m.Repository, repository) || m.Commit == "" {
|
||||
continue
|
||||
}
|
||||
at := strings.Trim(m.Path, "/")
|
||||
switch {
|
||||
case at == "":
|
||||
continue // a value that names no directory says nothing of where the package lies
|
||||
case at == RootPath:
|
||||
at = ""
|
||||
case dir != at && !strings.HasPrefix(dir, at+"/"):
|
||||
continue
|
||||
}
|
||||
if d := len(at); d > depth {
|
||||
commit, depth = m.Commit, d
|
||||
}
|
||||
}
|
||||
}
|
||||
if commit == "" {
|
||||
missing = append(missing, pkg)
|
||||
continue
|
||||
}
|
||||
if versions == nil {
|
||||
versions = PackageVersions{}
|
||||
}
|
||||
versions[pkg] = commit
|
||||
}
|
||||
slices.Sort(missing)
|
||||
return versions, missing
|
||||
}
|
||||
@@ -107,6 +107,8 @@ type PreviewModule struct {
|
||||
// Paths are, of a module whose every artifact names its paths, each named path's git object id at the merge; Tree
|
||||
// is then their NamedTree (novox/hq ADR 0499 clause 2). Absent otherwise.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
// Packages are the mesh packages the module's lock file names at the merge (MeshPackagesOf, novox/hq ADR 0573 §2).
|
||||
Packages []string `json:"packages,omitempty"`
|
||||
}
|
||||
|
||||
// Preview is the value of a preview key: the repository, the head and the trunk commit it was merged onto, the merge
|
||||
|
||||
+47
-4
@@ -284,6 +284,10 @@ type CandidateModule struct {
|
||||
// Paths are, of a module whose every artifact names its paths, each named path's git object id at the preview
|
||||
// (novox/hq ADR 0499 clause 2), so the builder's fetch request for the candidate names them. Absent otherwise.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
// Packages are the versions of the mesh packages FP2 was computed with (novox/hq ADR 0573 §2, §4): the latest
|
||||
// published version of each, or the head of the candidate's pull request that publishes it, so a group's candidate
|
||||
// builds against the group's own package. Absent for none.
|
||||
Packages PackageVersions `json:"packages,omitempty"`
|
||||
}
|
||||
|
||||
func (m CandidateModule) check() error {
|
||||
@@ -296,6 +300,9 @@ func (m CandidateModule) check() error {
|
||||
if err := checkPathObjects(m.Paths); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := m.Packages.Check(); err != nil {
|
||||
return err
|
||||
}
|
||||
return checkNewForm(m.FP2, m.Manifest)
|
||||
}
|
||||
|
||||
@@ -415,18 +422,28 @@ type Module struct {
|
||||
// so the checker tells whether a change touches an artifact's build source without reading a manifest (the lead's
|
||||
// decision L8 of R4e). Absent for an artifact that names none, and in a value written before.
|
||||
ArtifactPaths map[string][]string `json:"paths,omitempty"`
|
||||
// Packages are the versions of the mesh packages FP2 was computed with (novox/hq ADR 0573 §2): the trunk merge the
|
||||
// indexer held for each package of Inputs2.Packages. A builder builds the fingerprint against exactly these
|
||||
// (StepAsk.Packages). Empty when FP2 is empty or the module uses no mesh package.
|
||||
Packages PackageVersions `json:"packages,omitempty"`
|
||||
// Verdict is the indexer's verdict on Manifest, VerdictAccepted or VerdictRefused, and Problems each problem of a
|
||||
// refusal, as the verdicts state holds them (novox/hq ADR 0578 §1): a reader of modules takes it without a read of
|
||||
// that state. Empty in a value written before the indexer judged, which a reader takes as accepted (TakeManifestVerdict).
|
||||
Verdict string `json:"verdict,omitempty"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
// Warnings are the problems of an accepted manifest a stricter judge outgrew (novox/hq ADR 0579 §1): Verdict is then
|
||||
// empty (accepted), and the watchman raises delivery.ManifestOutgrown until the manifest is mended.
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// VerdictOf is the verdict a modules value carries, or nil when it carries none (taken as accepted, TakeManifestVerdict).
|
||||
func (m Module) VerdictOf(module string) *ManifestVerdict {
|
||||
if m.Verdict == "" {
|
||||
if m.Verdict == "" && len(m.Warnings) == 0 {
|
||||
return nil
|
||||
}
|
||||
if m.Verdict == "" {
|
||||
return &ManifestVerdict{Module: module, State: VerdictAccepted, Warnings: m.Warnings}
|
||||
}
|
||||
return &ManifestVerdict{Module: module, State: m.Verdict, Problems: m.Problems}
|
||||
}
|
||||
|
||||
@@ -446,6 +463,10 @@ type KeptInputs2 struct {
|
||||
// computation from kept inputs gives the merge's value (novox/hq ADR 0499 clause 2). Absent for every other module,
|
||||
// whose Tree is its directory's tree, and in a value written before.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
// Packages are the mesh packages the module's lock file names (MeshPackagesOf, novox/hq ADR 0573 §2): the
|
||||
// fingerprint takes the latest published version of each (Module.Packages). Absent for none, and in a value written
|
||||
// before.
|
||||
Packages []string `json:"packages,omitempty"`
|
||||
}
|
||||
|
||||
// checkPathObjects refuses a named path that is not clean and relative to the module's directory, or names no object.
|
||||
@@ -482,6 +503,19 @@ func (k KeptInputs2) With(module string, toolchains map[string]Toolchain2) Sourc
|
||||
Locks: k.Locks, Toolchains: toolchains}
|
||||
}
|
||||
|
||||
// WithPackages is With and the version of each package the module uses (Packages), as PackageVersionsOf read them: a
|
||||
// package with no version enters with none, and the fingerprint is then none (SourceFingerprint2).
|
||||
func (k KeptInputs2) WithPackages(module string, toolchains map[string]Toolchain2, versions PackageVersions) SourceInputs2 {
|
||||
in := k.With(module, toolchains)
|
||||
if len(k.Packages) > 0 {
|
||||
in.Packages = map[string]string{}
|
||||
for _, p := range k.Packages {
|
||||
in.Packages[p] = versions[p]
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func (m Module) check() error {
|
||||
switch {
|
||||
case m.FP != "" && m.None != "":
|
||||
@@ -498,6 +532,9 @@ func (m Module) check() error {
|
||||
return fmt.Errorf("delivery: the module's toolchain of %q needs its version and its recipe version", language)
|
||||
}
|
||||
}
|
||||
if err := m.Packages.Check(); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Inputs2 != nil && m.Manifest == "" {
|
||||
return fmt.Errorf("delivery: a module keeps new-form inputs but has no manifest digest")
|
||||
}
|
||||
@@ -523,6 +560,8 @@ func (m Module) check() error {
|
||||
return fmt.Errorf("delivery: a module carries a verdict and no manifest digest")
|
||||
case (m.Verdict == VerdictRefused) != (len(m.Problems) > 0):
|
||||
return fmt.Errorf("delivery: a module's problems are named only with a refusal, and a refusal names them")
|
||||
case m.Verdict == VerdictRefused && len(m.Warnings) > 0:
|
||||
return fmt.Errorf("delivery: a refused module warns: a refusal names problems")
|
||||
}
|
||||
return checkNewForm(m.FP2, m.Manifest)
|
||||
}
|
||||
@@ -555,9 +594,13 @@ type Forge struct {
|
||||
// id at the same commit as its manifest, by path relative to the directory (novox/hq ADR 0499 clause 3). The
|
||||
// indexer names the paths; a message without them gives such a module none. Older writers send none.
|
||||
Paths map[string]map[string]string `json:"paths,omitempty"`
|
||||
Act string `json:"act,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Open []OpenPull `json:"open,omitempty"`
|
||||
// Packages are, of each module directory, the mesh packages its lock file names (MeshPackagesOf; novox/hq ADR 0573
|
||||
// §2): the indexer keeps them in the module's inputs and takes each one's latest published version into its
|
||||
// fingerprint. Absent for a directory that uses none, and from an older writer.
|
||||
Packages map[string][]string `json:"packages,omitempty"`
|
||||
Act string `json:"act,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Open []OpenPull `json:"open,omitempty"`
|
||||
// RetiredPlacements is the placement files a bridge before ADR 0567 carried (ADR 0549 §2, ADR 0559 §2): placement is
|
||||
// the seat mesh-placement's state now and no repository holds it (ADR 0567 §11), so a message carrying them is
|
||||
// refused, loudly, rather than read with them dropped.
|
||||
|
||||
@@ -253,6 +253,30 @@ type StepAsk struct {
|
||||
Compose *ComposeJob `json:"compose,omitempty"`
|
||||
// Needs are what the node that runs the ask must have (AskNeeds); a check ask's alone. Empty: any node.
|
||||
Needs []string `json:"needs,omitempty"`
|
||||
// Packages are the version of each mesh package the build or the check builds against, by its module path (a Go
|
||||
// module's path, git.novox.be/novox/mesh-sdk/go; the shape takes a seat's package too), whatever version the
|
||||
// repository's go.mod names (novox/hq ADR 0573 §2, §4; issue 1585): a Go package's version is a full commit, the
|
||||
// trunk merge the indexer holds as the package's latest published version or, in a delivery group's check, the
|
||||
// head of the group's member that publishes it. The build module writes each into its workspace, checksum-verified,
|
||||
// and names them in its log (PackageVersions). A compose takes none: it builds the mesh-sdk its reader names.
|
||||
Packages PackageVersions `json:"packages,omitempty"`
|
||||
}
|
||||
|
||||
// PackageVersions are the versions of the mesh packages a build or a check builds against, by package (a Go module's
|
||||
// path), each a full commit (novox/hq ADR 0573 §2).
|
||||
type PackageVersions map[string]string
|
||||
|
||||
// Check refuses a package that is no module path or a version that is no full commit, naming it.
|
||||
func (p PackageVersions) Check() error {
|
||||
for pkg, version := range p {
|
||||
if pkg == "" || strings.ContainsAny(pkg, " \t\n@") || strings.HasPrefix(pkg, "/") {
|
||||
return fmt.Errorf("delivery: the package %q is no module path", pkg)
|
||||
}
|
||||
if !askCommitOK.MatchString(version) {
|
||||
return fmt.Errorf("delivery: the package %s names the version %q, which is no full commit", pkg, version)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ComposeJob is a compose ask's job (novox/hq ADR 0482 clause 4, issue 1622): fetch the commit, build the composer
|
||||
@@ -389,6 +413,11 @@ func (a StepAsk) check() error {
|
||||
return fmt.Errorf("delivery: the ask %s is a check of an image, which has no tests", a.ID)
|
||||
case len(a.Needs) > 0 && a.Check == nil:
|
||||
return fmt.Errorf("delivery: the ask %s names needs and is no check, whose tests alone need a node", a.ID)
|
||||
case len(a.Packages) > 0 && a.Compose != nil:
|
||||
return fmt.Errorf("delivery: the ask %s is a compose and names package versions, which a compose does not take", a.ID)
|
||||
}
|
||||
if err := a.Packages.Check(); err != nil {
|
||||
return fmt.Errorf("the ask %s: %w", a.ID, err)
|
||||
}
|
||||
for _, n := range a.Needs {
|
||||
if err := CheckNeed(n); err != nil {
|
||||
|
||||
@@ -17,7 +17,8 @@ func buildAsk() StepAsk {
|
||||
Build: &BuildJob{Module: "mesh-delivery-view", Path: "modules/mesh-delivery-view", FP: "ab12",
|
||||
Fetch: map[string]any{"repository": "novox/mesh-delivery", "commit": askCommit, "dir": "modules/mesh-delivery-view", "paths": []any{"cmd"}},
|
||||
Artifacts: []AskArtifact{{Artifact: "bundle"}, {Artifact: "base", From: "registry/x@sha256:00"}},
|
||||
Publish: true, Toolchain: &ToolchainUsed{Version: "go1.26.0", Recipe: "3"}}}
|
||||
Publish: true, Toolchain: &ToolchainUsed{Version: "go1.26.0", Recipe: "3"}},
|
||||
Packages: PackageVersions{"git.novox.be/novox/mesh-sdk/go": askCommit}}
|
||||
}
|
||||
|
||||
func checkAsk() StepAsk {
|
||||
@@ -25,10 +26,11 @@ func checkAsk() StepAsk {
|
||||
At: time.Date(2026, 10, 23, 13, 0, 0, 0, time.UTC),
|
||||
Check: &CheckJob{Fetch: map[string]any{"repository": "novox/mesh-catalog", "commit": askCommit, "ref": "refs/pull/3/head", "base": askCommit},
|
||||
Levels: []string{LevelUnit, LevelIntegration}, Kept: true, Runner: "workspace", RunnerWhen: "modules/mesh-build-go/", Run: "",
|
||||
NodeNames: []string{"anchor", "laptop"}}}
|
||||
NodeNames: []string{"anchor", "laptop"}},
|
||||
Packages: PackageVersions{"git.novox.be/novox/mesh-sdk/go": askCommit, "git.novox.be/novox/mesh-controller/modules/route-proxy/contract": askCommit}}
|
||||
}
|
||||
|
||||
// 1887/A1a: an ask encodes and decodes with every field.
|
||||
// 1887/A1a: an ask encodes and decodes with every field; 1585/S4: the package versions too.
|
||||
func TestAskRoundTrips(t *testing.T) {
|
||||
for _, a := range []StepAsk{buildAsk(), checkAsk()} {
|
||||
raw, err := EncodeStepAsk(a)
|
||||
@@ -58,6 +60,13 @@ func TestAskRefusesUnknownKindAndLanguage(t *testing.T) {
|
||||
"commit": func(a *StepAsk) { a.Build.Fetch = map[string]any{"repository": "novox/x", "commit": "main"} },
|
||||
"path": func(a *StepAsk) { a.Build.Path = "" },
|
||||
"needs": func(a *StepAsk) { a.Needs = []string{NeedNodeBuildNode} }, // issue 890: a build needs no node
|
||||
// Issue 1585: a package's version is a full commit, of a module path.
|
||||
"version": func(a *StepAsk) { a.Packages["git.novox.be/novox/mesh-sdk/go"] = "v0.1.7" },
|
||||
"package": func(a *StepAsk) { a.Packages["git.novox.be/novox/x y"] = askCommit },
|
||||
"compose": func(a *StepAsk) {
|
||||
a.Kind, a.Build = AskCheck, nil
|
||||
a.Compose = &ComposeJob{Fetch: map[string]any{"repository": "novox/mesh-sdk", "commit": askCommit}, Dir: "go", Package: "x", Request: json.RawMessage(`{}`)}
|
||||
},
|
||||
}
|
||||
for name, change := range cases {
|
||||
a := buildAsk()
|
||||
|
||||
+17
-13
@@ -3,6 +3,7 @@ package delivery
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -26,15 +27,23 @@ const (
|
||||
// ManifestRefusedEvent is the indexer's event saying a manifest was refused (ManifestRefused), so the watchman
|
||||
// raises it as a condition beside the indexer's finding.
|
||||
ManifestRefusedEvent = "manifest-refused"
|
||||
|
||||
// ManifestOutgrown is the finding of a manifest the indexer accepted before and a stricter judge finds problems in
|
||||
// (novox/hq ADR 0579 §1): it stays accepted, its problems are the verdict's Warnings, and the watchman raises it as a
|
||||
// warning until the manifest is mended.
|
||||
ManifestOutgrown = "manifest-outgrown"
|
||||
)
|
||||
|
||||
// ManifestVerdict is the value of a verdicts key: the module whose manifest it is, accepted or refused, every problem the judge
|
||||
// found (none when accepted), the judge that made it (the indexer's build fingerprint) and when.
|
||||
type ManifestVerdict struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
State string `json:"state"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
State string `json:"state"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
// Warnings are the problems a stricter judge found in a manifest it accepted before, digest unchanged (novox/hq ADR
|
||||
// 0579 §1): the verdict stays accepted.
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
Judge string `json:"judge"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
@@ -45,15 +54,7 @@ func (v ManifestVerdict) Refused() bool { return v.State == VerdictRefused }
|
||||
// Same is whether v and o say the same of a manifest: the module, the state and the problems, whoever judged and when.
|
||||
// The indexer writes a verdict only when it is not the same as the one it holds.
|
||||
func (v ManifestVerdict) Same(o ManifestVerdict) bool {
|
||||
if v.Module != o.Module || v.State != o.State || len(v.Problems) != len(o.Problems) {
|
||||
return false
|
||||
}
|
||||
for i := range v.Problems {
|
||||
if v.Problems[i] != o.Problems[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
return v.Module == o.Module && v.State == o.State && slices.Equal(v.Problems, o.Problems) && slices.Equal(v.Warnings, o.Warnings)
|
||||
}
|
||||
|
||||
func (v ManifestVerdict) check() error {
|
||||
@@ -69,6 +70,9 @@ func (v ManifestVerdict) check() error {
|
||||
if len(v.Problems) == 0 {
|
||||
return fmt.Errorf("delivery: the verdict on %s refuses its manifest and names no problem", v.Module)
|
||||
}
|
||||
if len(v.Warnings) > 0 {
|
||||
return fmt.Errorf("delivery: the verdict on %s refuses its manifest and warns: a refusal names problems", v.Module)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("delivery: the verdict on %s is %q, neither %s nor %s", v.Module, v.State, VerdictAccepted, VerdictRefused)
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestAManifestVerdictIsTheContractReadersTake(t *testing.T) {
|
||||
"another state": {Module: "m", State: "maybe", Judge: "j"},
|
||||
"refused, no problem": {Module: "m", State: VerdictRefused, Judge: "j"},
|
||||
"accepted, a problem": {Module: "m", State: VerdictAccepted, Problems: []string{"x"}, Judge: "j"},
|
||||
"refused, a warning": {Module: "m", State: VerdictRefused, Problems: []string{"x"}, Warnings: []string{"y"}, Judge: "j"},
|
||||
} {
|
||||
if _, err := EncodeManifestVerdict(bad); err == nil {
|
||||
t.Errorf("%s: encoded", name)
|
||||
@@ -62,4 +63,13 @@ func TestAManifestVerdictIsTheContractReadersTake(t *testing.T) {
|
||||
if _, err := EncodeModule(m); err == nil {
|
||||
t.Fatal("a module refused naming no problem was encoded")
|
||||
}
|
||||
// ADR 0579 §1: an accepted manifest a stricter judge outgrew carries its warnings and is taken as accepted.
|
||||
outgrown := ManifestVerdict{Module: "x", State: VerdictAccepted, Warnings: []string{"w"}, Judge: "j"}
|
||||
if _, err := EncodeManifestVerdict(outgrown); err != nil || outgrown.Same(ManifestVerdict{Module: "x", State: VerdictAccepted}) {
|
||||
t.Fatalf("an outgrown verdict: %v", err)
|
||||
}
|
||||
m.Verdict, m.Warnings = "", []string{"w"}
|
||||
if r, _ := TakeManifestVerdict(m.VerdictOf("x")); r || len(m.VerdictOf("x").Warnings) != 1 {
|
||||
t.Fatalf("an outgrown module: %+v", m.VerdictOf("x"))
|
||||
}
|
||||
}
|
||||
|
||||
+11
-28
@@ -1,8 +1,7 @@
|
||||
// Package handacts is the hand-act log's contract (novox/hq ADR 0565 clauses 10 and 11, issue 1953): the layout of an act,
|
||||
// moved unchanged from the controller's (mesh-controller internal/link HandAct) so one reader reads both logs alike; the
|
||||
// names of the seat mesh-hand-acts and of what its holder keeps; and the recorder every module records through, which
|
||||
// asks the seat and, while no holder answers, the controller's verb until the controller hands its lease away, saying
|
||||
// which log took the act.
|
||||
// asks the seat alone: the controller's hand-act and warranted are gone (issue 1953 L5b) and the bus refuses them.
|
||||
package handacts
|
||||
|
||||
import (
|
||||
@@ -95,13 +94,13 @@ type Taken struct {
|
||||
Answer json.RawMessage `json:"answer,omitempty"`
|
||||
}
|
||||
|
||||
// Recorder records acts for a module (ADR 0565 clause 11): through the seat mesh-hand-acts, and while the seat has no
|
||||
// holder that answers, through the controller's verb until the controller hands its lease away (the release's order
|
||||
// gives no seat's holder before its callers; 00-plan.md, "Missing" 1).
|
||||
// Recorder records acts for a module (ADR 0565 clause 11) through the seat mesh-hand-acts alone. An act the seat does
|
||||
// not take is an error that says nothing was recorded, never a quiet success.
|
||||
type Recorder struct {
|
||||
// Ask calls a seat verb by its address (`seat:<seat>.<verb>`): stdio.Ask through the runtime.
|
||||
Ask func(key string, body any) (json.RawMessage, error)
|
||||
// LeaseAway says the controller's lease is handed away: then nothing falls back to it. Nil is never handed away.
|
||||
// LeaseAway is no longer read: nothing falls back to the controller. Kept so a caller that sets it builds; it goes
|
||||
// once no module sets it.
|
||||
LeaseAway func() bool
|
||||
}
|
||||
|
||||
@@ -121,38 +120,22 @@ func (r Recorder) Record(q Request) (Taken, error) {
|
||||
body[k] = v
|
||||
}
|
||||
}
|
||||
controller := map[string]any{"what": q.What, "why": q.Why, "cause": q.Cause}
|
||||
if q.Condition != "" {
|
||||
controller["condition"] = q.Condition
|
||||
}
|
||||
if q.Cause == "" {
|
||||
controller["cause"] = q.Verb
|
||||
}
|
||||
return r.either("hand-act", body, "hand-act", controller)
|
||||
return r.record("hand-act", body)
|
||||
}
|
||||
|
||||
// RecordWarrant records what the asking module did on the operator's warrant for its ask.
|
||||
func (r Recorder) RecordWarrant(asker, ask string) (Taken, error) {
|
||||
body := map[string]any{"asker": asker, "ask": ask}
|
||||
return r.either("warranted", body, "warranted", body)
|
||||
return r.record("warranted", body)
|
||||
}
|
||||
|
||||
func (r Recorder) either(verb string, body any, controllerVerb string, controllerBody any) (Taken, error) {
|
||||
func (r Recorder) record(verb string, body any) (Taken, error) {
|
||||
answer, err := r.Ask("seat:"+Seat+"."+verb, body)
|
||||
if err == nil {
|
||||
return Taken{Log: LogSeat, Answer: answer}, nil
|
||||
}
|
||||
if !NoResponder(err) {
|
||||
return Taken{}, fmt.Errorf("%s.%s refused: %w", Seat, verb, err)
|
||||
if NoResponder(err) {
|
||||
return Taken{}, fmt.Errorf("%s has no holder that answers: nothing was recorded (%w)", Seat, err)
|
||||
}
|
||||
if r.LeaseAway != nil && r.LeaseAway() {
|
||||
return Taken{}, fmt.Errorf("%s has no holder that answers, and the controller handed its lease away: nothing "+
|
||||
"was recorded (%w)", Seat, err)
|
||||
}
|
||||
answer, cerr := r.Ask("seat:mesh-controller."+controllerVerb, controllerBody)
|
||||
if cerr != nil {
|
||||
return Taken{}, fmt.Errorf("%s has no holder that answers (%v), and the controller's %s refused too: %w", Seat, err,
|
||||
controllerVerb, cerr)
|
||||
}
|
||||
return Taken{Log: LogController, Answer: answer}, nil
|
||||
return Taken{}, fmt.Errorf("%s.%s refused, nothing was recorded: %w", Seat, verb, err)
|
||||
}
|
||||
|
||||
@@ -8,9 +8,8 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ADR 0565 clauses 10 and 11 (issue 1953 L5-S2, S4): an act reads the controller's entries unchanged, and a recorder
|
||||
// records through the seat, through the controller's verb only while the seat has no holder and the lease is the
|
||||
// controller's, saying which log took it, and refuses to record nowhere.
|
||||
// ADR 0565 clauses 10 and 11 (issue 1953 L5-S2, S4, L5b): an act reads the controller's entries unchanged, and a
|
||||
// recorder records through the seat alone, the controller's verbs being gone, and says loudly when nothing was recorded.
|
||||
func TestAnActIsRecordedInTheLogThatAnswers(t *testing.T) {
|
||||
// The shape of mesh-controller internal/link HandAct, as its bucket keeps an entry (handacts.go at 0fbb5ef3).
|
||||
raw := `{"id":"act-1","at":"2026-10-28T09:00:00Z","by":"person.jochen","verb":"hand-act record","why":"a stuck consumer",` +
|
||||
@@ -39,14 +38,13 @@ func TestAnActIsRecordedInTheLogThatAnswers(t *testing.T) {
|
||||
warrants bool
|
||||
}{
|
||||
{name: "the seat answers", log: LogSeat, asked: "seat:mesh-hand-acts.hand-act"},
|
||||
{name: "no holder yet", seat: noResponders, log: LogController, asked: "seat:mesh-hand-acts.hand-act,seat:mesh-controller.hand-act"},
|
||||
{name: "no holder, the lease handed away", seat: noResponders, away: true, refused: "handed its lease away",
|
||||
{name: "no holder: nothing recorded, the controller never asked", seat: noResponders, refused: "nothing was recorded",
|
||||
asked: "seat:mesh-hand-acts.hand-act"},
|
||||
{name: "the seat refuses", seat: errors.New("why is required"), refused: "refused", asked: "seat:mesh-hand-acts.hand-act"},
|
||||
{name: "neither answers", seat: noResponders, ctrl: noResponders, refused: "refused too",
|
||||
asked: "seat:mesh-hand-acts.hand-act,seat:mesh-controller.hand-act"},
|
||||
{name: "a warrant without a holder", seat: noResponders, warrants: true, log: LogController,
|
||||
asked: "seat:mesh-hand-acts.warranted,seat:mesh-controller.warranted"},
|
||||
{name: "the seat refuses", seat: errors.New("why is required"), refused: "refused, nothing was recorded",
|
||||
asked: "seat:mesh-hand-acts.hand-act"},
|
||||
{name: "a warrant without a holder", seat: noResponders, warrants: true, refused: "nothing was recorded",
|
||||
asked: "seat:mesh-hand-acts.warranted"},
|
||||
{name: "a warrant recorded", warrants: true, log: LogSeat, asked: "seat:mesh-hand-acts.warranted"},
|
||||
} {
|
||||
var asked []string
|
||||
r := Recorder{LeaseAway: func() bool { return c.away }, Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
// The same scan as mesh-catalog's checks/sdkjudge, run by mesh-sdk's own check (novox/hq ADR 0578 §4: the change that
|
||||
// adds a manifest judge to mesh-sdk is refused), so a change to mesh-sdk alone is refused too. Copied from mesh-catalog
|
||||
// feat/judge-once checks/sdkjudge/main.go; the two lists of composition's refusals are kept the same.
|
||||
package stdio
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// composition is every function or table the check lets stay in mesh-sdk, as "path-from-go/:name", and why. It names no
|
||||
// manifest judgement: each is the composer's refusal of composition itself (ADR 0578 clause 5).
|
||||
var composition = map[string]string{
|
||||
"catalogue/consumers.go:consumersPlaceholderProblems": "composition: rendering a consumer's template",
|
||||
"catalogue/collections.go:collectedPlaceholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/environment_into.go:placeholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/seat_contributions.go:seatPlaceholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/collections.go:itemTemplateProblem": "composition: filling an item's template",
|
||||
"catalogue/setting_takes.go:takenSettingsProblems": "composition: whether stored settings still compose",
|
||||
"catalogue/template.go:HeldTemplateProblem": "composition: the take-in refusal of a resolved build",
|
||||
}
|
||||
|
||||
// judgeProblems is what mesh-sdk's Go tree under root (its go/ directory) holds of a manifest judge, one line each, but for
|
||||
// what composition lets stay (the map the caller gives; the test of the real tree gives the package's), and the
|
||||
// entries of composition it no longer holds.
|
||||
func judgeProblems(root string, composition map[string]string) ([]string, error) {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if n := d.Name(); n == "testdata" || n == "vendor" || strings.HasPrefix(n, ".") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(root, path)
|
||||
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", rel, err)
|
||||
}
|
||||
inCatalogue := filepath.Base(filepath.Dir(path)) == "catalogue"
|
||||
for _, d := range file.Decls {
|
||||
switch d := d.(type) {
|
||||
case *ast.FuncDecl:
|
||||
if why := judgeFunc(d); why != "" {
|
||||
out = append(out, note(composition, seen, rel, d.Name.Name, why))
|
||||
}
|
||||
case *ast.GenDecl:
|
||||
if !inCatalogue || d.Tok != token.VAR {
|
||||
continue
|
||||
}
|
||||
for _, spec := range d.Specs {
|
||||
v, _ := spec.(*ast.ValueSpec)
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
for i, name := range v.Names {
|
||||
if i < len(v.Values) && allowName.MatchString(name.Name) && isTable(v.Values[i]) {
|
||||
out = append(out, note(composition, seen, rel, name.Name,
|
||||
"a table of what a manifest may declare (an allow-list)"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var kept []string
|
||||
for _, line := range out {
|
||||
if line != "" {
|
||||
kept = append(kept, line)
|
||||
}
|
||||
}
|
||||
for key, why := range composition {
|
||||
if !seen[key] {
|
||||
kept = append(kept, fmt.Sprintf("composition lists %s (%s), which mesh-sdk no longer holds: take it off the list", key, why))
|
||||
}
|
||||
}
|
||||
sort.Strings(kept)
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// note is the line refusing name in rel for why, or "" when composition lets it stay.
|
||||
func note(composition map[string]string, seen map[string]bool, rel, name, why string) string {
|
||||
key := rel + ":" + name
|
||||
if _, ok := composition[key]; ok {
|
||||
seen[key] = true
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%s: %s is %s; mesh-sdk carries the contract only, the judge is the indexer's and the checker's (ADR 0578 §4)", rel, name, why)
|
||||
}
|
||||
|
||||
// allowName is what names a table of what is allowed.
|
||||
var allowName = regexp.MustCompile(`(?i)allow|permit|removable|reserved|forbid|denied`)
|
||||
|
||||
// isTable says whether an expression is a map or a list literal of more than one entry.
|
||||
func isTable(e ast.Expr) bool {
|
||||
c, ok := e.(*ast.CompositeLit)
|
||||
if !ok || len(c.Elts) < 2 {
|
||||
return false
|
||||
}
|
||||
switch c.Type.(type) {
|
||||
case *ast.MapType, *ast.ArrayType:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// judgeFunc says why a function is a manifest judge, or "".
|
||||
func judgeFunc(f *ast.FuncDecl) string {
|
||||
name := f.Name.Name
|
||||
if judgeEntry.MatchString(name) {
|
||||
return "a function that judges a manifest"
|
||||
}
|
||||
if !strings.HasSuffix(name, "Problems") && !strings.HasSuffix(name, "Problem") {
|
||||
return ""
|
||||
}
|
||||
if takes(f.Recv) || takes(f.Type.Params) {
|
||||
return "a function that gathers a manifest's problems"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// judgeEntry is the names of the judge's entry points: the strict parsers and the validators of a manifest or a shelf.
|
||||
var judgeEntry = regexp.MustCompile(`^(ParseManifest(With|ForInputs)?|Validate(Manifest|Shelf)\w*)$`)
|
||||
|
||||
// takes says whether a list of parameters names a Manifest or a Shelf anywhere in a type.
|
||||
func takes(fields *ast.FieldList) bool {
|
||||
if fields == nil {
|
||||
return false
|
||||
}
|
||||
found := false
|
||||
for _, f := range fields.List {
|
||||
ast.Inspect(f.Type, func(n ast.Node) bool {
|
||||
if id, ok := n.(*ast.Ident); ok && (id.Name == "Manifest" || id.Name == "Shelf") {
|
||||
found = true
|
||||
}
|
||||
return !found
|
||||
})
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// mesh-sdk holds no manifest judge: no function gathering a manifest's problems, no judge entry point and no allow-list
|
||||
// table in the catalogue, but for composition's own refusals, each listed with why (ADR 0578 §4, §5).
|
||||
func TestMeshSDKHoldsNoManifestJudge(t *testing.T) {
|
||||
root, err := filepath.Abs(".")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
problems, err := judgeProblems(root, composition)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range problems {
|
||||
t.Error(p)
|
||||
}
|
||||
}
|
||||
@@ -230,4 +230,57 @@ func TestABundleKeepsAndWatchesStateThroughTheRuntime(t *testing.T) {
|
||||
t.Fatalf("the get answered %v", r)
|
||||
}
|
||||
_ = inW.Close()
|
||||
|
||||
// ADR 0580 clause 1: the watched copy answers keys and values after the watch's current values, a later put or delete
|
||||
// changes its answer, a follower's then sees the copy that took its change, each state is watched once, and a
|
||||
// value kept once is read by key once.
|
||||
handles, watches, gets := map[string]func(StateChange) error{}, map[string]int{}, 0
|
||||
w := WatchedStatesOver(func(state string, handle func(StateChange) error) error {
|
||||
watches[state]++
|
||||
handles[state] = handle
|
||||
return handle(StateChange{State: state, Key: "a", Op: OpPut, Value: []byte("1"), Current: true})
|
||||
}, func(state, key string) ([]byte, bool, error) { gets++; return []byte(key), key != "missing", nil })
|
||||
var thenSaw []string
|
||||
w.Follow("placement", func(c StateChange) error {
|
||||
v, found, _ := w.Get("placement", c.Key)
|
||||
thenSaw = append(thenSaw, fmt.Sprintf("%s %s %s %v", c.Op, c.Key, v, found))
|
||||
return nil
|
||||
})
|
||||
if keys, err := w.Keys("placement"); err != nil || fmt.Sprint(keys) != "[a]" {
|
||||
t.Fatalf("the copy's keys are %v (%v)", keys, err)
|
||||
}
|
||||
_ = handles["placement"](StateChange{Key: "b", Op: OpPut, Value: []byte("2")})
|
||||
_ = handles["placement"](StateChange{Key: "a", Op: OpDelete})
|
||||
vals, _ := w.Values("placement")
|
||||
if v, found, _ := w.Get("placement", "a"); found || string(vals["b"]) != "2" || len(vals) != 1 || v != nil {
|
||||
t.Fatalf("after a put and a delete the copy is %v", vals)
|
||||
}
|
||||
if fmt.Sprint(thenSaw) != "[put a 1 true put b 2 true delete a false]" {
|
||||
t.Fatalf("then saw %v", thenSaw)
|
||||
}
|
||||
_, _ = w.Keys("placement")
|
||||
if watches["placement"] != 1 {
|
||||
t.Fatalf("the state was watched %d times", watches["placement"])
|
||||
}
|
||||
// A read that starts a state's watch before its follower joins loses that follower no change: the follower is run over
|
||||
// the copy's current values when it joins, then hears every later change (review of ADR 0580 §2).
|
||||
if _, _, err := w.Get("haves", "n1.a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = handles["haves"](StateChange{Key: "n1.b", Op: OpPut, Value: []byte("2")}) // in the gap: no follower yet
|
||||
var joined []string
|
||||
w.Follow("haves", func(c StateChange) error { joined = append(joined, c.Key); return nil })
|
||||
_ = handles["haves"](StateChange{Key: "n2.c", Op: OpPut, Value: []byte("3")})
|
||||
if fmt.Sprint(joined) != "[a n1.b n2.c]" || watches["haves"] != 1 {
|
||||
t.Fatalf("a follower joining a watched state heard %v (watched %d times)", joined, watches["haves"])
|
||||
}
|
||||
for range 2 {
|
||||
if v, found, _ := w.Once("manifests", "d1"); !found || string(v) != "d1" {
|
||||
t.Fatalf("once read %q %v", v, found)
|
||||
}
|
||||
_, _, _ = w.Once("manifests", "missing")
|
||||
}
|
||||
if gets != 3 {
|
||||
t.Fatalf("a kept value was read by key %d times, want once, and the missing one twice", gets)
|
||||
}
|
||||
}
|
||||
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
package stdio
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WatchedStates is a copy of the states a module reads, each filled by its own watch and kept by its changes (novox/hq
|
||||
// ADR 0580 clause 1): a read of a whole state is one watch's current values, never a get per key, which took a
|
||||
// composition past 30 seconds and a compare past its call bound. A state is watched on its first read, or on Follow,
|
||||
// and the watch hands every current value over before the read is answered, so the copy is complete from the first
|
||||
// answer. A key whose value never changes once written (a manifest, by its digest) is read once with Once and kept.
|
||||
// A state helper, no rule: what a module reads is its own.
|
||||
type WatchedStates struct {
|
||||
watch func(state string, handle func(StateChange) error) error
|
||||
get func(state, key string) ([]byte, bool, error)
|
||||
|
||||
mu sync.Mutex
|
||||
states map[string]*watchedState
|
||||
// once holds each value read with Once, by state and key, for the process's life: a few kilobytes per manifest
|
||||
// digest. Past onceAtMost keys it is emptied, which costs only a read again.
|
||||
once map[string][]byte
|
||||
}
|
||||
|
||||
// onceAtMost bounds the values Once keeps.
|
||||
const onceAtMost = 4096
|
||||
|
||||
type watchedState struct {
|
||||
ready chan struct{} // closed once the watch's current values are all in the copy, or it was refused
|
||||
err error
|
||||
|
||||
mu sync.Mutex
|
||||
values map[string][]byte
|
||||
then []func(StateChange) error
|
||||
started bool // the current values are in, so a change runs then at once
|
||||
pending []StateChange // the current values, run through then once all are in
|
||||
thenMu sync.Mutex // then runs one change at a time
|
||||
}
|
||||
|
||||
// NewWatchedStates is the copy over this bundle's runtime: each state by KeptState.Watch, Once by KeptState.Get.
|
||||
func NewWatchedStates() *WatchedStates {
|
||||
return WatchedStatesOver(
|
||||
func(state string, handle func(StateChange) error) error { return State(state).Watch("", handle) },
|
||||
func(state, key string) ([]byte, bool, error) {
|
||||
e, err := State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
})
|
||||
}
|
||||
|
||||
// WatchedStatesOver is the copy over a watch and a get of one's own (a process's bus, a test's): watch hands handle
|
||||
// every current value and returns once they are handled, then hands every change; get reads one key.
|
||||
func WatchedStatesOver(watch func(state string, handle func(StateChange) error) error,
|
||||
get func(state, key string) ([]byte, bool, error)) *WatchedStates {
|
||||
return &WatchedStates{watch: watch, get: get, states: map[string]*watchedState{}, once: map[string][]byte{}}
|
||||
}
|
||||
|
||||
// watched is the state's copy, its watch started now when it has none, the caller waiting until its current values
|
||||
// are in. then, when given, runs after the copy took each change: the current values once they are all in (a then
|
||||
// that reads the copy reads it whole), each later change as it comes. No lock is held while the watch starts, so a
|
||||
// then may read any state of the copy.
|
||||
func (w *WatchedStates) watched(state string, then func(StateChange) error) (*watchedState, error) {
|
||||
w.mu.Lock()
|
||||
s, ok := w.states[state]
|
||||
if ok {
|
||||
w.mu.Unlock()
|
||||
<-s.ready
|
||||
if s.err != nil {
|
||||
return nil, s.err
|
||||
}
|
||||
if then != nil {
|
||||
s.join(state, then)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
s = &watchedState{ready: make(chan struct{}), values: map[string][]byte{}}
|
||||
if then != nil {
|
||||
s.then = []func(StateChange) error{then}
|
||||
}
|
||||
w.states[state] = s
|
||||
w.mu.Unlock()
|
||||
err := w.watch(state, func(c StateChange) error {
|
||||
s.mu.Lock()
|
||||
if c.Op == OpPut {
|
||||
s.values[c.Key] = append([]byte(nil), c.Value...)
|
||||
} else {
|
||||
delete(s.values, c.Key)
|
||||
}
|
||||
if !s.started {
|
||||
s.pending = append(s.pending, c)
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return s.run(state, c)
|
||||
})
|
||||
if err != nil {
|
||||
s.err = fmt.Errorf("the state %s is not watched: %w", state, err)
|
||||
w.mu.Lock()
|
||||
delete(w.states, state) // asked again on the next read
|
||||
w.mu.Unlock()
|
||||
close(s.ready)
|
||||
return nil, s.err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.started = true
|
||||
pending := s.pending
|
||||
s.pending = nil
|
||||
s.mu.Unlock()
|
||||
close(s.ready)
|
||||
for _, c := range pending {
|
||||
if err := s.run(state, c); err != nil {
|
||||
sayf("%v", err)
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// join adds then to a state already watched (a read started its watch first) and runs it over the copy's current values
|
||||
// before any later change, so no change that landed before it joined is lost to it (review of ADR 0580 §2: a read
|
||||
// beat the follower to the watch at start-up). Under thenMu, so a later change waits for these; a change the copy took
|
||||
// just before the snapshot may also be handed again, which a then that wakes work takes as a second wake.
|
||||
func (s *watchedState) join(state string, then func(StateChange) error) {
|
||||
s.thenMu.Lock()
|
||||
defer s.thenMu.Unlock()
|
||||
s.mu.Lock()
|
||||
s.then = append(s.then, then)
|
||||
current := make([]StateChange, 0, len(s.values))
|
||||
for k, v := range s.values {
|
||||
current = append(current, StateChange{State: state, Key: k, Op: OpPut, Value: v, Current: true})
|
||||
}
|
||||
s.mu.Unlock()
|
||||
sort.Slice(current, func(i, j int) bool { return current[i].Key < current[j].Key })
|
||||
for _, c := range current {
|
||||
if err := then(c); err != nil {
|
||||
sayf("%s %s: %v", state, c.Key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// run hands one change, already in the copy, to every then.
|
||||
func (s *watchedState) run(state string, c StateChange) error {
|
||||
s.thenMu.Lock()
|
||||
defer s.thenMu.Unlock()
|
||||
s.mu.Lock()
|
||||
then := append([]func(StateChange) error(nil), s.then...)
|
||||
s.mu.Unlock()
|
||||
var errs []error
|
||||
for _, f := range then {
|
||||
if err := f(c); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return fmt.Errorf("%s %s: %v", state, c.Key, errs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Follow watches the state into the copy and runs then on each change after the copy took it, so a reader then wakes
|
||||
// never reads a copy older than its change; it asks again until the watch is taken (2 s, doubled to a minute), as
|
||||
// KeptState.Follow does. On a state already watched then hears the changes from now on.
|
||||
func (w *WatchedStates) Follow(state string, then func(StateChange) error) {
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
if _, err := w.watched(state, then); err == nil {
|
||||
return
|
||||
} else {
|
||||
sayf("%v; asking again in %s", err, wait)
|
||||
}
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
// Get is a key's value in the copy, false when it has none.
|
||||
func (w *WatchedStates) Get(state, key string) ([]byte, bool, error) {
|
||||
s, err := w.watched(state, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v, ok := s.values[key]
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
// Keys are the state's keys in the copy, sorted.
|
||||
func (w *WatchedStates) Keys(state string) ([]string, error) {
|
||||
s, err := w.watched(state, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
keys := make([]string, 0, len(s.values))
|
||||
for k := range s.values {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// Values is every key's value in the copy.
|
||||
func (w *WatchedStates) Values(state string) (map[string][]byte, error) {
|
||||
s, err := w.watched(state, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make(map[string][]byte, len(s.values))
|
||||
for k, v := range s.values {
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Once is a key whose value never changes once written, read by key the first time it is found and kept after: a
|
||||
// manifest by its digest. A key not found is not kept, so it is read again on the next ask.
|
||||
func (w *WatchedStates) Once(state, key string) ([]byte, bool, error) {
|
||||
w.mu.Lock()
|
||||
v, ok := w.once[state+" "+key]
|
||||
w.mu.Unlock()
|
||||
if ok {
|
||||
return v, true, nil
|
||||
}
|
||||
v, found, err := w.get(state, key)
|
||||
if err != nil || !found {
|
||||
return nil, false, err
|
||||
}
|
||||
w.mu.Lock()
|
||||
if len(w.once) >= onceAtMost {
|
||||
clear(w.once)
|
||||
}
|
||||
w.once[state+" "+key] = v
|
||||
w.mu.Unlock()
|
||||
return v, true, nil
|
||||
}
|
||||
|
||||
// Direct is a key read by key now, not from the copy: a value that changes and is read one at a time (a build).
|
||||
func (w *WatchedStates) Direct(state, key string) ([]byte, bool, error) { return w.get(state, key) }
|
||||
Reference in New Issue
Block a user