Compare commits
9
Commits
go/v0.1.14
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c9beea9a38 | ||
|
|
74488f3e66 | ||
|
|
49cc5b3d7d | ||
|
|
08ec25e367 | ||
|
|
ec305f08e6 | ||
|
|
c050f23860 | ||
|
|
ae9bb17e0f | ||
|
|
be6eb0358f | ||
|
|
e1d2af8d6e |
@@ -2,5 +2,5 @@
|
|||||||
# The wallpaper (module feh, novox/hq ADR 0208). Owned by the mesh: replaced at every delivery. The
|
# The wallpaper (module feh, novox/hq ADR 0208). Owned by the mesh: replaced at every delivery. The
|
||||||
# session's start runs it, and so may anything that wants the declared wallpaper back. The image is
|
# session's start runs it, and so may anything that wants the declared wallpaper back. The image is
|
||||||
# the module's own, in ~/.local/share/feh/wallpapers. feh_set changes the wallpaper for a session
|
# the module's own, in ~/.local/share/feh/wallpapers. feh_set changes the wallpaper for a session
|
||||||
# without touching this file (the second single-module proof, 2026-11-17).
|
# without touching this file (design 47 H1, proof 1: delivered by its pipeline, 2026-11-21).
|
||||||
feh --no-fehbg --bg-fill "$HOME/.local/share/feh/wallpapers/default.jpg"
|
feh --no-fehbg --bg-fill "$HOME/.local/share/feh/wallpapers/default.jpg"
|
||||||
|
|||||||
+10
-1
@@ -182,6 +182,15 @@ export const FORGE_REQUEST_TIMEOUT_MS = 20_000;
|
|||||||
/** The most a tool's answer carries of one file: the bus's message bound is 1 MiB and a file travels as text (issue 1577). */
|
/** The most a tool's answer carries of one file: the bus's message bound is 1 MiB and a file travels as text (issue 1577). */
|
||||||
export const MAX_FILE_BYTES = 512 * 1024;
|
export const MAX_FILE_BYTES = 512 * 1024;
|
||||||
|
|
||||||
|
/** A directory of a listing's `files` the commit does not hold: the seat git answers it with the kind not-found, the error
|
||||||
|
* naming the directory (seat.ts failure; novox/hq design 47 H H-A-232). */
|
||||||
|
export class DirectoryNotInTree extends Error {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "DirectoryNotInTree";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** One request to the forge, cut at the limit and failing loud naming the request. */
|
/** One request to the forge, cut at the limit and failing loud naming the request. */
|
||||||
async function timedFetch(url: string, init: RequestInit, label: string, timeoutMs: number): Promise<Response> {
|
async function timedFetch(url: string, init: RequestInit, label: string, timeoutMs: number): Promise<Response> {
|
||||||
try {
|
try {
|
||||||
@@ -634,7 +643,7 @@ export class GiteaClient {
|
|||||||
// is whole (a cut one failed above), so a directory absent from it is absent at the commit.
|
// is whole (a cut one failed above), so a directory absent from it is absent at the commit.
|
||||||
filesOf = {};
|
filesOf = {};
|
||||||
for (const dir of files) {
|
for (const dir of files) {
|
||||||
if (dir !== "" && dirOf.get(dir) === undefined) throw new Error(`the directory ${dir} is not in ${owner}/${repo} at ${sha}`);
|
if (dir !== "" && dirOf.get(dir) === undefined) throw new DirectoryNotInTree(`the directory ${dir} is not in ${owner}/${repo} at ${sha}`);
|
||||||
const prefix = dir === "" ? "" : `${dir}/`;
|
const prefix = dir === "" ? "" : `${dir}/`;
|
||||||
const blobs: Record<string, string> = {};
|
const blobs: Record<string, string> = {};
|
||||||
for (const e of entries) {
|
for (const e of entries) {
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import {
|
|||||||
type TagAnswer, type TagsAnswer, type TreesAnswer, type ViewAnswer,
|
type TagAnswer, type TagsAnswer, type TreesAnswer, type ViewAnswer,
|
||||||
} from "@novox/mesh-delivery-contracts-git";
|
} from "@novox/mesh-delivery-contracts-git";
|
||||||
import type { CommitStatus, GiteaClient, GiteaComment, GiteaPull } from "./client.js";
|
import type { CommitStatus, GiteaClient, GiteaComment, GiteaPull } from "./client.js";
|
||||||
import { MAX_FILE_BYTES } from "./client.js";
|
import { DirectoryNotInTree, MAX_FILE_BYTES } from "./client.js";
|
||||||
import { containedFrom, newestByContext } from "./checks.js";
|
import { containedFrom, newestByContext } from "./checks.js";
|
||||||
import { Refused, checkName } from "./guard.js";
|
import { Refused, checkName } from "./guard.js";
|
||||||
import { appendNote, viewBody, viewComment, type Runner } from "./delivery.js";
|
import { appendNote, viewBody, viewComment, type Runner } from "./delivery.js";
|
||||||
@@ -64,6 +64,9 @@ export function failure(err: unknown): { error: string; "error-kind": ErrorKind
|
|||||||
const text = err instanceof Error ? err.message : String(err);
|
const text = err instanceof Error ? err.message : String(err);
|
||||||
// The forge's 404: what was asked about (a repository, a commit, a pull request, a branch, a tag) does not exist.
|
// The forge's 404: what was asked about (a repository, a commit, a pull request, a branch, a tag) does not exist.
|
||||||
if (/^Gitea API [^:]*: 404 /.test(text)) return { error: text, "error-kind": "not-found" };
|
if (/^Gitea API [^:]*: 404 /.test(text)) return { error: text, "error-kind": "not-found" };
|
||||||
|
// A directory of `files` the commit does not hold (client.ts commitTrees): not-found, the error naming the directory, so the
|
||||||
|
// indexer reads the others alone and leaves the gone one out (novox/hq design 47 H H-A-232).
|
||||||
|
if (err instanceof DirectoryNotInTree) return { error: text, "error-kind": "not-found" };
|
||||||
const refused = err instanceof Refused || /^Gitea API [^:]*: 4\d\d /.test(text) || /^Gitea API: /.test(text);
|
const refused = err instanceof Refused || /^Gitea API [^:]*: 4\d\d /.test(text) || /^Gitea API: /.test(text);
|
||||||
return { error: text, "error-kind": refused ? "refused" : "failed" };
|
return { error: text, "error-kind": refused ? "refused" : "failed" };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,8 +50,9 @@ async function serve(pages: any[]) {
|
|||||||
res.end(JSON.stringify(pages[page - 1] ?? { sha: COMMIT, tree: [], truncated: false, page, total_count: 0 }));
|
res.end(JSON.stringify(pages[page - 1] ?? { sha: COMMIT, tree: [], truncated: false, page, total_count: 0 }));
|
||||||
});
|
});
|
||||||
await new Promise<void>((r) => server.listen(0, r));
|
await new Promise<void>((r) => server.listen(0, r));
|
||||||
const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
|
const url = `http://127.0.0.1:${(server.address() as any).port}`;
|
||||||
return { client, server, asked };
|
const client = new GiteaClient(url, "t");
|
||||||
|
return { client, server, asked, url };
|
||||||
}
|
}
|
||||||
|
|
||||||
test("a commit's trees: the root and the tree of each directory holding a module.json", async () => {
|
test("a commit's trees: the root and the tree of each directory holding a module.json", async () => {
|
||||||
@@ -238,12 +239,19 @@ test("files: every blob under each asked directory, recursively, as its path rel
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("a directory of files the commit lacks, or spelled badly, or a listing the forge cut, fails and answers nothing", async () => {
|
test("a directory of files the commit lacks, or spelled badly, or a listing the forge cut, fails and answers nothing", async () => {
|
||||||
const { client, server } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
|
const { client, server, url } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
|
||||||
try {
|
try {
|
||||||
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, undefined, ["modules/a", "modules/gone"]), (err: Error) => {
|
// The seat answers it with the kind not-found naming the directory (novox/hq design 47 H H-A-232), never failed: the
|
||||||
assert.match(err.message, /modules\/gone/);
|
// holder's own client and failure, as the module runs them.
|
||||||
|
const { failure } = await import("../dist/seat.js");
|
||||||
|
const { GiteaClient } = await import("../dist/client.js");
|
||||||
|
await assert.rejects(new GiteaClient(url, "t").commitTrees("novox", "hq", COMMIT, undefined, ["modules/a", "modules/gone"]), (err: Error) => {
|
||||||
|
const said = failure(err);
|
||||||
|
assert.equal(said["error-kind"], "not-found", JSON.stringify(said));
|
||||||
|
assert.match(said.error, /the directory modules\/gone is not in novox\/hq/);
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
|
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, undefined, ["modules/a", "modules/gone"]), /modules\/gone/);
|
||||||
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, undefined, ["README.md"]), /README\.md/, "a file is no directory");
|
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, undefined, ["README.md"]), /README\.md/, "a file is no directory");
|
||||||
for (const bad of ["../x", "/abs", "a//b", "modules/a/", "."]) {
|
for (const bad of ["../x", "/abs", "a//b", "modules/a/", "."]) {
|
||||||
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, undefined, [bad]), /files/, bad);
|
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, undefined, [bad]), /files/, bad);
|
||||||
|
|||||||
@@ -208,11 +208,16 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str
|
|||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
defer func() { <-slots }()
|
defer func() { <-slots }()
|
||||||
|
// The checked repository's own modules come from the fetched tree, at whatever version go.mod names them
|
||||||
|
// (pins.go takeOwnRequirements; design 47 H, S31).
|
||||||
|
if taken[i], errs[i] = m.takeOwnRequirements(root, repository, filepath.Join(tree, d), d); errs[i] != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
// The asked versions of the mesh's packages go into the modfile before anything is downloaded (versions.go).
|
// The asked versions of the mesh's packages go into the modfile before anything is downloaded (versions.go).
|
||||||
if taken[i], errs[i] = m.takeVersions(ctx, root, filepath.Join(tree, d), d, req.Packages, &pins); errs[i] != nil {
|
more, err := m.takeVersions(ctx, root, repository, filepath.Join(tree, d), d, req.Packages, &pins)
|
||||||
|
if taken[i], errs[i] = append(taken[i], more...), err; errs[i] != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var more []TakenVersion
|
|
||||||
// Then what only a module the directory replaces names, which the download would not miss (pins.go).
|
// Then what only a module the directory replaces names, which the download would not miss (pins.go).
|
||||||
more, errs[i] = m.takeReplacedZeroPins(ctx, root, repository, filepath.Join(tree, d), d, req.Packages, taken[i], &pins)
|
more, errs[i] = m.takeReplacedZeroPins(ctx, root, repository, filepath.Join(tree, d), d, req.Packages, taken[i], &pins)
|
||||||
if taken[i] = append(taken[i], more...); errs[i] != nil {
|
if taken[i] = append(taken[i], more...); errs[i] != nil {
|
||||||
|
|||||||
@@ -296,8 +296,9 @@ func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) {
|
|||||||
return forgekey.Ran{}
|
return forgekey.Ran{}
|
||||||
}}
|
}}
|
||||||
m, states := fetchModule(t, f)
|
m, states := fetchModule(t, f)
|
||||||
// Asked for the module of tools, as a build is for one module (A-447b).
|
// Asked for the module of tools, as a build is for one module (A-447b), of a repository other than mesh-sdk's,
|
||||||
if _, err := fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "dir": "tools"}); err != nil {
|
// whose packages its go.mod requires: one of its own repository is built from the tree (S31).
|
||||||
|
if _, err := fetchTool(t, m)(map[string]any{"repository": "novox/mesh-member", "commit": commitSHA, "dir": "tools"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
st := states.waitEnd(t)
|
st := states.waitEnd(t)
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ var narrowedKeptAtRoot = []string{"module.json", "go.sum", "package-lock.json"}
|
|||||||
// narrowTree narrows the fetched tree for a build of an artifact that names its paths (ADR 0499 clause 4,
|
// narrowTree narrows the fetched tree for a build of an artifact that names its paths (ADR 0499 clause 4,
|
||||||
// steprun.Narrow). Beside the artifact's paths and the module's manifest, the workspace holds every generated-from
|
// steprun.Narrow). Beside the artifact's paths and the module's manifest, the workspace holds every generated-from
|
||||||
// directory an artifact of the module names, and the directory (with its go.mod) of every package of the same repository
|
// directory an artifact of the module names, and the directory (with its go.mod) of every package of the same repository
|
||||||
// the artifact's go.mod requires at the zero version, found by the manifests of that repository's modules; nothing else
|
// the artifact's go.mod requires at any version (S31), found by the manifests of that repository's modules; nothing else
|
||||||
// (design 47 H, lead answer H-A-136). A package of another repository comes published, as before (ADR 0573 §4).
|
// (design 47 H, lead answer H-A-136). A package of another repository comes published, as before (ADR 0573 §4).
|
||||||
func narrowTree(tree, repository string, req sdkstep.FetchRequest) error {
|
func narrowTree(tree, repository string, req sdkstep.FetchRequest) error {
|
||||||
dir := cleanDir(req.Dir)
|
dir := cleanDir(req.Dir)
|
||||||
@@ -30,7 +30,7 @@ func narrowTree(tree, repository string, req sdkstep.FetchRequest) error {
|
|||||||
return steprun.Narrow(tree, req) // refused there: named paths without the module's directory
|
return steprun.Narrow(tree, req) // refused there: named paths without the module's directory
|
||||||
}
|
}
|
||||||
extra := generatedFromDirs(tree, dir)
|
extra := generatedFromDirs(tree, dir)
|
||||||
extra = append(extra, ownZeroPackageDirs(tree, repository, dir, req.Paths)...)
|
extra = append(extra, ownPackageDirs(tree, repository, dir, req.Paths)...)
|
||||||
var outside []string
|
var outside []string
|
||||||
inside := slices.Clone(req.Paths)
|
inside := slices.Clone(req.Paths)
|
||||||
for _, e := range extra {
|
for _, e := range extra {
|
||||||
@@ -127,18 +127,19 @@ func generatedFromDirs(tree, dir string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// ownZeroPackageDirs are the directories (from the tree's root) of the packages of the checked repository that a go.mod
|
// ownPackageDirs are the directories (from the tree's root) of the packages of the checked repository (the fetch's
|
||||||
// under the artifact's named paths requires at the zero version (H-A-136; ADR 0573 §2), transitively: the go.mod of each
|
// repository, sameRepository) that a go.mod under the artifact's named paths requires, at whatever version it names them
|
||||||
// such directory is read in turn for the packages of the repository it requires at the zero version, each directory
|
// (H-A-136; ADR 0573 §2; design 47 H, S31), transitively: the go.mod of each such directory is read in turn for the
|
||||||
// once (design 47 H, H-A-202, H-A-198). Each is found by the manifests of the repository's modules: a module's directory
|
// packages of the repository it requires, each directory once (design 47 H, H-A-202, H-A-198). Each is found by the manifests of the repository's modules: a module's directory
|
||||||
// or the directory of one of its artifacts' paths whose go.mod declares the required module path. One no manifest leads
|
// or the directory of one of its artifacts' paths whose go.mod declares the required module path. One no manifest leads
|
||||||
// to is left to the fetch's resolution, which fails naming it.
|
// to is left to the fetch's resolution, which fails naming it.
|
||||||
func ownZeroPackageDirs(tree, repository, dir string, paths []string) []string {
|
func ownPackageDirs(tree, repository, dir string, paths []string) []string {
|
||||||
want := map[string]bool{}
|
want := map[string]bool{}
|
||||||
addReqs := func(raw []byte) {
|
addReqs := func(raw []byte) {
|
||||||
for _, pn := range forgePins(string(raw)) {
|
required, _ := goModRequirements(string(raw))
|
||||||
if pn.short == zeroShort && "novox/"+pn.repo == repository {
|
for module := range required {
|
||||||
want[pn.module] = true
|
if sameRepository(module, repository) {
|
||||||
|
want[module] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ func TestANarrowedWorkspaceHoldsTheGeneratedFromDirectoriesAndItsRepositorysZero
|
|||||||
zero := " v0.0.0-00010101000000-" + zeroShort
|
zero := " v0.0.0-00010101000000-" + zeroShort
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name, dir string
|
name, dir string
|
||||||
|
repository string // the fetch's repository, novox/mesh-delivery when empty
|
||||||
paths []string
|
paths []string
|
||||||
files map[string]string
|
files map[string]string
|
||||||
kept, gone []string
|
kept, gone []string
|
||||||
@@ -114,6 +115,25 @@ func TestANarrowedWorkspaceHoldsTheGeneratedFromDirectoriesAndItsRepositorysZero
|
|||||||
kept: []string{"modules/a/state/s.go", "modules/b/builds/b.go", "modules/c/client/c.go"},
|
kept: []string{"modules/a/state/s.go", "modules/b/builds/b.go", "modules/c/client/c.go"},
|
||||||
gone: []string{"modules/b/other", "modules/c/other", "modules/d"},
|
gone: []string{"modules/b/other", "modules/c/other", "modules/d"},
|
||||||
modules: []string{"modules/a/state"}},
|
modules: []string{"modules/a/state"}},
|
||||||
|
// S31, L8: a package of the repository required at a real version comes along too, transitively, and "the
|
||||||
|
// repository" is the fetch's, under any owner; a package of another repository at a real version does not.
|
||||||
|
{name: "the repository's packages at a real version, under another owner", repository: "kin/app", dir: "modules/a", paths: []string{"state"},
|
||||||
|
files: map[string]string{
|
||||||
|
"modules/a/module.json": `{"module":"a","build":{"artifacts":[{"name":"state","kind":"package","language":"go","paths":["state"]}]}}`,
|
||||||
|
"modules/a/state/go.mod": "module git.novox.be/kin/app/modules/a/state\n\ngo 1.26\n\nrequire (\n\tgit.novox.be/kin/app/modules/b/builds v0.1.3\n\tgit.novox.be/novox/app/modules/d/x v0.1.3\n)\n",
|
||||||
|
"modules/a/state/s.go": "package state\n",
|
||||||
|
"modules/b/module.json": `{"module":"b","build":{"artifacts":[{"name":"builds","kind":"package","language":"go","paths":["builds"]}]}}`,
|
||||||
|
"modules/b/builds/go.mod": "module git.novox.be/kin/app/modules/b/builds\n\ngo 1.26\n\nrequire git.novox.be/kin/app/modules/c/client v0.2.0\n",
|
||||||
|
"modules/b/builds/b.go": "package builds\n",
|
||||||
|
"modules/c/module.json": `{"module":"c","build":{"artifacts":[{"name":"client","kind":"package","language":"go","paths":["client"]}]}}`,
|
||||||
|
"modules/c/client/go.mod": "module git.novox.be/kin/app/modules/c/client\n\ngo 1.26\n",
|
||||||
|
"modules/c/client/c.go": "package client\n",
|
||||||
|
"modules/d/module.json": `{"module":"d","build":{"artifacts":[{"name":"x","kind":"package","language":"go","paths":["x"]}]}}`,
|
||||||
|
"modules/d/x/go.mod": "module git.novox.be/novox/app/modules/d/x\n\ngo 1.26\n",
|
||||||
|
},
|
||||||
|
kept: []string{"modules/a/state/s.go", "modules/b/builds/b.go", "modules/c/client/c.go"},
|
||||||
|
gone: []string{"modules/d"},
|
||||||
|
modules: []string{"modules/a/state"}},
|
||||||
// A named path that leaves the repository is refused.
|
// A named path that leaves the repository is refused.
|
||||||
{name: "a named path that leaves the repository", dir: "mesh-grants", paths: []string{".", "../../outside"},
|
{name: "a named path that leaves the repository", dir: "mesh-grants", paths: []string{".", "../../outside"},
|
||||||
files: map[string]string{"mesh-grants/module.json": `{"module":"mesh-grants"}`, "mesh-grants/go.mod": "module mesh-grants\n\ngo 1.26\n"},
|
files: map[string]string{"mesh-grants/module.json": `{"module":"mesh-grants"}`, "mesh-grants/go.mod": "module mesh-grants\n\ngo 1.26\n"},
|
||||||
@@ -153,7 +173,11 @@ func TestANarrowedWorkspaceHoldsTheGeneratedFromDirectoriesAndItsRepositorysZero
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
req := sdkstep.FetchRequest{Repository: "novox/mesh-delivery", Commit: commitSHA, Dir: tc.dir, Paths: tc.paths}
|
repository := tc.repository
|
||||||
|
if repository == "" {
|
||||||
|
repository = "novox/mesh-delivery"
|
||||||
|
}
|
||||||
|
req := sdkstep.FetchRequest{Repository: repository, Commit: commitSHA, Dir: tc.dir, Paths: tc.paths}
|
||||||
err := narrowTree(tree, req.Repository, req)
|
err := narrowTree(tree, req.Repository, req)
|
||||||
if tc.refused != "" {
|
if tc.refused != "" {
|
||||||
if err == nil || !strings.Contains(err.Error(), tc.refused) {
|
if err == nil || !strings.Contains(err.Error(), tc.refused) {
|
||||||
|
|||||||
@@ -25,7 +25,19 @@ import (
|
|||||||
dforgekey "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/forgekey"
|
dforgekey "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/forgekey"
|
||||||
)
|
)
|
||||||
|
|
||||||
const forgePrefix = "git.novox.be/novox/"
|
// goHost is the host the forge's Go modules are named under: a module of the repository <owner>/<name> has the path
|
||||||
|
// goHost+"<owner>/<name>" or one under it.
|
||||||
|
const goHost = "git.novox.be/"
|
||||||
|
|
||||||
|
const forgePrefix = goHost + "novox/"
|
||||||
|
|
||||||
|
// sameRepository says whether the Go module path lies in the repository the fetch is of (its owner/name, as the ask
|
||||||
|
// names it): the repository's root module or one nested in it (design 47 H, S31, L8: read from the fetch, never from an
|
||||||
|
// owner name written into the code).
|
||||||
|
func sameRepository(module, repository string) bool {
|
||||||
|
prefix := goHost + repository
|
||||||
|
return repository != "" && (module == prefix || strings.HasPrefix(module, prefix+"/"))
|
||||||
|
}
|
||||||
|
|
||||||
// zeroShort is the 12 digits a requirement at the zero version v0.0.0-00010101000000-000000000000 ends in: no commit.
|
// zeroShort is the 12 digits a requirement at the zero version v0.0.0-00010101000000-000000000000 ends in: no commit.
|
||||||
const zeroShort = "000000000000"
|
const zeroShort = "000000000000"
|
||||||
@@ -381,10 +393,16 @@ func (m *Module) takeZeroRequirements(ctx context.Context, root, repository, dir
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
replacedBefore := ""
|
||||||
|
if modfile, ok := m.pinModfile(dir); ok {
|
||||||
|
if b, err := os.ReadFile(modfile); err == nil {
|
||||||
|
replacedBefore = string(b)
|
||||||
|
}
|
||||||
|
}
|
||||||
var taken []TakenVersion
|
var taken []TakenVersion
|
||||||
for _, r := range reqs {
|
for _, r := range reqs {
|
||||||
p := r.pin
|
p := r.pin
|
||||||
if v := resolved[p.module]; v != "" && !strings.HasSuffix(v, zeroShort) {
|
if v := resolved[p.module]; (v != "" && !strings.HasSuffix(v, zeroShort)) || strings.Contains(replacedBefore, "replace "+p.module+" =>") {
|
||||||
continue // taken before
|
continue // taken before
|
||||||
}
|
}
|
||||||
named := "v0.0.0-00010101000000-" + zeroShort
|
named := "v0.0.0-00010101000000-" + zeroShort
|
||||||
@@ -392,10 +410,10 @@ func (m *Module) takeZeroRequirements(ctx context.Context, root, repository, dir
|
|||||||
if r.by != "" {
|
if r.by != "" {
|
||||||
namedBy = "; " + namedBySaid(r.by)
|
namedBy = "; " + namedBySaid(r.by)
|
||||||
}
|
}
|
||||||
if "novox/"+p.repo == repository {
|
if sameRepository(p.module, repository) {
|
||||||
// A module of the checked repository itself: it is in the fetched tree and nowhere else, since nothing of the
|
// A module of the checked repository itself: it is in the fetched tree and nowhere else, since nothing of the
|
||||||
// repository is published before its merge. Every module of the tree goes in, as one may require another.
|
// repository is published before its merge. Every module of the tree goes in, as one may require another.
|
||||||
own, err := m.useOwnModules(dir, root, p.repo)
|
own, err := m.useOwnModules(dir, root, repository)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return taken, fmt.Errorf("the requirement of %s at the zero version: %w", p.module, err)
|
return taken, fmt.Errorf("the requirement of %s at the zero version: %w", p.module, err)
|
||||||
}
|
}
|
||||||
@@ -431,12 +449,79 @@ func (m *Module) takeZeroRequirements(ctx context.Context, root, repository, dir
|
|||||||
// namedBySaid says that a module dir replaces by the directory by names a package, and dir's go.mod does not.
|
// namedBySaid says that a module dir replaces by the directory by names a package, and dir's go.mod does not.
|
||||||
func namedBySaid(by string) string { return "named by " + by + ", not by this go.mod" }
|
func namedBySaid(by string) string { return "named by " + by + ", not by this go.mod" }
|
||||||
|
|
||||||
|
// ownRequirement is a requirement of a module of the checked repository in the module graph of a directory, at the
|
||||||
|
// version its go.mod names (any version): by as in zeroRequirement.
|
||||||
|
type ownRequirement struct {
|
||||||
|
module, named, by string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownRequirementsOf are the requirements of dir's module graph (its go.mod, then the go.mod of each module it replaces by
|
||||||
|
// a directory, as zeroRequirementsOf reads them) that lie in the repository the fetch is of, at whatever version they are
|
||||||
|
// named, each once, own first. One dir's go.mod replaces is left out, since its replace decides it.
|
||||||
|
func ownRequirementsOf(dir, repository string) []ownRequirement {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(dir, "go.mod"))
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, replaced := goModRequirements(string(raw))
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []ownRequirement
|
||||||
|
add := func(gomod, by string) {
|
||||||
|
required, _ := goModRequirements(gomod)
|
||||||
|
for _, module := range slices.Sorted(maps.Keys(required)) {
|
||||||
|
if !sameRepository(module, repository) || seen[module] || replaced[module] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[module] = true
|
||||||
|
out = append(out, ownRequirement{module: module, named: required[module], by: by})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
add(string(raw), "")
|
||||||
|
local := localReplaces(string(raw))
|
||||||
|
for _, module := range slices.Sorted(maps.Keys(local)) {
|
||||||
|
if b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(local[module]), "go.mod")); err == nil {
|
||||||
|
add(string(b), local[module])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeOwnRequirements takes, before anything is downloaded, every requirement of dir's module graph that lies in the
|
||||||
|
// checked repository from the fetched tree, at whatever version go.mod names it, the zero version or a real one (design
|
||||||
|
// 47 H, S3, S31; H-A-198, H-A-202): the indexer never pins it and its fingerprint takes the tree's objects, so the build
|
||||||
|
// takes the tree too and never the Go seat's published version. One no module of the tree declares fails the fetch,
|
||||||
|
// naming it.
|
||||||
|
func (m *Module) takeOwnRequirements(root, repository, dir, d string) ([]TakenVersion, error) {
|
||||||
|
reqs := ownRequirementsOf(dir, repository)
|
||||||
|
if len(reqs) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
own, err := m.useOwnModules(dir, root, repository)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the requirements of %s: %w", repository, err)
|
||||||
|
}
|
||||||
|
var taken []TakenVersion
|
||||||
|
for _, r := range reqs {
|
||||||
|
if !slices.Contains(own, r.module) {
|
||||||
|
return taken, fmt.Errorf("the requirement of %s at %s: no module of that path is in the fetched tree of %s, and a module of the checked repository is built from its tree", r.module, r.named, repository)
|
||||||
|
}
|
||||||
|
said := "a module of the checked repository: built from its tree"
|
||||||
|
if r.by != "" {
|
||||||
|
said += "; " + namedBySaid(r.by)
|
||||||
|
}
|
||||||
|
if m.Say != nil {
|
||||||
|
m.Say("%s: %s (go.mod names %s) is built from the fetched tree of %s", d, r.module, r.named, repository)
|
||||||
|
}
|
||||||
|
taken = append(taken, TakenVersion{Dir: d, Package: r.module, Named: r.named, Built: "the fetched tree", Said: said})
|
||||||
|
}
|
||||||
|
return taken, nil
|
||||||
|
}
|
||||||
|
|
||||||
// useOwnModules replaces, in dir's temporary modfile, every module of the fetched tree whose path lies in the repository
|
// useOwnModules replaces, in dir's temporary modfile, every module of the fetched tree whose path lies in the repository
|
||||||
// repo by its directory in the tree (once), and answers the paths of those modules. A replace of a module dir does not
|
// the fetch is of (sameRepository) by its directory in the tree (once), and answers the paths of those modules. A
|
||||||
// require is harmless to Go; one module of a repository requires another of it at the zero version.
|
// replace of a module dir does not require is harmless to Go; one module of a repository requires another of it.
|
||||||
func (m *Module) useOwnModules(dir, root, repo string) ([]string, error) {
|
func (m *Module) useOwnModules(dir, root, repository string) ([]string, error) {
|
||||||
tree := filepath.Join(root, "tree")
|
tree := filepath.Join(root, "tree")
|
||||||
prefix := forgePrefix + repo
|
|
||||||
type found struct{ module, where string }
|
type found struct{ module, where string }
|
||||||
var mods []found
|
var mods []found
|
||||||
err := filepath.WalkDir(tree, func(path string, e fs.DirEntry, err error) error {
|
err := filepath.WalkDir(tree, func(path string, e fs.DirEntry, err error) error {
|
||||||
@@ -458,7 +543,7 @@ func (m *Module) useOwnModules(dir, root, repo string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
for _, line := range strings.Split(string(raw), "\n") {
|
for _, line := range strings.Split(string(raw), "\n") {
|
||||||
if f := strings.Fields(line); len(f) == 2 && f[0] == "module" {
|
if f := strings.Fields(line); len(f) == 2 && f[0] == "module" {
|
||||||
if mod := strings.Trim(f[1], `"`); (mod == prefix || strings.HasPrefix(mod, prefix+"/")) && filepath.Dir(path) != dir {
|
if mod := strings.Trim(f[1], `"`); sameRepository(mod, repository) && filepath.Dir(path) != dir {
|
||||||
mods = append(mods, found{mod, filepath.Dir(path)})
|
mods = append(mods, found{mod, filepath.Dir(path)})
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -82,7 +82,9 @@ func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinH
|
|||||||
|
|
||||||
func (h *pinHarness) fetch(states *keptStates) (FetchResult, sdkstep.Step) {
|
func (h *pinHarness) fetch(states *keptStates) (FetchResult, sdkstep.Step) {
|
||||||
h.t.Helper()
|
h.t.Helper()
|
||||||
if _, err := fetchTool(h.t, h.m)(fetchArgs); err != nil {
|
// A member of a repository other than mesh-sdk's, whose packages it pins: one of its own repository is built from the
|
||||||
|
// tree (S31).
|
||||||
|
if _, err := fetchTool(h.t, h.m)(map[string]any{"repository": "novox/mesh-member", "commit": commitSHA}); err != nil {
|
||||||
h.t.Fatal(err)
|
h.t.Fatal(err)
|
||||||
}
|
}
|
||||||
st := states.waitEnd(h.t)
|
st := states.waitEnd(h.t)
|
||||||
@@ -405,55 +407,64 @@ func TestARequirementAtTheZeroVersionIsTakenFromItsPinOrRefused(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A requirement at the zero version of a module of the checked repository itself (a nested module not yet published, which
|
// A requirement of a module of the checked repository itself (a nested module not yet published, which another module of
|
||||||
// another module of the repository requires) is built from the fetched tree, and never asked of Go or the forge (issue 1921).
|
// the repository requires) is built from the fetched tree, and never asked of Go or the forge (issue 1921), at whatever
|
||||||
|
// version go.mod names it: the zero version or a real one, which the Go seat would otherwise serve while the indexer's
|
||||||
|
// fingerprint takes the tree (design 47 H, S31; H-A-198, H-A-202). "The checked repository" is the fetch's repository as
|
||||||
|
// the ask names it, under any owner, never an owner written into the code (L8).
|
||||||
// Narrowed: a fetch of one artifact of a tree shaped like mesh-delivery's (modules/mesh-delivery-contracts/go/clients/<c>
|
// Narrowed: a fetch of one artifact of a tree shaped like mesh-delivery's (modules/mesh-delivery-contracts/go/clients/<c>
|
||||||
// requiring modules/mesh-delivery-sdk at the zero version), asked with the module's directory and the artifact's paths,
|
// requiring modules/mesh-delivery-sdk), asked with the module's directory and the artifact's paths, keeps the sdk's
|
||||||
// keeps the sdk's directory, found by its manifest, and resolves it there (design 47 H, H-A-136; live 2026-11-20,
|
// directory, found by its manifest, and resolves it there (design 47 H, H-A-136; live 2026-11-20, pipeline main.3: "no
|
||||||
// pipeline main.3: "no module of that path is in the fetched tree of novox/mesh-delivery").
|
// module of that path is in the fetched tree of novox/mesh-delivery").
|
||||||
func TestARequirementAtTheZeroVersionOfTheCheckedRepositorysOwnModuleIsTakenFromItsTree(t *testing.T) {
|
func TestARequirementOfTheCheckedRepositorysOwnModuleIsTakenFromItsTree(t *testing.T) {
|
||||||
|
zero := "v0.0.0-00010101000000-" + zeroShort
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name, own, ownDir, gomods string
|
name, own, ownDir, gomods, version string
|
||||||
args map[string]any
|
args map[string]any
|
||||||
narrowed bool
|
narrowed bool
|
||||||
}{
|
}{
|
||||||
{name: "whole tree", own: "git.novox.be/novox/mesh-delivery/nested/go", ownDir: "nested/go", gomods: "go.mod\n",
|
{name: "whole tree", own: "git.novox.be/novox/mesh-delivery/nested/go", ownDir: "nested/go", gomods: "go.mod\n", version: zero,
|
||||||
args: map[string]any{"repository": "novox/mesh-delivery", "commit": commitSHA}},
|
args: map[string]any{"repository": "novox/mesh-delivery", "commit": commitSHA}},
|
||||||
|
{name: "whole tree, at a real version", own: "git.novox.be/novox/mesh-delivery/nested/go", ownDir: "nested/go", gomods: "go.mod\n",
|
||||||
|
version: "v0.1.3", args: map[string]any{"repository": "novox/mesh-delivery", "commit": commitSHA}},
|
||||||
|
{name: "a repository of another owner, judged by the ask's repository", own: "git.novox.be/kin/app/nested/go", ownDir: "nested/go",
|
||||||
|
gomods: "go.mod\n", version: zero, args: map[string]any{"repository": "kin/app", "commit": commitSHA}},
|
||||||
{name: "narrowed to an artifact's paths", own: "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk",
|
{name: "narrowed to an artifact's paths", own: "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk",
|
||||||
ownDir: "modules/mesh-delivery-sdk", gomods: "modules/mesh-delivery-contracts/go/clients/meshcatalogclient/go.mod\n",
|
ownDir: "modules/mesh-delivery-sdk", gomods: "modules/mesh-delivery-contracts/go/clients/meshcatalogclient/go.mod\n", version: zero,
|
||||||
|
args: map[string]any{"repository": "novox/mesh-delivery", "commit": commitSHA, "dir": "modules/mesh-delivery-contracts",
|
||||||
|
"paths": []any{"go/clients/meshcatalogclient"}}, narrowed: true},
|
||||||
|
{name: "narrowed to an artifact's paths, at a real version", own: "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk",
|
||||||
|
ownDir: "modules/mesh-delivery-sdk", gomods: "modules/mesh-delivery-contracts/go/clients/meshcatalogclient/go.mod\n", version: "v0.1.3",
|
||||||
args: map[string]any{"repository": "novox/mesh-delivery", "commit": commitSHA, "dir": "modules/mesh-delivery-contracts",
|
args: map[string]any{"repository": "novox/mesh-delivery", "commit": commitSHA, "dir": "modules/mesh-delivery-contracts",
|
||||||
"paths": []any{"go/clients/meshcatalogclient"}}, narrowed: true},
|
"paths": []any{"go/clients/meshcatalogclient"}}, narrowed: true},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
zeroMod := "module fixture/member\n\ngo 1.26\n\nrequire (\n\t" + tc.own + " v0.0.0-00010101000000-" + zeroShort + "\n)\n"
|
ownMod := "module fixture/member\n\ngo 1.26\n\nrequire (\n\t" + tc.own + " " + tc.version + "\n)\n"
|
||||||
h, states := newPinHarness(t, zeroMod, false, pullList)
|
h, states := newPinHarness(t, ownMod, false, pullList)
|
||||||
h.f.gomods = tc.gomods
|
h.f.gomods = tc.gomods
|
||||||
base := h.m.Run
|
base := h.m.Run
|
||||||
h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||||
line := name + " " + strings.Join(args, " ")
|
line := name + " " + strings.Join(args, " ")
|
||||||
if name == "git" && strings.Contains(line, " checkout ") && tc.narrowed {
|
if name == "git" && strings.Contains(line, " checkout ") {
|
||||||
r := base(ctx, env, name, args...)
|
r := base(ctx, env, name, args...)
|
||||||
tree := args[slices.Index(args, "-C")+1]
|
tree := args[slices.Index(args, "-C")+1]
|
||||||
for f, body := range map[string]string{
|
files := map[string]string{tc.ownDir + "/go.mod": "module " + tc.own + "\n\ngo 1.26\n"}
|
||||||
"modules/mesh-delivery-sdk/go.mod": "module " + tc.own + "\n\ngo 1.26\n",
|
if tc.narrowed {
|
||||||
"modules/mesh-delivery-sdk/module.json": `{"module":"mesh-delivery-sdk","build":{"artifacts":[{"name":"go-sdk","kind":"package","language":"go"}]}}`,
|
files["modules/mesh-delivery-sdk/module.json"] = `{"module":"mesh-delivery-sdk","build":{"artifacts":[{"name":"go-sdk","kind":"package","language":"go"}]}}`
|
||||||
"modules/mesh-delivery-contracts/module.json": `{"module":"mesh-delivery-contracts","build":{"artifacts":[{"name":"go-meshcatalogclient","kind":"package","language":"go","paths":["go/clients/meshcatalogclient"]}]}}`,
|
files["modules/mesh-delivery-contracts/module.json"] = `{"module":"mesh-delivery-contracts","build":{"artifacts":[{"name":"go-meshcatalogclient","kind":"package","language":"go","paths":["go/clients/meshcatalogclient"]}]}}`
|
||||||
"modules/mesh-delivery-view/go.mod": "module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view\n\ngo 1.26\n",
|
files["modules/mesh-delivery-view/go.mod"] = "module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view\n\ngo 1.26\n"
|
||||||
} {
|
}
|
||||||
|
for f, body := range files {
|
||||||
os.MkdirAll(filepath.Join(tree, filepath.Dir(f)), 0o755)
|
os.MkdirAll(filepath.Join(tree, filepath.Dir(f)), 0o755)
|
||||||
os.WriteFile(filepath.Join(tree, f), []byte(body), 0o644)
|
os.WriteFile(filepath.Join(tree, f), []byte(body), 0o644)
|
||||||
}
|
}
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
if name == "go" && strings.Contains(line, " mod download") && !strings.Contains(strings.Join(env, " "), "-modfile=") {
|
if name == "go" && strings.Contains(line, " mod download") && !strings.Contains(strings.Join(env, " "), "-modfile=") {
|
||||||
if !tc.narrowed {
|
if tc.version != zero {
|
||||||
trees, _ := filepath.Glob(filepath.Join(h.m.Work, "*", "tree"))
|
return forgekey.Ran{} // the Go seat serves the published version: what a build of the tree must never take
|
||||||
for _, tree := range trees {
|
|
||||||
os.MkdirAll(filepath.Join(tree, tc.ownDir), 0o755)
|
|
||||||
os.WriteFile(filepath.Join(tree, tc.ownDir, "go.mod"), []byte("module "+tc.own+"\n\ngo 1.26\n"), 0o644)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return forgekey.Ran{Status: 1, Stderr: "go: " + tc.own + "@v0.0.0-00010101000000-" + zeroShort + ": invalid version: unknown revision " + zeroShort + "\n"}
|
return forgekey.Ran{Status: 1, Stderr: "go: " + tc.own + "@" + zero + ": invalid version: unknown revision " + zeroShort + "\n"}
|
||||||
}
|
}
|
||||||
return base(ctx, env, name, args...)
|
return base(ctx, env, name, args...)
|
||||||
}
|
}
|
||||||
@@ -485,6 +496,11 @@ func TestARequirementAtTheZeroVersionOfTheCheckedRepositorysOwnModuleIsTakenFrom
|
|||||||
if h.ran(" mod download -json ") || h.ran("ls-remote") {
|
if h.ran(" mod download -json ") || h.ran("ls-remote") {
|
||||||
t.Error("a module of the checked repository was asked of Go or looked for among the pull request heads")
|
t.Error("a module of the checked repository was asked of Go or looked for among the pull request heads")
|
||||||
}
|
}
|
||||||
|
var res FetchResult
|
||||||
|
json.Unmarshal(st.Result, &res)
|
||||||
|
if len(res.Packages) != 1 || res.Packages[0].Package != tc.own || res.Packages[0].Built != "the fetched tree" || res.Packages[0].Named != tc.version {
|
||||||
|
t.Errorf("the taken versions = %+v, want %s named %s, built from the fetched tree", res.Packages, tc.own, tc.version)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,10 +58,10 @@ type downloadInfo struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// takeVersions writes the asked versions into the modfile of dir (d in the tree) and answers what it took, one entry per
|
// takeVersions writes the asked versions into the modfile of dir (d in the tree) and answers what it took, one entry per
|
||||||
// asked package its go.mod requires. A package go.mod replaces is built as go.mod says, and said. A version that can be
|
// asked package its go.mod requires that is not of the checked repository repository. A package go.mod replaces is built as go.mod says, and said. A version that can be
|
||||||
// neither fetched nor found among the forge's pull request heads fails the fetch, naming it. pins serialises the
|
// neither fetched nor found among the forge's pull request heads fails the fetch, naming it. pins serialises the
|
||||||
// laid copies a workspace's directories share.
|
// laid copies a workspace's directories share.
|
||||||
func (m *Module) takeVersions(ctx context.Context, root, dir, d string, asked sdkstep.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) {
|
func (m *Module) takeVersions(ctx context.Context, root, repository, dir, d string, asked sdkstep.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) {
|
||||||
raw, err := os.ReadFile(filepath.Join(dir, "go.mod"))
|
raw, err := os.ReadFile(filepath.Join(dir, "go.mod"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -77,8 +77,8 @@ func (m *Module) takeVersions(ctx context.Context, root, dir, d string, asked sd
|
|||||||
var taken []TakenVersion
|
var taken []TakenVersion
|
||||||
for _, pkg := range slices.Sorted(maps.Keys(asked)) {
|
for _, pkg := range slices.Sorted(maps.Keys(asked)) {
|
||||||
named, ok := required[pkg]
|
named, ok := required[pkg]
|
||||||
if !ok {
|
if !ok || sameRepository(pkg, repository) {
|
||||||
continue // the module does not use it
|
continue // the module does not use it, or it is of the checked repository: built from its tree (takeOwnRequirements)
|
||||||
}
|
}
|
||||||
tv := TakenVersion{Dir: d, Package: pkg, Asked: asked[pkg], Named: named}
|
tv := TakenVersion{Dir: d, Package: pkg, Asked: asked[pkg], Named: named}
|
||||||
switch {
|
switch {
|
||||||
|
|||||||
@@ -45,11 +45,15 @@ func AsksOf(e catalog.CatalogEntry) (Asks, error) {
|
|||||||
return Asks{}, fmt.Errorf("the manifest of %s is not readable: %w", e.Module, err)
|
return Asks{}, fmt.Errorf("the manifest of %s is not readable: %w", e.Module, err)
|
||||||
}
|
}
|
||||||
state := m.BusState
|
state := m.BusState
|
||||||
|
// The old sections' entries are asked for too (design 47 H H-A-229, H-A-230), every manifest's, named and capped as
|
||||||
|
// the grants part composes them.
|
||||||
|
m.Module = cmp.Or(m.Module, e.Module)
|
||||||
|
old := keeper.OldPlansOf(m)
|
||||||
if e.Verdict != "" && e.Verdict != "accepted" {
|
if e.Verdict != "" && e.Verdict != "accepted" {
|
||||||
state = nil
|
state, old = nil, nil
|
||||||
}
|
}
|
||||||
sum := sha256.Sum256(mustJSON(state))
|
sum := sha256.Sum256(mustJSON(state))
|
||||||
out := Asks{Accepted: keeper.Accepted{Module: e.Module, Digest: hex.EncodeToString(sum[:]), BusState: state},
|
out := Asks{Accepted: keeper.Accepted{Module: e.Module, Digest: hex.EncodeToString(sum[:]), BusState: state, Old: old},
|
||||||
EmitsOwn: len(m.Emits) > 0 || len(e.Streams) > 0}
|
EmitsOwn: len(m.Emits) > 0 || len(e.Streams) > 0}
|
||||||
for _, c := range m.Claims {
|
for _, c := range m.Claims {
|
||||||
out.Claims = append(out.Claims, c.Name)
|
out.Claims = append(out.Claims, c.Name)
|
||||||
@@ -81,7 +85,7 @@ type Branch struct {
|
|||||||
// branches in its registration's order (order: the registration's branches; a branch it does not name comes after, in
|
// branches in its registration's order (order: the registration's branches; a branch it does not name comes after, in
|
||||||
// the order given) whose entry asks it; another branch asking that local name with other settings is a drift naming
|
// the order given) whose entry asks it; another branch asking that local name with other settings is a drift naming
|
||||||
// the branch, and plans nothing. The module sends when a branch's entry says so; it is a foundation part, and its old
|
// the branch, and plans nothing. The module sends when a branch's entry says so; it is a foundation part, and its old
|
||||||
// sections' caps are read, from the first branch.
|
// sections' caps and entries are read, from the first branch (design 47 H H-A-229).
|
||||||
func ViewOf(branches []Branch, order func(module string) []string) (keeper.CatalogView, error) {
|
func ViewOf(branches []Branch, order func(module string) []string) (keeper.CatalogView, error) {
|
||||||
modules := map[string]bool{}
|
modules := map[string]bool{}
|
||||||
for _, b := range branches {
|
for _, b := range branches {
|
||||||
@@ -117,8 +121,8 @@ func ViewOf(branches []Branch, order func(module string) []string) (keeper.Catal
|
|||||||
return keeper.CatalogView{}, fmt.Errorf("the branch %s: %w", b.Name, err)
|
return keeper.CatalogView{}, fmt.Errorf("the branch %s: %w", b.Name, err)
|
||||||
}
|
}
|
||||||
if i == 0 {
|
if i == 0 {
|
||||||
cm = keeper.CatalogModule{Accepted: keeper.Accepted{Module: module, Digest: b.Entries[module].ManifestDigest, Revision: uint64(max(b.Revision, 0))},
|
cm = keeper.CatalogModule{Accepted: keeper.Accepted{Module: module, Digest: b.Entries[module].ManifestDigest, Revision: uint64(max(b.Revision, 0)),
|
||||||
Foundation: a.Foundation, OldBytes: a.OldBytes}
|
Old: a.Accepted.Old}, Foundation: a.Foundation, OldBytes: a.OldBytes}
|
||||||
}
|
}
|
||||||
cm.EmitsOwn = cm.EmitsOwn || a.EmitsOwn
|
cm.EmitsOwn = cm.EmitsOwn || a.EmitsOwn
|
||||||
for _, c := range a.Claims {
|
for _, c := range a.Claims {
|
||||||
|
|||||||
@@ -1,9 +1,14 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||||
|
|
||||||
|
"nats-tools/internal/keeper"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAsksOf(t *testing.T) {
|
func TestAsksOf(t *testing.T) {
|
||||||
@@ -22,6 +27,42 @@ func TestAsksOf(t *testing.T) {
|
|||||||
if b.Accepted.Digest != a.Accepted.Digest {
|
if b.Accepted.Digest != a.Accepted.Digest {
|
||||||
t.Errorf("the digest is of the bus-state asked alone: %s vs %s", b.Accepted.Digest, a.Accepted.Digest)
|
t.Errorf("the digest is of the bus-state asked alone: %s vs %s", b.Accepted.Digest, a.Accepted.Digest)
|
||||||
}
|
}
|
||||||
|
// Design 47 H H-A-229, H-A-230: the old sections of the artifact registry and the forge, as their manifests declare
|
||||||
|
// them, are asked for under the bus names the grants part gives them, the names the live orphans of 2026-11-21 hold;
|
||||||
|
// a manifest not accepted asks none.
|
||||||
|
for _, row := range []struct{ module, sections, want string }{
|
||||||
|
{"distribution", `{"state": ["transfers", {"name": "idle", "key-ttl": true}, "holds", "windows", "mirrors", "findings"],
|
||||||
|
"streams": [{"name": "distribution-events", "events": ["old-name-closed"], "max-age-days": 7, "max-mib": 16}],
|
||||||
|
"objects": [{"name": "incoming", "max-mib": 8192}]}`,
|
||||||
|
"[KV_distribution_transfers KV_distribution_idle KV_distribution_holds KV_distribution_windows KV_distribution_mirrors KV_distribution_findings OBJ_distribution_incoming DISTRIBUTION_EVENTS]"},
|
||||||
|
{"gitea", `{"state": ["bound", {"name": "idle", "key-ttl": true}, "transfers"], "objects": [{"name": "incoming", "max-mib": 2048}],
|
||||||
|
"logs": [{"name": "calls", "max-mib": 8}]}`,
|
||||||
|
"[KV_gitea_bound KV_gitea_idle KV_gitea_transfers OBJ_gitea_incoming LOG_gitea_calls]"},
|
||||||
|
} {
|
||||||
|
var m map[string]any
|
||||||
|
if err := json.Unmarshal([]byte(row.sections), &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m["module"] = row.module
|
||||||
|
a, err := AsksOf(catalog.CatalogEntry{Module: row.module, Verdict: "accepted", Manifest: m})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, p := range a.Accepted.Old {
|
||||||
|
names = append(names, p.Objects[0].Stream())
|
||||||
|
}
|
||||||
|
if fmt.Sprint(names) != row.want {
|
||||||
|
t.Errorf("%s asks %v of its old sections, want %s", row.module, names, row.want)
|
||||||
|
}
|
||||||
|
i := slices.IndexFunc(a.Accepted.Old, func(p keeper.Plan) bool { return p.Local == "incoming" })
|
||||||
|
if i < 0 || a.Accepted.Old[i].Objects[0].Settings.MaxBytes != int64(m["objects"].([]any)[0].(map[string]any)["max-mib"].(float64))*keeper.MiB {
|
||||||
|
t.Errorf("%s's incoming is not capped as declared: %+v", row.module, a.Accepted.Old)
|
||||||
|
}
|
||||||
|
if n, _ := AsksOf(catalog.CatalogEntry{Module: row.module, Verdict: "refused", Manifest: m}); len(n.Accepted.Old) != 0 {
|
||||||
|
t.Errorf("a refused manifest of %s asks %v", row.module, n.Accepted.Old)
|
||||||
|
}
|
||||||
|
}
|
||||||
if _, err := AsksOf(catalog.CatalogEntry{}); err == nil {
|
if _, err := AsksOf(catalog.CatalogEntry{}); err == nil {
|
||||||
t.Error("an entry with no module was read")
|
t.Error("an entry with no module was read")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
// newest revision lists, each local name from the first of its watched branches in its registration's order whose entry
|
// newest revision lists, each local name from the first of its watched branches in its registration's order whose entry
|
||||||
// asks it, another branch asking it with other settings a drift naming that branch; a branch the registration does not
|
// asks it, another branch asking it with other settings a drift naming that branch; a branch the registration does not
|
||||||
// name after; a verdict not accepted asks nothing; the claims of every branch; a foundation part and its old sections'
|
// name after; a verdict not accepted asks nothing; the claims of every branch; a foundation part and its old sections'
|
||||||
// caps from the first branch.
|
// caps and entries from the first branch (design 47 H H-A-229).
|
||||||
func TestTheCatalogHalfIsReadPerWatchedBranch(t *testing.T) {
|
func TestTheCatalogHalfIsReadPerWatchedBranch(t *testing.T) {
|
||||||
entryOf := func(verdict string, manifest map[string]any) catalog.CatalogEntry {
|
entryOf := func(verdict string, manifest map[string]any) catalog.CatalogEntry {
|
||||||
manifest["module"] = "m"
|
manifest["module"] = "m"
|
||||||
@@ -44,17 +44,17 @@ func TestTheCatalogHalfIsReadPerWatchedBranch(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{"one branch", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}}, []string{"main"},
|
{"one branch", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}}, []string{"main"},
|
||||||
"m 3 [drafts/key-value/state] [a] false 0;"},
|
"m 3 [drafts/key-value/state] [a] false 0 0;"},
|
||||||
{"a feature branch asking a local name with other settings drifts, its new one planned",
|
{"a feature branch asking a local name with other settings drifts, its new one planned",
|
||||||
[]Branch{{Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}, {Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}},
|
[]Branch{{Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}, {Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}},
|
||||||
[]string{"main", "feature"}, "m 3 [drafts/key-value/state log/stream/log] [a b] false 0; drifted m drafts feature from main"},
|
[]string{"main", "feature"}, "m 3 [drafts/key-value/state log/stream/log] [a b] false 0 0; drifted m drafts feature from main"},
|
||||||
{"the registration's order decides", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}, {Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}},
|
{"the registration's order decides", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}, {Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}},
|
||||||
[]string{"feature"}, "m 9 [drafts/key-value/state log/stream/log] [b a] false 0; drifted m drafts main from feature"},
|
[]string{"feature"}, "m 9 [drafts/key-value/state log/stream/log] [b a] false 0 0; drifted m drafts main from feature"},
|
||||||
{"a verdict not accepted asks nothing", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("refused", map[string]any{"bus-state": []any{state("")}})}}}, []string{"main"},
|
{"a verdict not accepted asks nothing", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("refused", map[string]any{"bus-state": []any{state("")}})}}}, []string{"main"},
|
||||||
"m 3 [] [] false 0;"},
|
"m 3 [] [] false 0 0;"},
|
||||||
{"a foundation part, its old sections counted", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("accepted", map[string]any{
|
{"a foundation part, its old sections counted", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("accepted", map[string]any{
|
||||||
"objects": []any{map[string]any{"name": "sent-objects", "max-mib": 256}}, "logs": []any{map[string]any{"name": "l", "max-mib": 4}}, "streams": []any{map[string]any{"name": "s"}}})}}}, []string{"main"},
|
"objects": []any{map[string]any{"name": "sent-objects", "max-mib": 256}}, "logs": []any{map[string]any{"name": "l", "max-mib": 4}}, "streams": []any{map[string]any{"name": "s"}}})}}}, []string{"main"},
|
||||||
"m 3 [] [] true 1284;"},
|
"m 3 [] [] true 1284 3;"},
|
||||||
} {
|
} {
|
||||||
t.Run(row.name, func(t *testing.T) {
|
t.Run(row.name, func(t *testing.T) {
|
||||||
v, err := ViewOf(row.branches, func(string) []string { return row.order })
|
v, err := ViewOf(row.branches, func(string) []string { return row.order })
|
||||||
@@ -67,7 +67,7 @@ func TestTheCatalogHalfIsReadPerWatchedBranch(t *testing.T) {
|
|||||||
for _, e := range m.BusState {
|
for _, e := range m.BusState {
|
||||||
es = append(es, e.Name+"/"+e.Storage+"/"+e.Template)
|
es = append(es, e.Name+"/"+e.Storage+"/"+e.Template)
|
||||||
}
|
}
|
||||||
got += fmt.Sprintf("%s %d [%s] %v %v %d;", m.Module, m.Revision, strings.Join(es, " "), m.Claims, m.Foundation, m.OldBytes/keeper.MiB)
|
got += fmt.Sprintf("%s %d [%s] %v %v %d %d;", m.Module, m.Revision, strings.Join(es, " "), m.Claims, m.Foundation, m.OldBytes/keeper.MiB, len(m.Old))
|
||||||
}
|
}
|
||||||
for _, d := range v.Drifts {
|
for _, d := range v.Drifts {
|
||||||
got += fmt.Sprintf(" drifted %s %s %s from %s", d.Module, d.Local, d.Branch, d.From)
|
got += fmt.Sprintf(" drifted %s %s %s from %s", d.Module, d.Local, d.Branch, d.From)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -84,11 +85,14 @@ type EventFill struct {
|
|||||||
|
|
||||||
// Accepted is one module's entry as the catalog half's watched branches give it (design 47 G §344, §433, §532): the
|
// Accepted is one module's entry as the catalog half's watched branches give it (design 47 G §344, §433, §532): the
|
||||||
// module, its manifest's digest, the catalog revision of the branch it was read from, and its bus-state entries whole.
|
// module, its manifest's digest, the catalog revision of the branch it was read from, and its bus-state entries whole.
|
||||||
|
// Old are the entries its old sections declare, as OldPlansOf names and caps them (design 47 H H-A-229, H-A-230): asked
|
||||||
|
// for too, but never made by the keeper.
|
||||||
type Accepted struct {
|
type Accepted struct {
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
Digest string `json:"digest"`
|
Digest string `json:"digest"`
|
||||||
Revision uint64 `json:"revision"`
|
Revision uint64 `json:"revision"`
|
||||||
BusState []catalogue.BusStateEntry `json:"bus-state"`
|
BusState []catalogue.BusStateEntry `json:"bus-state"`
|
||||||
|
Old []Plan `json:"old-sections,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CatalogModule is one module of the catalog half for the bus module (design 47 G §344, §532, §533, §547): what its
|
// CatalogModule is one module of the catalog half for the bus module (design 47 G §344, §532, §533, §547): what its
|
||||||
@@ -146,7 +150,9 @@ type Asked struct {
|
|||||||
Digest string `json:"digest,omitempty"`
|
Digest string `json:"digest,omitempty"`
|
||||||
Revision uint64 `json:"revision"`
|
Revision uint64 `json:"revision"`
|
||||||
Entries []catalogue.BusStateEntry `json:"entries"`
|
Entries []catalogue.BusStateEntry `json:"entries"`
|
||||||
Made map[string]Made `json:"made,omitempty"`
|
// Old are the entries its old sections declare, as last planned (Accepted's Old; design 47 H H-A-229, H-A-230).
|
||||||
|
Old []Plan `json:"old-sections,omitempty"`
|
||||||
|
Made map[string]Made `json:"made,omitempty"`
|
||||||
// Drifts are the local names a later watched branch asks with other settings, as last said (design 47 G §532).
|
// Drifts are the local names a later watched branch asks with other settings, as last said (design 47 G §532).
|
||||||
Drifts map[string]Drift `json:"drifts,omitempty"`
|
Drifts map[string]Drift `json:"drifts,omitempty"`
|
||||||
At time.Time `json:"at"`
|
At time.Time `json:"at"`
|
||||||
@@ -259,7 +265,8 @@ type Limits struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CapSum is the sum, in bytes, of the caps of every object the entries made hold, but for the entry of skipModule and
|
// CapSum is the sum, in bytes, of the caps of every object the entries made hold, but for the entry of skipModule and
|
||||||
// skipLocal (design 47 G §547): what the bus's limits are read against. An entry never made, or refused, holds none.
|
// skipLocal (design 47 G §547): what the bus's limits are read against. An entry never made, or refused, holds none, and
|
||||||
|
// one an old objects, log or stream section keeps is counted once, in the old sections' caps (design 47 H H-A-229).
|
||||||
func CapSum(held map[string]Asked, skipModule, skipLocal string) int64 {
|
func CapSum(held map[string]Asked, skipModule, skipLocal string) int64 {
|
||||||
var sum int64
|
var sum int64
|
||||||
for module, a := range held {
|
for module, a := range held {
|
||||||
@@ -267,6 +274,11 @@ func CapSum(held map[string]Asked, skipModule, skipLocal string) int64 {
|
|||||||
if (module == skipModule && local == skipLocal) || !(m.Created || m.Orphaned) {
|
if (module == skipModule && local == skipLocal) || !(m.Created || m.Orphaned) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// A kept entry of an objects, log or stream section is in the old sections' caps already; a state's is not
|
||||||
|
// (design 47 G §547, H H-A-229).
|
||||||
|
if p, kept := oldKept(a.Old, local, m); kept && p.Storage != catalogue.StorageKeyValue {
|
||||||
|
continue
|
||||||
|
}
|
||||||
for _, o := range m.Plan.Objects {
|
for _, o := range m.Plan.Objects {
|
||||||
sum += o.Settings.MaxBytes
|
sum += o.Settings.MaxBytes
|
||||||
}
|
}
|
||||||
@@ -596,6 +608,15 @@ func (k *Keeper) Heard(a Accepted) {
|
|||||||
held.Made = map[string]Made{}
|
held.Made = map[string]Made{}
|
||||||
}
|
}
|
||||||
digest := EntriesDigest(a.BusState)
|
digest := EntriesDigest(a.BusState)
|
||||||
|
if len(a.Old) > 0 {
|
||||||
|
// The old sections' entries are planned too (design 47 H H-A-229): a module that declares none keeps its digest.
|
||||||
|
raw, _ := json.Marshal(struct {
|
||||||
|
BusState []catalogue.BusStateEntry `json:"bus-state"`
|
||||||
|
Old []Plan `json:"old-sections"`
|
||||||
|
}{a.BusState, a.Old})
|
||||||
|
sum := sha256.Sum256(raw)
|
||||||
|
digest = "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
if p, ok := k.planned(a.Module); ok && found && p.Digest == digest {
|
if p, ok := k.planned(a.Module); ok && found && p.Digest == digest {
|
||||||
if held.Digest != a.Digest || held.Revision != a.Revision {
|
if held.Digest != a.Digest || held.Revision != a.Revision {
|
||||||
held.Digest, held.Revision, held.At = a.Digest, a.Revision, k.now()
|
held.Digest, held.Revision, held.At = a.Digest, a.Revision, k.now()
|
||||||
@@ -603,13 +624,18 @@ func (k *Keeper) Heard(a Accepted) {
|
|||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
held.Digest, held.Entries, held.Revision, held.At = a.Digest, a.BusState, a.Revision, k.now()
|
held.Digest, held.Entries, held.Old, held.Revision, held.At = a.Digest, a.BusState, a.Old, a.Revision, k.now()
|
||||||
defer k.writePlan(a.Module, digest, held.Revision, a.BusState)
|
defer k.writePlan(a.Module, digest, held.Revision, a.BusState)
|
||||||
asked := map[string]bool{}
|
asked := map[string]bool{}
|
||||||
for _, e := range a.BusState {
|
for _, e := range a.BusState {
|
||||||
asked[e.Name] = true
|
asked[e.Name] = true
|
||||||
k.makeEntry(&held, e)
|
k.makeEntry(&held, e)
|
||||||
}
|
}
|
||||||
|
for _, local := range sortedKeys(held.Made) {
|
||||||
|
if !asked[local] && k.keepOld(&held, local) {
|
||||||
|
asked[local] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, local := range sortedKeys(held.Made) {
|
for _, local := range sortedKeys(held.Made) {
|
||||||
m := held.Made[local]
|
m := held.Made[local]
|
||||||
if asked[local] || m.Orphaned || local == SentLocal {
|
if asked[local] || m.Orphaned || local == SentLocal {
|
||||||
@@ -650,6 +676,35 @@ func (k *Keeper) Sends(module string) {
|
|||||||
k.write(held)
|
k.write(held)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// keepOld keeps an entry the keeper made that an old section of the module declares under the same bus name (design 47
|
||||||
|
// H H-A-229, H-A-230, H-A-231): never an orphan (one that was is asked for again, its condition cleared), compared with
|
||||||
|
// the bus and grown to the declared cap, and rebuilt on request; the keeper creates none of its objects the bus lacks and
|
||||||
|
// says no created for it, as the grants part makes and binds them. It answers whether an old section keeps it. An entry
|
||||||
|
// never made is none of the keeper's: every other old-section entry is the grants part's alone.
|
||||||
|
func (k *Keeper) keepOld(held *Asked, local string) bool {
|
||||||
|
m := held.Made[local]
|
||||||
|
p, ok := oldKept(held.Old, local, m)
|
||||||
|
if !ok || !m.Created || local == SentLocal {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if m.Orphaned {
|
||||||
|
m.Orphaned = false
|
||||||
|
k.clear(conditionOrphaned, held.Module, local)
|
||||||
|
}
|
||||||
|
// Only the objects the keeper made are its own: one the old plan names beside them (a state's `<state>-objects` the
|
||||||
|
// grants part made) is never compared, grown or rebuilt by the keeper (H-A-231).
|
||||||
|
var objects []Object
|
||||||
|
for _, o := range p.Objects {
|
||||||
|
if slices.ContainsFunc(m.Plan.Objects, func(made Object) bool { return made.Stream() == o.Stream() }) {
|
||||||
|
objects = append(objects, o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.Plan.Objects = objects
|
||||||
|
k.ensureMade(held, &m, false)
|
||||||
|
held.Made[local] = m
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// makeEntry makes one asked entry where the bus lacks it, compares it where it holds it, and says what it did.
|
// makeEntry makes one asked entry where the bus lacks it, compares it where it holds it, and says what it did.
|
||||||
func (k *Keeper) makeEntry(held *Asked, e catalogue.BusStateEntry) {
|
func (k *Keeper) makeEntry(held *Asked, e catalogue.BusStateEntry) {
|
||||||
module := held.Module
|
module := held.Module
|
||||||
@@ -679,7 +734,11 @@ func (k *Keeper) makeEntry(held *Asked, e catalogue.BusStateEntry) {
|
|||||||
|
|
||||||
// ensure brings one planned entry onto the bus: each object the bus lacks created, a held one compared. A difference
|
// ensure brings one planned entry onto the bus: each object the bus lacks created, a held one compared. A difference
|
||||||
// that only grows is applied; any other is a drift, said and never fixed quietly.
|
// that only grows is applied; any other is a drift, said and never fixed quietly.
|
||||||
func (k *Keeper) ensure(held *Asked, m *Made) {
|
func (k *Keeper) ensure(held *Asked, m *Made) { k.ensureMade(held, m, true) }
|
||||||
|
|
||||||
|
// ensureMade is ensure; with create false an object the bus lacks is left to the part that makes it (an old section's,
|
||||||
|
// the grants part: design 47 H H-A-230) and nothing is created.
|
||||||
|
func (k *Keeper) ensureMade(held *Asked, m *Made, create bool) {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
asked, holds := map[string]Settings{}, map[string]Settings{}
|
asked, holds := map[string]Settings{}, map[string]Settings{}
|
||||||
@@ -694,6 +753,9 @@ func (k *Keeper) ensure(held *Asked, m *Made) {
|
|||||||
m.Waiting = fmt.Sprintf("%s cannot be read: %v", o.Stream(), err)
|
m.Waiting = fmt.Sprintf("%s cannot be read: %v", o.Stream(), err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if !found && !create {
|
||||||
|
continue
|
||||||
|
}
|
||||||
if !found {
|
if !found {
|
||||||
if err := k.capRoom(ctx, held, m); err != nil {
|
if err := k.capRoom(ctx, held, m); err != nil {
|
||||||
if errors.Is(err, ErrRetry) {
|
if errors.Is(err, ErrRetry) {
|
||||||
@@ -794,11 +856,16 @@ func (k *Keeper) CheckAll() []CheckLine {
|
|||||||
out = append(out, lineOf(module, local, m))
|
out = append(out, lineOf(module, local, m))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
asked := false
|
||||||
for _, e := range held.Entries {
|
for _, e := range held.Entries {
|
||||||
if e.Name == local {
|
if e.Name == local {
|
||||||
|
asked = true
|
||||||
k.makeEntry(&held, e)
|
k.makeEntry(&held, e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !asked {
|
||||||
|
k.keepOld(&held, local)
|
||||||
|
}
|
||||||
out = append(out, lineOf(module, local, held.Made[local]))
|
out = append(out, lineOf(module, local, held.Made[local]))
|
||||||
}
|
}
|
||||||
k.write(held)
|
k.write(held)
|
||||||
|
|||||||
@@ -1112,6 +1112,90 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
|
|||||||
t.Errorf("the hand acts: %v", acts)
|
t.Errorf("the hand acts: %v", acts)
|
||||||
}
|
}
|
||||||
}},
|
}},
|
||||||
|
// Design 47 H H-A-229, H-A-230, H-A-231: an entry the keeper made, now declared in the module's old sections under
|
||||||
|
// the same bus name, is asked for: its orphan cleared, grown to the declared cap, rebuilt on request; the keeper
|
||||||
|
// creates nothing an old section declares and says no created for it.
|
||||||
|
{"an entry an old section declares is no orphan and is rebuilt, nothing of the old sections made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||||
|
store := k.Store.(*memStore)
|
||||||
|
incoming := entry("incoming", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "8"})
|
||||||
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, incoming}})
|
||||||
|
k.Heard(Accepted{Module: "m", Revision: 2})
|
||||||
|
if a := k.Held()["m"]; !a.Made["incoming"].Orphaned || !a.Made["drafts"].Orphaned {
|
||||||
|
t.Fatalf("held %+v: both orphans once no section asks for them", a.Made)
|
||||||
|
}
|
||||||
|
delete(bus.held, "OBJ_m_drafts-objects")
|
||||||
|
made := fmt.Sprint(bus.created)
|
||||||
|
out.reset()
|
||||||
|
old := OldPlansOf(catalogue.Manifest{Module: "m", State: []catalogue.StateDeclaration{{Name: "drafts"}, {Name: "bound"}},
|
||||||
|
Objects: []catalogue.ObjectsDeclaration{{Name: "incoming", MaxMiB: 16}}})
|
||||||
|
k.Heard(Accepted{Module: "m", Revision: 3, Old: old})
|
||||||
|
a := k.Held()["m"]
|
||||||
|
if a.Made["incoming"].Orphaned || a.Made["drafts"].Orphaned || len(a.Made) != 2 {
|
||||||
|
t.Errorf("held %+v: incoming and drafts asked for by the old sections, bound none of the keeper's", a.Made)
|
||||||
|
}
|
||||||
|
if fmt.Sprint(bus.created) != made || len(out.events) != 0 || bus.held["OBJ_m_incoming"].MaxBytes != 16*MiB {
|
||||||
|
t.Errorf("created %v (was %s), said %v, incoming held %+v: nothing made or said, grown to the declared 16 MiB",
|
||||||
|
bus.created, made, out.events, bus.held["OBJ_m_incoming"])
|
||||||
|
}
|
||||||
|
if fmt.Sprint(out.conds) != "[cleared nats.orphaned.m_drafts cleared nats.orphaned.m_incoming]" {
|
||||||
|
t.Errorf("conditions %v: both orphans cleared", out.conds)
|
||||||
|
}
|
||||||
|
k.CheckAll()
|
||||||
|
if fmt.Sprint(bus.created) != made {
|
||||||
|
t.Errorf("the check created %v (was %s): the grants part makes an old section's objects", bus.created, made)
|
||||||
|
}
|
||||||
|
binds, _ := occ.ActDigest(occ.Act{"verb": "recreate", "module": "m", "local": "incoming"})
|
||||||
|
ask := oc.Ask{ID: "bus-9", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour)), Options: []oc.Option{{ID: "go", Label: "Rebuild", Level: occ.Destroy, Binds: binds}}}
|
||||||
|
w := &oc.Warrant{Ask: "bus-9", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator},
|
||||||
|
Option: "go", Label: "Rebuild", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)}
|
||||||
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil }
|
||||||
|
out.reset()
|
||||||
|
raw, _ := json.Marshal(Request{ID: "r9", Verb: VerbRecreate, Module: "m", Local: "incoming", Ask: "bus-9"})
|
||||||
|
_, _ = store.Put(StateRequests, "r9", raw)
|
||||||
|
k.ActOnRequests()
|
||||||
|
raw, _, _, _ = store.Get(StateRequests, "r9")
|
||||||
|
var got Request
|
||||||
|
_ = json.Unmarshal(raw, &got)
|
||||||
|
if got.Error != "" || fmt.Sprint(out.events) != "[recreated m incoming]" || bus.held["OBJ_m_incoming"].MaxBytes != 16*MiB {
|
||||||
|
t.Errorf("request %+v, said %v, held %+v: rebuilt at the declared cap", got, out.events, bus.held["OBJ_m_incoming"])
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
// Design 47 H H-A-231, G §547: a state the keeper made as its bucket alone keeps only that bucket; the store beside it
|
||||||
|
// the grants part made is neither compared, grown nor rebuilt by the keeper; a kept state's cap stays in the sum, a
|
||||||
|
// kept objects section's is counted in the old sections' caps alone.
|
||||||
|
{"a kept state's store made by the grants part is left alone, its bucket's cap summed", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||||
|
store := k.Store.(*memStore)
|
||||||
|
incoming := entry("incoming", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "8"})
|
||||||
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{incoming}})
|
||||||
|
old := OldPlansOf(catalogue.Manifest{Module: "m", State: []catalogue.StateDeclaration{{Name: "findings"}},
|
||||||
|
Objects: []catalogue.ObjectsDeclaration{{Name: "incoming", MaxMiB: 8}}})
|
||||||
|
bucket := old[0].Objects[0]
|
||||||
|
bus.held[bucket.Stream()] = bucket.Settings
|
||||||
|
bus.held["OBJ_m_findings-objects"] = Settings{MaxBytes: 8 * MiB, Discard: DiscardNew}
|
||||||
|
held := k.Held()["m"]
|
||||||
|
held.Made["findings"] = Made{Plan: Plan{Module: "m", Local: "findings", Storage: catalogue.StorageKeyValue, Template: catalogue.TemplateState,
|
||||||
|
Objects: []Object{bucket}}, Created: true, Orphaned: true}
|
||||||
|
k.write(held)
|
||||||
|
k.Heard(Accepted{Module: "m", Revision: 2, Old: old})
|
||||||
|
k.CheckAll()
|
||||||
|
if m := k.Held()["m"].Made["findings"]; m.Orphaned || len(m.Plan.Objects) != 1 || len(bus.updated) != 0 || m.Drift != "" {
|
||||||
|
t.Errorf("findings %+v, updated %v: kept, its bucket alone, the grants part's store not grown", m, bus.updated)
|
||||||
|
}
|
||||||
|
if sum := CapSum(k.Held(), "", ""); sum != 64*MiB {
|
||||||
|
t.Errorf("the sum is %d MiB, want 64: the kept state's bucket, not the kept objects section's store", sum/MiB)
|
||||||
|
}
|
||||||
|
binds, _ := occ.ActDigest(occ.Act{"verb": "recreate", "module": "m", "local": "findings"})
|
||||||
|
ask := oc.Ask{ID: "bus-8", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour)), Options: []oc.Option{{ID: "go", Label: "Rebuild", Level: occ.Destroy, Binds: binds}}}
|
||||||
|
w := &oc.Warrant{Ask: "bus-8", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator},
|
||||||
|
Option: "go", Label: "Rebuild", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)}
|
||||||
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil }
|
||||||
|
raw, _ := json.Marshal(Request{ID: "r8", Verb: VerbRecreate, Module: "m", Local: "findings", Ask: "bus-8"})
|
||||||
|
_, _ = store.Put(StateRequests, "r8", raw)
|
||||||
|
k.ActOnRequests()
|
||||||
|
if fmt.Sprint(bus.deleted) != "[KV_m_findings]" || bus.held["OBJ_m_findings-objects"].MaxBytes != 8*MiB {
|
||||||
|
t.Errorf("deleted %v, store held %+v: the recreate rebuilds the bucket alone", bus.deleted, bus.held["OBJ_m_findings-objects"])
|
||||||
|
}
|
||||||
|
}},
|
||||||
{"a recreate rebuilds a drifted entry after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
{"a recreate rebuilds a drifted entry after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||||
store := k.Store.(*memStore)
|
store := k.Store.(*memStore)
|
||||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
||||||
|
|||||||
@@ -423,3 +423,70 @@ func Differs(asked, held Settings) string {
|
|||||||
add("key-ttl", asked.KeyTTL, held.KeyTTL)
|
add("key-ttl", asked.KeyTTL, held.KeyTTL)
|
||||||
return strings.Join(out, "; ")
|
return strings.Join(out, "; ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OldPlansOf are the entries a manifest declares in its old sections (state, objects, logs, streams), each named on the
|
||||||
|
// bus and capped as the grants part composes it, by the SDK's one naming per section (design 47 H H-A-229, H-A-230): a
|
||||||
|
// state `<module>_<state>` with its object store `<state>-objects` beside it (none for a state only read, or one whose
|
||||||
|
// store's name is one of the module's own object buckets), an objects section `<module>_<name>`, a log
|
||||||
|
// `LOG_<module>_<name>` and a stream its local name in capitals. They are asked for: an entry the keeper made under one
|
||||||
|
// of their bus names is never an orphan, is kept, compared and grown to the declared cap, and rebuilt on request; the
|
||||||
|
// keeper creates none of them and says no created for one (the grants part makes and binds them). Pure.
|
||||||
|
func OldPlansOf(m catalogue.Manifest) []Plan {
|
||||||
|
module := m.Module
|
||||||
|
var out []Plan
|
||||||
|
own := map[string]bool{}
|
||||||
|
for _, o := range m.Objects {
|
||||||
|
own[o.Name] = true
|
||||||
|
}
|
||||||
|
for _, s := range m.State {
|
||||||
|
capMiB := int64(s.MaxMiB)
|
||||||
|
if capMiB <= 0 {
|
||||||
|
capMiB = StateCap / MiB
|
||||||
|
}
|
||||||
|
p := Plan{Module: module, Local: s.Name, Storage: catalogue.StorageKeyValue, Template: catalogue.TemplateState,
|
||||||
|
Objects: []Object{{Kind: KindBucket, Name: buscompose.BucketName(module, s.Name),
|
||||||
|
Settings: Settings{MaxBytes: capMiB * MiB, MaxAgeSeconds: int64(s.TTLSeconds), History: int64(max(s.History, 1)),
|
||||||
|
Discard: DiscardNew, MaxMsgSize: ValueMost, KeyTTL: s.KeyTTL}}}}
|
||||||
|
if objects := catalogue.StateObjectsName(s.Name); !s.ReadOnly && !own[objects] {
|
||||||
|
objCap := int64(s.MaxMiB)
|
||||||
|
if objCap <= 0 {
|
||||||
|
objCap = StateObjectsCapMiB
|
||||||
|
}
|
||||||
|
objCap = min(max(objCap, catalogue.ObjectsLeastMiB), catalogue.ObjectsMostMiB)
|
||||||
|
p.Objects = append(p.Objects, Object{Kind: KindObjects, Name: catalogue.ObjectsBucket(module, objects),
|
||||||
|
Settings: Settings{MaxBytes: objCap * MiB, Discard: DiscardNew}})
|
||||||
|
}
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
for _, o := range m.Objects {
|
||||||
|
out = append(out, Plan{Module: module, Local: o.Name, Storage: catalogue.StorageObjectStore, Template: catalogue.TemplateObjects,
|
||||||
|
Objects: []Object{{Kind: KindObjects, Name: o.Bucket(module), Settings: Settings{MaxBytes: int64(o.MaxMiB) * MiB, Discard: DiscardNew}}}})
|
||||||
|
}
|
||||||
|
for _, l := range m.Logs {
|
||||||
|
out = append(out, Plan{Module: module, Local: l.Name, Storage: catalogue.StorageStream, Template: catalogue.TemplateLog,
|
||||||
|
Objects: []Object{{Kind: KindStream, Name: buscompose.LogStreamName(module, l.Name), Subjects: []string{buscompose.LogSubject(module, l.Name) + ".>"},
|
||||||
|
Settings: Settings{MaxBytes: int64(l.Cap()) * MiB, MaxAgeSeconds: days(int64(l.MaxAgeDays)), Discard: DiscardNew, MaxMsgSize: buscompose.LogMaxMessageBytes}}}})
|
||||||
|
}
|
||||||
|
for _, s := range m.Streams {
|
||||||
|
discard := DiscardOld
|
||||||
|
if s.KeepsHistory() {
|
||||||
|
discard = DiscardNew
|
||||||
|
}
|
||||||
|
out = append(out, Plan{Module: module, Local: s.Name, Storage: catalogue.StorageStream,
|
||||||
|
Objects: []Object{{Kind: KindStream, Name: s.BusName(), Settings: Settings{MaxBytes: int64(s.Cap()) * MiB,
|
||||||
|
MaxAgeSeconds: days(int64(s.Days())), Discard: discard, WorkQueue: s.Retention == catalogue.StreamRetentionWorkQueue}}}})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// oldKept is the old-section plan that keeps the entry made under local, or false: one of olds of the same local name
|
||||||
|
// whose objects are on the bus under the same names as the entry's (design 47 H H-A-229, H-A-230).
|
||||||
|
func oldKept(olds []Plan, local string, m Made) (Plan, bool) {
|
||||||
|
for _, p := range olds {
|
||||||
|
if p.Local != local || len(p.Objects) == 0 || len(m.Plan.Objects) == 0 || p.Objects[0].Stream() != m.Plan.Objects[0].Stream() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return p, true
|
||||||
|
}
|
||||||
|
return Plan{}, false
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user