From 3d87ddcc9a54460f6299cab841f75601c5cee7ca Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 18:15:35 +0100 Subject: [PATCH] =?UTF-8?q?delivery:=20the=20indexer's=20packages=20state?= =?UTF-8?q?=20and=20the=20imports=20a=20build=20reports,=20so=20only=20the?= =?UTF-8?q?=20indexer=20computes=20a=20fingerprint=20over=20what=20a=20mod?= =?UTF-8?q?ule=20imports=20(ADR=200593=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- go/delivery/history_records.go | 11 ++- go/delivery/published_packages.go | 126 +++++++++++++++++++++++++ go/delivery/published_packages_test.go | 87 +++++++++++++++++ go/delivery/source.go | 8 ++ go/delivery/stepasks.go | 4 + 5 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 go/delivery/published_packages.go create mode 100644 go/delivery/published_packages_test.go diff --git a/go/delivery/history_records.go b/go/delivery/history_records.go index 2649a74..1ce6146 100644 --- a/go/delivery/history_records.go +++ b/go/delivery/history_records.go @@ -171,6 +171,10 @@ type BuildLogEntry struct { Manifest string `json:"manifest,omitempty"` Against []string `json:"against,omitempty"` Made []BuiltArtifact `json:"made,omitempty"` + // Imports are, at a build's end, the mesh packages (import paths) the built programs import, their tests left out, + // sorted and each once, as the build module answered them (BuildAnswer.Imports; novox/hq ADR 0593 §2): the indexer + // keeps them on the module (KeptInputs2.Imports) when the build's fingerprint is the module's. + Imports []string `json:"imports,omitempty"` } // BuildsSince is the answer of BuildsSinceTool: every ended build since a time, oldest first, and whether the log held @@ -216,8 +220,11 @@ func EncodeBuildLogEntry(e BuildLogEntry) ([]byte, error) { return nil, fmt.Errorf("delivery: a build log entry names its module, fingerprint, repository, commit, node and time") case e.Event == BuildEndedEvent && e.Outcome == "": return nil, fmt.Errorf("delivery: a build's end names its outcome") - case e.Event == BuildStartedEvent && (e.Outcome != "" || len(e.Made) > 0 || e.Manifest != "" || len(e.Against) > 0): - return nil, fmt.Errorf("delivery: a build's start names no outcome, manifest, base or artifact") + case e.Event == BuildStartedEvent && (e.Outcome != "" || len(e.Made) > 0 || e.Manifest != "" || len(e.Against) > 0 || len(e.Imports) > 0): + return nil, fmt.Errorf("delivery: a build's start names no outcome, manifest, base, artifact or import") + } + if err := CheckImports(e.Imports); err != nil { + return nil, err } return json.Marshal(e) } diff --git a/go/delivery/published_packages.go b/go/delivery/published_packages.go new file mode 100644 index 0000000..a631821 --- /dev/null +++ b/go/delivery/published_packages.go @@ -0,0 +1,126 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "time" +) + +// The published mesh packages (novox/hq ADR 0593 §2): the indexer alone reads each mesh Go module's package digests +// from the forge's listing at its merge and writes, in its own state, the latest published version of the module and +// the content digest of each package in it. The builder, the checker and the build modules read this state; no other +// part computes a package's version or digest. This is the contract only: the key, the value and its checks. How a +// digest is computed is the indexer's own. + +const ( + // PackagesState is the name of the indexer's state of published packages, as its manifest declares it, and + // PackagesBucket its bucket: one key per mesh Go module (PackagesKey). + PackagesState = "packages" + PackagesBucket = IndexerModule + "_" + PackagesState + // PackagesHistory is the revisions the packages state keeps per key: the module's newest publications. + PackagesHistory = 5 +) + +// PackagesKey is the key, in PackagesBucket, of a mesh Go module (its module path, such as +// git.novox.be/novox/mesh-sdk/go): the path as a key part (EscapeName). It refuses a path that is no mesh Go module's +// (PackageRepository). +func PackagesKey(module string) (string, error) { + if _, _, ok := PackageRepository(module); !ok { + return "", fmt.Errorf("delivery: %q is no mesh Go module's path (it starts with %s)", module, ForgeModulePrefix) + } + return EscapeName(module), nil +} + +// PublishedPackages is the value of a packages key: a mesh Go module as its latest trunk merge published it. Module is +// its module path, Repository (`/`) and Dir its repository and its directory there (RootPath for the +// root), Commit the merge commit (the version a build pins, StepAsk.Packages), Requires the digest of its go.mod and +// go.sum together, and Packages the content digest of each package of the module (its files, tests and sub-packages +// excluded), by import path, sorted and each once. Packages is a list, never a map keyed by import path: a state value +// refuses a field name ending in a credential word, and a package such as .../internal/token is one (issue 1930). +type PublishedPackages struct { + V int `json:"v"` + Module string `json:"module"` + Repository string `json:"repository"` + Dir string `json:"dir"` + Commit string `json:"commit"` + Requires string `json:"requires"` + Packages []PackageDigest `json:"packages"` + At time.Time `json:"at"` +} + +// DigestOf is the content digest of the package at import path pkg, and whether p holds it. +func (p PublishedPackages) DigestOf(pkg string) (string, bool) { + i, found := slices.BinarySearchFunc(p.Packages, pkg, func(d PackageDigest, s string) int { return strings.Compare(d.Package, s) }) + if !found { + return "", false + } + return p.Packages[i].Digest, true +} + +func (p PublishedPackages) check() error { + repository, _, ok := PackageRepository(p.Module) + switch { + case !ok: + return fmt.Errorf("delivery: %q is no mesh Go module's path (it starts with %s)", p.Module, ForgeModulePrefix) + case !SameRepository(p.Repository, repository): + return fmt.Errorf("delivery: the module %s is published from %s, and the value names the repository %q", p.Module, repository, p.Repository) + case p.Dir == "": + return fmt.Errorf("delivery: the published module %s names no directory (%q for the repository's root)", p.Module, RootPath) + case !askCommitOK.MatchString(p.Commit): + return fmt.Errorf("delivery: the published module %s names the commit %q, which is no full commit", p.Module, p.Commit) + case p.Requires == "": + return fmt.Errorf("delivery: the published module %s names no digest of its requirements", p.Module) + case p.At.IsZero(): + return fmt.Errorf("delivery: the published module %s names no time", p.Module) + } + for i, d := range p.Packages { + if d.Package != p.Module && !strings.HasPrefix(d.Package, p.Module+"/") { + return fmt.Errorf("delivery: the package %q is not in the module %s", d.Package, p.Module) + } + if d.Digest == "" { + return fmt.Errorf("delivery: the package %s names no digest", d.Package) + } + if i > 0 && p.Packages[i-1].Package >= d.Package { + return fmt.Errorf("delivery: the packages of %s are not sorted and each once (%s after %s)", p.Module, d.Package, p.Packages[i-1].Package) + } + } + return nil +} + +// EncodePublishedPackages is p as JSON at Version; it refuses what a reader could not take (check). +func EncodePublishedPackages(p PublishedPackages) ([]byte, error) { + if err := p.check(); err != nil { + return nil, err + } + p.V = Version + return json.Marshal(p) +} + +// DecodePublishedPackages reads a packages value; it refuses an unknown version (ErrUnknownVersion) and what +// EncodePublishedPackages refuses. +func DecodePublishedPackages(data []byte) (PublishedPackages, error) { + var p PublishedPackages + if err := decode(data, &p); err != nil { + return PublishedPackages{}, err + } + if err := p.check(); err != nil { + return PublishedPackages{}, err + } + return p, nil +} + +// CheckImports refuses an import set (KeptInputs2.Imports, BuildAnswer.Imports, BuildLogEntry.Imports) that holds a +// path that is no mesh package's, or is not sorted and each once. +func CheckImports(imports []string) error { + for i, pkg := range imports { + if _, _, ok := PackageRepository(pkg); !ok || strings.ContainsAny(pkg, " \t\n@") { + return fmt.Errorf("delivery: the import %q is no mesh package's path (it starts with %s)", pkg, ForgeModulePrefix) + } + if i > 0 && imports[i-1] >= pkg { + return fmt.Errorf("delivery: the imports are not sorted and each once (%s after %s)", pkg, imports[i-1]) + } + } + return nil +} diff --git a/go/delivery/published_packages_test.go b/go/delivery/published_packages_test.go new file mode 100644 index 0000000..a8b2240 --- /dev/null +++ b/go/delivery/published_packages_test.go @@ -0,0 +1,87 @@ +package delivery + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" +) + +// The packages state, the build end's imports and the kept imports (ADR 0593 §2) round-trip, and a reader refuses what it +// could not take, naming it. +func TestPublishedPackagesAndImportsRoundTripAndRefuse(t *testing.T) { + commit := strings.Repeat("a", 40) + sdk := "git.novox.be/novox/mesh-sdk/go" + good := PublishedPackages{Module: sdk, Repository: "novox/mesh-sdk", Dir: "go", Commit: commit, Requires: "sha256:01", + Packages: []PackageDigest{{Package: sdk, Digest: "d0"}, {Package: sdk + "/announce", Digest: "d1"}, {Package: sdk + "/internal/token", Digest: "d2"}}, + At: time.Date(2026, 11, 2, 18, 0, 0, 0, time.UTC)} + raw, err := EncodePublishedPackages(good) + if err != nil { + t.Fatal(err) + } + back, err := DecodePublishedPackages(raw) + good.V = Version + if err != nil || !reflect.DeepEqual(back, good) { + t.Fatalf("round trip: %v, %+v", err, back) + } + if d, ok := back.DigestOf(sdk + "/announce"); !ok || d != "d1" { + t.Fatalf("DigestOf announce = %q, %v", d, ok) + } + if _, ok := back.DigestOf(sdk + "/buildstep"); ok { + t.Fatal("DigestOf finds a package the value does not hold") + } + if key, err := PackagesKey(sdk); err != nil || key != "git=2enovox=2ebe/novox/mesh-sdk/go" { + t.Fatalf("PackagesKey = %q, %v", key, err) + } + if _, err := PackagesKey("github.com/x/y"); err == nil { + t.Fatal("PackagesKey takes a path that is no mesh Go module's") + } + for name, mutate := range map[string]func(p *PublishedPackages){ + "no mesh module": func(p *PublishedPackages) { p.Module = "github.com/x/y" }, + "other repository": func(p *PublishedPackages) { p.Repository = "novox/mesh-host" }, + "short commit": func(p *PublishedPackages) { p.Commit = "abc" }, + "no requires": func(p *PublishedPackages) { p.Requires = "" }, + "package elsewhere": func(p *PublishedPackages) { + p.Packages = []PackageDigest{{Package: "git.novox.be/novox/mesh-host/x", Digest: "d"}} + }, + "unsorted": func(p *PublishedPackages) { p.Packages = []PackageDigest{p.Packages[1], p.Packages[0]} }, + "no digest": func(p *PublishedPackages) { p.Packages = []PackageDigest{{Package: sdk}} }, + } { + p := good + p.Packages = slicesClone(good.Packages) + mutate(&p) + if _, err := EncodePublishedPackages(p); err == nil { + t.Errorf("%s: encoded", name) + } + } + + imports := []string{sdk, sdk + "/announce"} + end := BuildLogEntry{Event: BuildEndedEvent, Module: "netcheck", FP: "src2:x", Repository: "novox/mesh-catalog", Commit: commit, + Node: "ace", At: good.At, Outcome: "done", Imports: imports} + raw, err = EncodeBuildLogEntry(end) + if err != nil { + t.Fatal(err) + } + if e, err := DecodeBuildLogEntry(raw); err != nil || !reflect.DeepEqual(e.Imports, imports) { + t.Fatalf("build end round trip: %v, %v", err, e.Imports) + } + start := end + start.Event, start.Outcome = BuildStartedEvent, "" + if _, err := EncodeBuildLogEntry(start); err == nil { + t.Error("a build's start with imports encoded") + } + end.Imports = []string{sdk + "/announce", sdk} + if _, err := EncodeBuildLogEntry(end); err == nil { + t.Error("unsorted imports encoded") + } + var answer BuildAnswer + if err := json.Unmarshal([]byte(`{"artifacts":[],"imports":["`+sdk+`"]}`), &answer); err != nil || len(answer.Imports) != 1 { + t.Fatalf("build answer imports: %v, %v", err, answer.Imports) + } + if err := CheckImports([]string{"fmt"}); err == nil { + t.Error("CheckImports takes a standard package") + } +} + +func slicesClone(d []PackageDigest) []PackageDigest { return append([]PackageDigest(nil), d...) } diff --git a/go/delivery/source.go b/go/delivery/source.go index a627671..d424962 100644 --- a/go/delivery/source.go +++ b/go/delivery/source.go @@ -467,6 +467,11 @@ type KeptInputs2 struct { // fingerprint takes the latest published version of each (Module.Packages). Absent for none, and in a value written // before. Packages []string `json:"packages,omitempty"` + // Imports are the mesh packages (import paths) the module's programs import, their tests left out, as the module's + // last trunk build reported them at its end (BuildLogEntry.Imports; novox/hq ADR 0593 §2), sorted and each once: + // the fingerprint takes each one's content digest (PublishedPackages) instead of Packages' versions. Absent for a + // module with no reported import set, which keeps the version line, and in a value written before. + Imports []string `json:"imports,omitempty"` } // checkPathObjects refuses a named path that is not clean and relative to the module's directory, or names no object. @@ -549,6 +554,9 @@ func (m Module) check() error { } } if m.Inputs2 != nil { + if err := CheckImports(m.Inputs2.Imports); err != nil { + return err + } if err := checkNamed(m.Inputs2.Tree, m.Manifest, m.Inputs2.Paths, "the module's kept inputs"); err != nil { return err } diff --git a/go/delivery/stepasks.go b/go/delivery/stepasks.go index f1d932d..bf8aa49 100644 --- a/go/delivery/stepasks.go +++ b/go/delivery/stepasks.go @@ -555,6 +555,10 @@ func DecodeStepAsk(data []byte) (StepAsk, error) { type BuildAnswer struct { Artifacts []AskBuilt `json:"artifacts"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` + // Imports are the mesh packages (import paths) the built programs import, their tests left out, sorted and each + // once (novox/hq ADR 0593 §2): a Go build module reports them, the builder carries them onto the build's end + // (BuildLogEntry.Imports). Absent from a build module that reports none. + Imports []string `json:"imports,omitempty"` } // AskBuilt is one artifact a build ask built.