A merge rebuilds what it changed, and what packages it

Three faults in one path. A merge rebuilt every module built from the repository, so one change in
a repository holding twenty-six of them meant twenty-six builds. A merge into a repository a module
only *packages* source from rebuilt nothing — two modules are built from the control plane's own
repository and neither had ever been rebuilt when it moved — because the manifest the mesh keeps
carries no build section, so a build now says which repositories it read and the mesh keeps that
beside what it stood on. And a module handed over by hand could record a repository with no
directory inside it, which is a module nothing can ever rebuild (novox/hq 04-ISSUES/131, /132).

A change inside no module's own directory is a change to what they share, and everything built from
that repository is rebuilt: rebuilding too much is the safe direction, because the fault this whole
path exists for is a mesh that believes it is current and is not.

internal/builder/builder.go

@@ -61,6 +61,12 @@ type Result struct {
 	Commit string
 	// Built is each artifact, for reporting.
 	Built []catalogue.Built
+
+	// Read is every repository this build read source from besides the module's own — the second
+	// repository an artifact's recipe names (ArtifactContext). Reported because the manifest the
+	// mesh keeps carries no build section, so nothing else could say that a merge there is a
+	// change to this module (novox/hq 04-ISSUES/131).
+	Read []catalogue.ArtifactContext
 }
 
 // GitCredential is the forge credential a clone may present when the server asks for one.
@@ -220,7 +226,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
 	}
 	say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built))
 	return Result{Manifest: resolved, Commit: commit, Built: built,
-		Against: against(within, manifest, stoodOn)}, nil
+		Against: against(within, manifest, stoodOn), Read: readBy(manifest)}, nil
 }
 
 // Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none,
@@ -1016,3 +1022,31 @@ func instructions(recipe string) []string {
 	flush()
 	return out
 }
+
+// readBy is every repository other than the module's own that this build's recipes read source from,
+// each once and in a fixed order, so two builds of one commit report the same thing the same way.
+func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext {
+	if manifest.Build == nil {
+		return nil
+	}
+	seen := map[string]bool{}
+	var out []catalogue.ArtifactContext
+	for _, a := range manifest.Build.Artifacts {
+		if a.Context == nil || a.Context.Repository == "" {
+			continue
+		}
+		key := a.Context.Repository + "#" + a.Context.Ref
+		if seen[key] {
+			continue
+		}
+		seen[key] = true
+		out = append(out, *a.Context)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Repository != out[j].Repository {
+			return out[i].Repository < out[j].Repository
+		}
+		return out[i].Ref < out[j].Ref
+	})
+	return out
+}

internal/builder/standing_on_test.go

@@ -179,3 +179,24 @@ func TestTheBasesABuildWasHandedAreWhatItStoodOn(t *testing.T) {
 		t.Fatalf("the bases the build was handed were not what it stood on: %v", got)
 	}
 }
+
+// What a build read besides its module's own repository is the second repository its recipes name,
+// each once: a module that packages source living elsewhere is affected when that source moves.
+func TestWhatABuildReadIsTheRepositoriesItsRecipesName(t *testing.T) {
+	elsewhere := catalogue.ArtifactContext{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"}
+	manifest := catalogue.Manifest{
+		Module: "builder",
+		Build: &catalogue.Build{Artifacts: []catalogue.Artifact{
+			{Name: "server", Kind: catalogue.ArtifactImage, From: "Dockerfile", Context: &elsewhere},
+			{Name: "tools", Kind: catalogue.ArtifactImage, From: "Dockerfile", Context: &elsewhere},
+			{Name: "config", Kind: catalogue.ArtifactArchive, From: "etc"},
+		}},
+	}
+	read := readBy(manifest)
+	if len(read) != 1 || read[0] != elsewhere {
+		t.Fatalf("the repositories this build read are %+v", read)
+	}
+	if readBy(catalogue.Manifest{Module: "gitea", Build: &catalogue.Build{}}) != nil {
+		t.Fatal("a module whose recipes name no other repository read one")
+	}
+}

internal/inventory/builds.go

@@ -36,12 +36,21 @@ type Build struct {
 	// Against is every artifact this build stood on, as references rather than module names —
 	// what makes a build edge derived rather than declared (ADR 0009).
 	Against []string
+	// Read is every repository this build read source from besides the module's own (novox/hq
+	// 04-ISSUES/131), at the ref it read.
+	Read []ReadRepository
 	// Failed is the builder's own words, empty when it worked.
 	Failed string
 	Made   []Artifact
 	At     time.Time
 }
 
+// ReadRepository is a repository a build read source from besides the module's own.
+type ReadRepository struct {
+	Repository string `json:"repository"`
+	Ref        string `json:"ref,omitempty"`
+}
+
 // Artifact is one thing a build published.
 type Artifact struct {
 	Name      string `json:"name"`
@@ -66,17 +75,21 @@ func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
 	if err != nil {
 		return err
 	}
+	read, err := json.Marshal(b.Read)
+	if err != nil {
+		return err
+	}
 	var module *string
 	if b.Module != "" {
 		module = &b.Module
 	}
 	_, err = i.store.Pool().Exec(ctx,
 		`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
-		                    source_path, manifest, built_against)
-		 values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+		                    source_path, manifest, built_against, built_contexts)
+		 values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
 		 on conflict (id) do nothing`,
 		b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
-		b.Path, manifestOrNil(b.Manifest), against)
+		b.Path, manifestOrNil(b.Manifest), against, read)
 	return err
 }
 
@@ -199,6 +212,44 @@ func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, erro
 	return against, rows.Err()
 }
 
+// ReadRepositories is what each module's newest successful build read source from besides its own
+// repository, by module name.
+//
+// The mirror of BuiltAgainst, and derived the same way and for the same reason: a merge into a
+// repository a module only packages is a change to that module, and the manifest the mesh keeps
+// carries nothing that would say so (novox/hq 04-ISSUES/131).
+func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select distinct on (module) module, built_contexts
+		   from build
+		  where module is not null and module <> '' and failed = ''
+		  order by module, at desc`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	read := map[string][]ReadRepository{}
+	for rows.Next() {
+		var module string
+		var raw []byte
+		if err := rows.Scan(&module, &raw); err != nil {
+			return nil, err
+		}
+		if len(raw) == 0 {
+			continue
+		}
+		var of []ReadRepository
+		if err := json.Unmarshal(raw, &of); err != nil {
+			continue
+		}
+		if len(of) > 0 {
+			read[module] = of
+		}
+	}
+	return read, rows.Err()
+}
+
 // manifestOrNil keeps the difference between "declared nothing" and "predates this being kept".
 //
 // A build recorded before the mesh kept manifests has no manifest, and that is not the same as one

internal/inventory/builds_test.go

@@ -136,3 +136,36 @@ func ids(builds []Build) []string {
 	}
 	return out
 }
+
+// What a build read besides its module's own repository comes back for the newest build of each
+// module, and only for builds that worked. Nothing recorded is absent rather than empty, which is how
+// a build made before the mesh kept this is told from one that read nothing (novox/hq 04-ISSUES/131).
+func TestWhatABuildReadComesBackForTheNewestBuildOfEachModule(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	older := aBuild("older", "builder", "")
+	older.Read = []ReadRepository{{Repository: "novox/mesh-controller", Ref: "release"}}
+	newer := aBuild("newer", "builder", "")
+	newer.Read = []ReadRepository{{Repository: "novox/mesh-controller", Ref: "main"}}
+	plain := aBuild("plain", "gitea", "")
+	failed := aBuild("failed", "route-proxy", "cannot clone")
+	failed.Read = []ReadRepository{{Repository: "novox/mesh-controller", Ref: "main"}}
+	for _, b := range []Build{older, newer, plain, failed} {
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+	}
+	read, err := inv.ReadRepositories(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(read["builder"]) != 1 || read["builder"][0].Ref != "main" {
+		t.Fatalf("the newest build's reading is %+v", read["builder"])
+	}
+	if _, has := read["gitea"]; has {
+		t.Fatalf("a build that read nothing but its own repository reads as %+v", read["gitea"])
+	}
+	if _, has := read["route-proxy"]; has {
+		t.Fatal("a failed build's reading was kept as what that module reads")
+	}
+}

internal/inventory/migrations/0042-a-build-says-which-repositories-it-read.sql

@@ -0,0 +1,13 @@
+-- A build says which repositories it read, so a merge can find everything it affects.
+--
+-- A module is built from the one repository the mesh records — where its module.json lives — and some
+-- modules' recipes reach into a second for the source they package: the packaging and the source are
+-- allowed to live apart (catalogue's ArtifactContext). That second repository is named in the manifest
+-- the build read, and the manifest the mesh *keeps* carries no build section, so nothing on the mesh
+-- could say that a merge into the other repository is a change to this module at all. Two modules are
+-- built from the control plane's own repository, and neither had ever been rebuilt when it moved
+-- (novox/hq 04-ISSUES/131).
+--
+-- Nullable, like the two derived columns beside it: null is a build recorded before the mesh kept
+-- this, which is not the same as a build that read nothing but its module's own repository.
+alter table build add column built_contexts jsonb;

internal/link/build.go

@@ -88,10 +88,23 @@ type BuildResult struct {
 	// (novox/hq ADR 0009). The catalogue turns these into edges; nothing else need care.
 	Against []string `json:"against,omitempty"`
 
+	// Read is every repository this build read source from besides the module's own. A module whose
+	// recipe packages source that lives elsewhere is affected when that repository moves, and the
+	// manifest the mesh keeps says nothing about it (novox/hq 04-ISSUES/131).
+	Read []ReadRepository `json:"read,omitempty"`
+
 	// Failed is why, when it did.
 	Failed string `json:"failed,omitempty"`
 }
 
+// ReadRepository is a repository a build read source from besides the module's own, at the branch,
+// tag or commit it read. Spelled here as well as in the catalogue and the inventory, for the reason
+// MadeArtifact is: one direction of dependency.
+type ReadRepository struct {
+	Repository string `json:"repository"`
+	Ref        string `json:"ref,omitempty"`
+}
+
 // MadeArtifact is one thing a build produced, as a person would want it reported.
 type MadeArtifact struct {
 	Name      string `json:"name"`

internal/link/events.go

@@ -130,6 +130,16 @@ type SourceMoved struct {
 	HTMLURL  string `json:"html_url"`
 	// MergedAt is when the forge merged it, RFC 3339. What decides whether this is news.
 	MergedAt string `json:"merged_at"`
+
+	// Paths are the files the merge changed, from the repository's root. Empty means the forge said
+	// nothing about them, and every module built from the repository is treated as affected.
+	Paths []string `json:"paths,omitempty"`
+
+	// PathsTruncated says the merge changed more files than the forge was asked to list, so Paths is
+	// a beginning rather than the whole change — and again, everything is treated as affected. Said
+	// rather than inferred from a round number, because "this is all of it" and "this is as much as
+	// I asked for" are the difference between rebuilding a module and leaving it stale.
+	PathsTruncated bool `json:"paths_truncated,omitempty"`
 }
 
 type Upgraded struct {