jochen
887de9b5f2
A what-if named the build seat's holder as packaging the controller's source, "rebuilt without their own source moving", while it was in the plan because an open plan had not built it yet. The what-if and the merge log now say, per module, which changed files of its build source moved it, or that it is read whole and why: no build source recorded, a newer build failed, or a plan has not built it yet.
526 lines
20 KiB
Go
526 lines
20 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"sort"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
// What has been built.
|
|
//
|
|
// A build result was answered to whoever asked and kept nowhere, so "when did this last build",
|
|
// "why did it fail" and "which machine built what is running" had no answer. Failures are recorded
|
|
// too: one that leaves no trace is indistinguishable from a build nobody asked for, and the
|
|
// difference is the whole of whether somebody should be looking at something.
|
|
|
|
// Build is one attempt, whichever way it went.
|
|
type Build struct {
|
|
ID string
|
|
Repository string
|
|
Ref string
|
|
// Module is empty for a build that failed before it knew what it was building.
|
|
Module string
|
|
Commit string
|
|
// On is the machine that did it.
|
|
On string
|
|
// Path is where inside the repository the module lives (novox/hq ADR 0069).
|
|
Path string
|
|
// Manifest is the declaration the builder resolved, as it announced it.
|
|
//
|
|
// **Kept because the catalogue may not have been listening.** The announcement carries this
|
|
// and the catalogue turns it into the module's requires/provides edges. On a fresh mesh the
|
|
// modules built before the catalogue exists are exactly the ones it most needs, so the mesh
|
|
// has to be able to say afterwards what they declared (novox/hq 04-ISSUES/050).
|
|
Manifest []byte
|
|
// Against is every artifact this build stood on, as references rather than module names —
|
|
// what makes a build edge derived rather than declared (ADR 0009).
|
|
Against []string
|
|
// Mirrored is every base this build copied into the artifact store (novox/hq ADR 0257), said
|
|
// whether the build worked or not: the copy is the mesh's, recorded in artifact_mirrored.
|
|
Mirrored []string
|
|
// Read is every repository this build read source from besides the module's own (novox/hq
|
|
// 04-ISSUES/131), at the ref it read.
|
|
Read []ReadRepository
|
|
// Sources are what the build was made from, as files, per repository (novox/hq ADR 0267): said by the
|
|
// builder for a build of a trunk commit; nil where it said none.
|
|
Sources []BuildSource
|
|
// SourceFingerprint is what the build was made from, hashed, as its builder said it (novox/hq
|
|
// issue 280); empty from a builder that predates it, or where the source does not pin the build.
|
|
SourceFingerprint string
|
|
// Failed is the builder's own words, empty when it worked.
|
|
Failed string
|
|
Made []Artifact
|
|
// Asked is when the build was requested, zero when that is not known (an id of another shape,
|
|
// or a build recorded before the mesh kept it). **What orders one build of a module against
|
|
// another** (novox/hq 04-ISSUES/219): builds in flight together finish in any order, and the
|
|
// one asked last stood on the newest bases.
|
|
Asked time.Time
|
|
// At is when the outcome was recorded — when it finished, not when it was asked.
|
|
At time.Time
|
|
}
|
|
|
|
// AskedOrAt is when the build was asked, or when it was recorded when that is not known — the
|
|
// order the mesh had before it kept the request time.
|
|
func (b Build) AskedOrAt() time.Time {
|
|
if !b.Asked.IsZero() {
|
|
return b.Asked
|
|
}
|
|
return b.At
|
|
}
|
|
|
|
// newestRequestFirst is the ordering every "what a module currently is" question uses: the newest
|
|
// request wins, whenever it finished (novox/hq 04-ISSUES/219). A build whose request time is not
|
|
// known is placed at the moment it was recorded, which is the rule that held before.
|
|
const newestRequestFirst = `coalesce(asked, at) desc, at desc`
|
|
|
|
// ReadRepository is a repository a build read source from besides the module's own.
|
|
//
|
|
// Paths and Own are never stored in a build's `built_contexts` (a controller that predates them would read
|
|
// an own entry as a context, and every module of a repository as packaging every other): ReadRepositories
|
|
// lays them over what it reads, from the build's build sources (novox/hq ADR 0267).
|
|
type ReadRepository struct {
|
|
Repository string `json:"repository"`
|
|
Ref string `json:"ref,omitempty"`
|
|
// Paths are the build source the build read in this repository (builder.SourceHolds): a changed file
|
|
// outside them is no change to the module. Empty is the whole repository, as before.
|
|
Paths []string `json:"paths,omitempty"`
|
|
// Own is the module's own repository, whose Paths narrow what its own directory — or, for a module
|
|
// built from its repository's root, the whole repository — would otherwise be.
|
|
Own bool `json:"own,omitempty"`
|
|
// Built is when the build these were read from was asked, and Looked when the module's newest build of
|
|
// any outcome was: what the planner judges a build source's age, and a merge's news, by. Never stored.
|
|
Built time.Time `json:"-"`
|
|
Looked time.Time `json:"-"`
|
|
// LookedAt are the merge commits a plan that built the module, or is building it, answered: a merge
|
|
// of one of them is history for the module whatever the clocks say. Never stored.
|
|
LookedAt []string `json:"-"`
|
|
// Whole is why the module is read whole though a build source may have been recorded — a newer build
|
|
// failed, or a plan has not built it yet — said on an Own entry with no Paths. Never stored.
|
|
Whole string `json:"-"`
|
|
}
|
|
|
|
// BuildSource is the build source a build read in one repository (novox/hq ADR 0267): Repository and Ref
|
|
// a context's, empty for the module's own.
|
|
type BuildSource struct {
|
|
Repository string `json:"repository,omitempty"`
|
|
Ref string `json:"ref,omitempty"`
|
|
Paths []string `json:"paths"`
|
|
}
|
|
|
|
// Artifact is one thing a build published.
|
|
type Artifact struct {
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind"`
|
|
Reference string `json:"reference"`
|
|
}
|
|
|
|
// Worked reports whether this build produced something.
|
|
func (b Build) Worked() bool { return b.Failed == "" }
|
|
|
|
// RecordBuild keeps what a builder said.
|
|
//
|
|
// Idempotent on the correlation id, because a result can arrive twice: once as the answer to
|
|
// whoever asked and once on the exchange when nobody was. Recording both would show one build as
|
|
// two, and which of the two is real is not a question anybody could answer afterwards.
|
|
func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
|
|
made, err := json.Marshal(b.Made)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
against, err := json.Marshal(b.Against)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
read, err := json.Marshal(b.Read)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var sources any
|
|
if len(b.Sources) > 0 {
|
|
raw, err := json.Marshal(b.Sources)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sources = raw
|
|
}
|
|
var module *string
|
|
if b.Module != "" {
|
|
module = &b.Module
|
|
}
|
|
var asked *time.Time
|
|
if !b.Asked.IsZero() {
|
|
asked = &b.Asked
|
|
}
|
|
_, err = i.store.Pool().Exec(ctx,
|
|
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
|
|
source_path, manifest, built_against, built_contexts, asked, source_fingerprint,
|
|
build_sources)
|
|
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
|
|
on conflict (id) do nothing`,
|
|
b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
|
|
b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint, sources)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
|
|
}
|
|
|
|
// RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257):
|
|
// by the build that said so, or by a person who says why. The first record of a copy stands.
|
|
//
|
|
// **A copy a build says it holds is held again.** The builder answers a copy only once the store holds
|
|
// it whole, copying again what was let go of; so a copy the records say was collected, and a build now
|
|
// says it copied, is no longer collected. Left collected, it would stay in the store and out of every
|
|
// sweep for ever.
|
|
func (i *Inventory) RecordMirrored(ctx context.Context, build, why string, references []string) error {
|
|
var by *string
|
|
if build != "" {
|
|
by = &build
|
|
}
|
|
for _, reference := range references {
|
|
if reference == "" {
|
|
continue
|
|
}
|
|
if _, err := i.store.Pool().Exec(ctx,
|
|
`insert into artifact_mirrored (reference, build, why) values ($1, $2, $3)
|
|
on conflict (reference) do nothing`, reference, by, why); err != nil {
|
|
return err
|
|
}
|
|
if build != "" {
|
|
if _, err := i.store.Pool().Exec(ctx,
|
|
`delete from artifact_collected where reference = $1`, reference); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Builds is what has happened lately, newest first.
|
|
//
|
|
// For one module when named, or across the mesh when not. Both are asked: *what happened just
|
|
// now* after something goes wrong, and *what has happened to this* when deciding whether to
|
|
// trust it.
|
|
func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Build, error) {
|
|
if limit <= 0 {
|
|
limit = 20
|
|
}
|
|
query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
|
|
from build order by at desc limit $1`
|
|
args := []any{limit}
|
|
if module != "" {
|
|
query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
|
|
from build where module = $2 order by at desc limit $1`
|
|
args = append(args, module)
|
|
}
|
|
|
|
rows, err := i.store.Pool().Query(ctx, query, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var out []Build
|
|
for rows.Next() {
|
|
var b Build
|
|
var made []byte
|
|
var asked *time.Time
|
|
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
|
|
&b.On, &b.Failed, &made, &asked, &b.At); err != nil {
|
|
return nil, err
|
|
}
|
|
if asked != nil {
|
|
b.Asked = *asked
|
|
}
|
|
if err := json.Unmarshal(made, &b.Made); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, b)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// BuildByID is one build's record, by the id its request carried — what a plan matches its outcome
|
|
// by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the
|
|
// repository, path, ref and commit from. False when no outcome with that id was recorded.
|
|
func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) {
|
|
var b Build
|
|
var made []byte
|
|
var asked *time.Time
|
|
err := i.store.Pool().QueryRow(ctx,
|
|
`select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made,
|
|
coalesce(source_path,''), asked, at
|
|
from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
|
|
&b.On, &b.Failed, &made, &b.Path, &asked, &b.At)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return Build{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return Build{}, false, err
|
|
}
|
|
if asked != nil {
|
|
b.Asked = *asked
|
|
}
|
|
if err := json.Unmarshal(made, &b.Made); err != nil {
|
|
return Build{}, false, err
|
|
}
|
|
return b, true, nil
|
|
}
|
|
|
|
// Held is every artifact this mesh has built, keyed "<module>/<artifact>".
|
|
//
|
|
// **The successful build of each module asked last wins**, which is the same rule the rest of the
|
|
// mesh uses for what a module currently is — asked last, not finished last (novox/hq
|
|
// 04-ISSUES/219): an older request that finishes later stood on older bases. A module rebuilt to something broken and then rebuilt again
|
|
// is at the second one; a module whose last build failed is at the last one that worked, because a
|
|
// failure published nothing and the thing it published before is still what exists.
|
|
//
|
|
// Only successes, and only builds that knew what they were building: a build that failed before it
|
|
// could read a manifest has no module to be the artifact of.
|
|
func (i *Inventory) Held(ctx context.Context) (map[string]string, error) {
|
|
// **A rebuild of an unchanged source holds what the first build of it made** (novox/hq issue 280):
|
|
// the mesh registers that build's artifacts, so a module standing on it is handed the same base
|
|
// and is unchanged too, rather than moving for a digest an image rebuild could not help changing.
|
|
made, err := i.heldMade(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
held := map[string]string{}
|
|
for module, artifacts := range made {
|
|
for _, artifact := range artifacts {
|
|
if artifact.Name == "" || artifact.Reference == "" {
|
|
continue
|
|
}
|
|
held[module+"/"+artifact.Name] = artifact.Reference
|
|
}
|
|
}
|
|
return held, nil
|
|
}
|
|
|
|
// BuiltAgainst is what each module's newest successful build stood on, as recorded — the build
|
|
// edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
|
|
// module standing on nothing looks like: an edge the mesh has not derived is not an edge.
|
|
func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) {
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select distinct on (module) module, built_against
|
|
from build
|
|
where module is not null and module <> '' and failed = ''
|
|
order by module, `+newestRequestFirst)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
against := map[string][]string{}
|
|
for rows.Next() {
|
|
var module string
|
|
var raw []byte
|
|
if err := rows.Scan(&module, &raw); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
continue
|
|
}
|
|
var refs []string
|
|
if err := json.Unmarshal(raw, &refs); err != nil {
|
|
continue
|
|
}
|
|
if len(refs) > 0 {
|
|
against[module] = refs
|
|
}
|
|
}
|
|
return against, rows.Err()
|
|
}
|
|
|
|
// ReadRepositories is what each module's newest successful build read source from besides its own
|
|
// repository, by module name.
|
|
//
|
|
// The mirror of BuiltAgainst, and derived the same way and for the same reason: a merge into a
|
|
// repository a module only packages is a change to that module, and the manifest the mesh keeps
|
|
// carries nothing that would say so (novox/hq 04-ISSUES/131).
|
|
//
|
|
// **With the build source that build said** (novox/hq ADR 0267): a context's entry carries the paths the
|
|
// build read there, and the module's own repository an entry marked Own with its paths. A build that said
|
|
// none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read
|
|
// as before: every file of a context, and its own directory or root.
|
|
func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) {
|
|
// The newest build of each module whatever its outcome: a failed one newer than the newest that
|
|
// worked leaves that one's build source stale — the merge it was asked for may have changed the closure
|
|
// (novox/hq ADR 0267), so its module is read whole until a build works again.
|
|
newest := map[string]struct {
|
|
at time.Time
|
|
failed bool
|
|
}{}
|
|
tried, err := i.store.Pool().Query(ctx,
|
|
`select distinct on (module) module, coalesce(asked, at), failed <> ''
|
|
from build
|
|
where module is not null and module <> ''
|
|
order by module, `+newestRequestFirst)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for tried.Next() {
|
|
var module string
|
|
var at time.Time
|
|
var failed bool
|
|
if err := tried.Scan(&module, &at, &failed); err != nil {
|
|
tried.Close()
|
|
return nil, err
|
|
}
|
|
newest[module] = struct {
|
|
at time.Time
|
|
failed bool
|
|
}{at, failed}
|
|
}
|
|
tried.Close()
|
|
if err := tried.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select distinct on (module) module, built_contexts, build_sources, coalesce(asked, at)
|
|
from build
|
|
where module is not null and module <> '' and failed = ''
|
|
order by module, `+newestRequestFirst)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
read := map[string][]ReadRepository{}
|
|
for rows.Next() {
|
|
var module string
|
|
var raw, rawSources []byte
|
|
var built time.Time
|
|
if err := rows.Scan(&module, &raw, &rawSources, &built); err != nil {
|
|
return nil, err
|
|
}
|
|
var of []ReadRepository
|
|
if len(raw) > 0 {
|
|
if err := json.Unmarshal(raw, &of); err != nil {
|
|
of = nil
|
|
}
|
|
}
|
|
var sources []BuildSource
|
|
if len(rawSources) > 0 {
|
|
if err := json.Unmarshal(rawSources, &sources); err != nil {
|
|
sources = nil
|
|
}
|
|
}
|
|
whole := ""
|
|
if n, known := newest[module]; known && n.failed && n.at.After(built) && len(sources) > 0 {
|
|
sources, whole = nil, "a newer build of it failed"
|
|
}
|
|
of = WithBuildSources(of, sources)
|
|
if whole != "" {
|
|
of = append(of, ReadRepository{Own: true, Whole: whole})
|
|
}
|
|
if len(of) > 0 {
|
|
for k := range of {
|
|
of[k].Built, of[k].Looked = built, newest[module].at
|
|
}
|
|
read[module] = of
|
|
}
|
|
}
|
|
return read, rows.Err()
|
|
}
|
|
|
|
// WithBuildSources lays a build's build sources over the repositories it read: a context's paths on its
|
|
// entry, and the module's own as an entry of its own. A context the build read that its sources do not
|
|
// name stays whole; a source naming a context the build did not say it read is dropped, since nothing
|
|
// moves a module through a repository it is not recorded as reading.
|
|
func WithBuildSources(read []ReadRepository, sources []BuildSource) []ReadRepository {
|
|
out := make([]ReadRepository, 0, len(read)+1)
|
|
for _, r := range read {
|
|
r.Paths, r.Own = nil, false
|
|
for _, s := range sources {
|
|
if s.Repository != "" && s.Repository == r.Repository && s.Ref == r.Ref && len(s.Paths) > 0 {
|
|
r.Paths = append([]string(nil), s.Paths...)
|
|
}
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
for _, s := range sources {
|
|
if s.Repository == "" && len(s.Paths) > 0 {
|
|
out = append(out, ReadRepository{Ref: s.Ref, Paths: append([]string(nil), s.Paths...), Own: true})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept".
|
|
//
|
|
// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one
|
|
// whose manifest was empty. A replay can then say which it is holding instead of inventing an
|
|
// empty declaration for a module that certainly had one.
|
|
func manifestOrNil(raw []byte) any {
|
|
if len(raw) == 0 {
|
|
return nil
|
|
}
|
|
return raw
|
|
}
|
|
|
|
// Announceable is every build worth telling a catalogue about, oldest first.
|
|
//
|
|
// **Oldest first, because a graph is built in the order things happened.** Registering a module
|
|
// that stands on a base before the base itself would make the edge point at a version the
|
|
// catalogue has not seen, and the shape of a fresh mesh guarantees that order matters: the base is
|
|
// always first and always the one that was missed.
|
|
//
|
|
// Only builds that succeeded and know what they built. A failure produced no module-version, and
|
|
// announcing one would put something in the graph that was never made — the same rule the builder
|
|
// follows when it decides whether to announce at all.
|
|
//
|
|
// One row per module and commit: a module built twice at the same commit is one fact, and the
|
|
// row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219).
|
|
func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) {
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select distinct on (module, commit_hash)
|
|
id, repository, ref, module, commit_hash, built_on, failed, made,
|
|
source_path, manifest, built_against, at
|
|
from build
|
|
where failed = '' and module is not null and module <> '' and commit_hash <> ''
|
|
order by module, commit_hash, `+newestRequestFirst)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var out []Build
|
|
for rows.Next() {
|
|
var b Build
|
|
var made []byte
|
|
var manifest, against []byte
|
|
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
|
|
&b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := json.Unmarshal(made, &b.Made); err != nil {
|
|
return nil, err
|
|
}
|
|
// Null rather than empty is a build recorded before the mesh kept these, and saying so is
|
|
// the point of keeping them nullable: the replay carries nothing rather than an empty
|
|
// declaration for a module that certainly had one.
|
|
if len(manifest) > 0 {
|
|
b.Manifest = manifest
|
|
}
|
|
if len(against) > 0 {
|
|
if err := json.Unmarshal(against, &b.Against); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
out = append(out, b)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
// Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and
|
|
// the order that matters to a catalogue is a different one.
|
|
sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) })
|
|
return out, nil
|
|
}
|