jschoubben
aa771616bb
Three faults in one path. A merge rebuilt every module built from the repository, so one change in a repository holding twenty-six of them meant twenty-six builds. A merge into a repository a module only *packages* source from rebuilt nothing — two modules are built from the control plane's own repository and neither had ever been rebuilt when it moved — because the manifest the mesh keeps carries no build section, so a build now says which repositories it read and the mesh keeps that beside what it stood on. And a module handed over by hand could record a repository with no directory inside it, which is a module nothing can ever rebuild (novox/hq 04-ISSUES/131, /132). A change inside no module's own directory is a change to what they share, and everything built from that repository is rebuilt: rebuilding too much is the safe direction, because the fault this whole path exists for is a mesh that believes it is current and is not.
324 lines
11 KiB
Go
324 lines
11 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"sort"
|
|
"time"
|
|
)
|
|
|
|
// What has been built.
|
|
//
|
|
// A build result was answered to whoever asked and kept nowhere, so "when did this last build",
|
|
// "why did it fail" and "which machine built what is running" had no answer. Failures are recorded
|
|
// too: one that leaves no trace is indistinguishable from a build nobody asked for, and the
|
|
// difference is the whole of whether somebody should be looking at something.
|
|
|
|
// Build is one attempt, whichever way it went.
|
|
type Build struct {
|
|
ID string
|
|
Repository string
|
|
Ref string
|
|
// Module is empty for a build that failed before it knew what it was building.
|
|
Module string
|
|
Commit string
|
|
// On is the machine that did it.
|
|
On string
|
|
// Path is where inside the repository the module lives (novox/hq ADR 0069).
|
|
Path string
|
|
// Manifest is the declaration the builder resolved, as it announced it.
|
|
//
|
|
// **Kept because the catalogue may not have been listening.** The announcement carries this
|
|
// and the catalogue turns it into the module's requires/provides edges. On a fresh mesh the
|
|
// modules built before the catalogue exists are exactly the ones it most needs, so the mesh
|
|
// has to be able to say afterwards what they declared (novox/hq 04-ISSUES/050).
|
|
Manifest []byte
|
|
// Against is every artifact this build stood on, as references rather than module names —
|
|
// what makes a build edge derived rather than declared (ADR 0009).
|
|
Against []string
|
|
// Read is every repository this build read source from besides the module's own (novox/hq
|
|
// 04-ISSUES/131), at the ref it read.
|
|
Read []ReadRepository
|
|
// Failed is the builder's own words, empty when it worked.
|
|
Failed string
|
|
Made []Artifact
|
|
At time.Time
|
|
}
|
|
|
|
// ReadRepository is a repository a build read source from besides the module's own.
|
|
type ReadRepository struct {
|
|
Repository string `json:"repository"`
|
|
Ref string `json:"ref,omitempty"`
|
|
}
|
|
|
|
// Artifact is one thing a build published.
|
|
type Artifact struct {
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind"`
|
|
Reference string `json:"reference"`
|
|
}
|
|
|
|
// Worked reports whether this build produced something.
|
|
func (b Build) Worked() bool { return b.Failed == "" }
|
|
|
|
// RecordBuild keeps what a builder said.
|
|
//
|
|
// Idempotent on the correlation id, because a result can arrive twice: once as the answer to
|
|
// whoever asked and once on the exchange when nobody was. Recording both would show one build as
|
|
// two, and which of the two is real is not a question anybody could answer afterwards.
|
|
func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
|
|
made, err := json.Marshal(b.Made)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
against, err := json.Marshal(b.Against)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
read, err := json.Marshal(b.Read)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var module *string
|
|
if b.Module != "" {
|
|
module = &b.Module
|
|
}
|
|
_, err = i.store.Pool().Exec(ctx,
|
|
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
|
|
source_path, manifest, built_against, built_contexts)
|
|
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
|
on conflict (id) do nothing`,
|
|
b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
|
|
b.Path, manifestOrNil(b.Manifest), against, read)
|
|
return err
|
|
}
|
|
|
|
// Builds is what has happened lately, newest first.
|
|
//
|
|
// For one module when named, or across the mesh when not. Both are asked: *what happened just
|
|
// now* after something goes wrong, and *what has happened to this* when deciding whether to
|
|
// trust it.
|
|
func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Build, error) {
|
|
if limit <= 0 {
|
|
limit = 20
|
|
}
|
|
query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, at
|
|
from build order by at desc limit $1`
|
|
args := []any{limit}
|
|
if module != "" {
|
|
query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, at
|
|
from build where module = $2 order by at desc limit $1`
|
|
args = append(args, module)
|
|
}
|
|
|
|
rows, err := i.store.Pool().Query(ctx, query, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var out []Build
|
|
for rows.Next() {
|
|
var b Build
|
|
var made []byte
|
|
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
|
|
&b.On, &b.Failed, &made, &b.At); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := json.Unmarshal(made, &b.Made); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, b)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// Held is every artifact this mesh has built, keyed "<module>/<artifact>".
|
|
//
|
|
// **The newest successful build of each module wins**, which is the same rule the rest of the mesh
|
|
// uses for what a module currently is. A module rebuilt to something broken and then rebuilt again
|
|
// is at the second one; a module whose last build failed is at the last one that worked, because a
|
|
// failure published nothing and the thing it published before is still what exists.
|
|
//
|
|
// Only successes, and only builds that knew what they were building: a build that failed before it
|
|
// could read a manifest has no module to be the artifact of.
|
|
func (i *Inventory) Held(ctx context.Context) (map[string]string, error) {
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select distinct on (module) module, made
|
|
from build
|
|
where module is not null and module <> '' and failed = ''
|
|
order by module, at desc`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
held := map[string]string{}
|
|
for rows.Next() {
|
|
var module string
|
|
var raw []byte
|
|
if err := rows.Scan(&module, &raw); err != nil {
|
|
return nil, err
|
|
}
|
|
var made []Artifact
|
|
if err := json.Unmarshal(raw, &made); err != nil {
|
|
// Skipped rather than fatal. One unreadable build record should not stop every other
|
|
// module's base from being answerable — and the build that needs this one will say
|
|
// plainly that it is missing.
|
|
continue
|
|
}
|
|
for _, artifact := range made {
|
|
if artifact.Name == "" || artifact.Reference == "" {
|
|
continue
|
|
}
|
|
held[module+"/"+artifact.Name] = artifact.Reference
|
|
}
|
|
}
|
|
return held, rows.Err()
|
|
}
|
|
|
|
// BuiltAgainst is what each module's newest successful build stood on, as recorded — the build
|
|
// edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
|
|
// module standing on nothing looks like: an edge the mesh has not derived is not an edge.
|
|
func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) {
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select distinct on (module) module, built_against
|
|
from build
|
|
where module is not null and module <> '' and failed = ''
|
|
order by module, at desc`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
against := map[string][]string{}
|
|
for rows.Next() {
|
|
var module string
|
|
var raw []byte
|
|
if err := rows.Scan(&module, &raw); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
continue
|
|
}
|
|
var refs []string
|
|
if err := json.Unmarshal(raw, &refs); err != nil {
|
|
continue
|
|
}
|
|
if len(refs) > 0 {
|
|
against[module] = refs
|
|
}
|
|
}
|
|
return against, rows.Err()
|
|
}
|
|
|
|
// ReadRepositories is what each module's newest successful build read source from besides its own
|
|
// repository, by module name.
|
|
//
|
|
// The mirror of BuiltAgainst, and derived the same way and for the same reason: a merge into a
|
|
// repository a module only packages is a change to that module, and the manifest the mesh keeps
|
|
// carries nothing that would say so (novox/hq 04-ISSUES/131).
|
|
func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) {
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select distinct on (module) module, built_contexts
|
|
from build
|
|
where module is not null and module <> '' and failed = ''
|
|
order by module, at desc`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
read := map[string][]ReadRepository{}
|
|
for rows.Next() {
|
|
var module string
|
|
var raw []byte
|
|
if err := rows.Scan(&module, &raw); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
continue
|
|
}
|
|
var of []ReadRepository
|
|
if err := json.Unmarshal(raw, &of); err != nil {
|
|
continue
|
|
}
|
|
if len(of) > 0 {
|
|
read[module] = of
|
|
}
|
|
}
|
|
return read, rows.Err()
|
|
}
|
|
|
|
// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept".
|
|
//
|
|
// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one
|
|
// whose manifest was empty. A replay can then say which it is holding instead of inventing an
|
|
// empty declaration for a module that certainly had one.
|
|
func manifestOrNil(raw []byte) any {
|
|
if len(raw) == 0 {
|
|
return nil
|
|
}
|
|
return raw
|
|
}
|
|
|
|
// Announceable is every build worth telling a catalogue about, oldest first.
|
|
//
|
|
// **Oldest first, because a graph is built in the order things happened.** Registering a module
|
|
// that stands on a base before the base itself would make the edge point at a version the
|
|
// catalogue has not seen, and the shape of a fresh mesh guarantees that order matters: the base is
|
|
// always first and always the one that was missed.
|
|
//
|
|
// Only builds that succeeded and know what they built. A failure produced no module-version, and
|
|
// announcing one would put something in the graph that was never made — the same rule the builder
|
|
// follows when it decides whether to announce at all.
|
|
//
|
|
// One row per module and commit: a module built twice at the same commit is one fact, and the
|
|
// latest row is the one whose artifacts are current.
|
|
func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) {
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select distinct on (module, commit_hash)
|
|
id, repository, ref, module, commit_hash, built_on, failed, made,
|
|
source_path, manifest, built_against, at
|
|
from build
|
|
where failed = '' and module is not null and module <> '' and commit_hash <> ''
|
|
order by module, commit_hash, at desc`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var out []Build
|
|
for rows.Next() {
|
|
var b Build
|
|
var made []byte
|
|
var manifest, against []byte
|
|
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
|
|
&b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := json.Unmarshal(made, &b.Made); err != nil {
|
|
return nil, err
|
|
}
|
|
// Null rather than empty is a build recorded before the mesh kept these, and saying so is
|
|
// the point of keeping them nullable: the replay carries nothing rather than an empty
|
|
// declaration for a module that certainly had one.
|
|
if len(manifest) > 0 {
|
|
b.Manifest = manifest
|
|
}
|
|
if len(against) > 0 {
|
|
if err := json.Unmarshal(against, &b.Against); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
out = append(out, b)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
// Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and
|
|
// the order that matters to a catalogue is a different one.
|
|
sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) })
|
|
return out, nil
|
|
}
|