package inventory import ( "context" "encoding/json" "errors" "sort" "time" "github.com/jackc/pgx/v5" ) // What has been built. // // A build result was answered to whoever asked and kept nowhere, so "when did this last build", // "why did it fail" and "which machine built what is running" had no answer. Failures are recorded // too: one that leaves no trace is indistinguishable from a build nobody asked for, and the // difference is the whole of whether somebody should be looking at something. // Build is one attempt, whichever way it went. type Build struct { ID string Repository string Ref string // Module is empty for a build that failed before it knew what it was building. Module string Commit string // On is the machine that did it. On string // Path is where inside the repository the module lives (novox/hq ADR 0069). Path string // Manifest is the declaration the builder resolved, as it announced it. // // **Kept because the catalogue may not have been listening.** The announcement carries this // and the catalogue turns it into the module's requires/provides edges. On a fresh mesh the // modules built before the catalogue exists are exactly the ones it most needs, so the mesh // has to be able to say afterwards what they declared (novox/hq 04-ISSUES/050). Manifest []byte // Against is every artifact this build stood on, as references rather than module names — // what makes a build edge derived rather than declared (ADR 0009). Against []string // Mirrored is every base this build copied into the artifact store (novox/hq ADR 0257), said // whether the build worked or not: the copy is the mesh's, recorded in artifact_mirrored. Mirrored []string // Read is every repository this build read source from besides the module's own (novox/hq // 04-ISSUES/131), at the ref it read. Read []ReadRepository // Sources are what the build was made from, as files, per repository (novox/hq ADR 0267): said by the // builder for a build of a trunk commit; nil where it said none. Sources []BuildSource // SourceFingerprint is what the build was made from, hashed, as its builder said it (novox/hq // issue 280); empty from a builder that predates it, or where the source does not pin the build. SourceFingerprint string // Failed is the builder's own words, empty when it worked. Failed string Made []Artifact // Asked is when the build was requested, zero when that is not known (an id of another shape, // or a build recorded before the mesh kept it). **What orders one build of a module against // another** (novox/hq 04-ISSUES/219): builds in flight together finish in any order, and the // one asked last stood on the newest bases. Asked time.Time // At is when the outcome was recorded — when it finished, not when it was asked. At time.Time } // AskedOrAt is when the build was asked, or when it was recorded when that is not known — the // order the mesh had before it kept the request time. func (b Build) AskedOrAt() time.Time { if !b.Asked.IsZero() { return b.Asked } return b.At } // newestRequestFirst is the ordering every "what a module currently is" question uses: the newest // request wins, whenever it finished (novox/hq 04-ISSUES/219). A build whose request time is not // known is placed at the moment it was recorded, which is the rule that held before. const newestRequestFirst = `coalesce(asked, at) desc, at desc` // ReadRepository is a repository a build read source from besides the module's own. // // Paths and Own are never stored in a build's `built_contexts` (a controller that predates them would read // an own entry as a context, and every module of a repository as packaging every other): ReadRepositories // lays them over what it reads, from the build's build sources (novox/hq ADR 0267). type ReadRepository struct { Repository string `json:"repository"` Ref string `json:"ref,omitempty"` // Paths are the build source the build read in this repository (builder.SourceHolds): a changed file // outside them is no change to the module. Empty is the whole repository, as before. Paths []string `json:"paths,omitempty"` // Own is the module's own repository, whose Paths narrow what its own directory — or, for a module // built from its repository's root, the whole repository — would otherwise be. Own bool `json:"own,omitempty"` // Built is when the build these were read from was asked, and Looked when the module's newest build of // any outcome was: what the planner judges a build source's age, and a merge's news, by. Never stored. Built time.Time `json:"-"` Looked time.Time `json:"-"` // LookedAt are the merge commits a plan that built the module, or is building it, answered: a merge // of one of them is history for the module whatever the clocks say. Never stored. LookedAt []string `json:"-"` // Whole is why the module is read whole though a build source may have been recorded — a newer build // failed, or a plan has not built it yet — said on an Own entry with no Paths. Never stored. Whole string `json:"-"` } // BuildSource is the build source a build read in one repository (novox/hq ADR 0267): Repository and Ref // a context's, empty for the module's own. type BuildSource struct { Repository string `json:"repository,omitempty"` Ref string `json:"ref,omitempty"` Paths []string `json:"paths"` } // Artifact is one thing a build published. type Artifact struct { Name string `json:"name"` Kind string `json:"kind"` Reference string `json:"reference"` } // Worked reports whether this build produced something. func (b Build) Worked() bool { return b.Failed == "" } // RecordBuild keeps what a builder said. // // Idempotent on the correlation id, because a result can arrive twice: once as the answer to // whoever asked and once on the exchange when nobody was. Recording both would show one build as // two, and which of the two is real is not a question anybody could answer afterwards. func (i *Inventory) RecordBuild(ctx context.Context, b Build) error { made, err := json.Marshal(b.Made) if err != nil { return err } against, err := json.Marshal(b.Against) if err != nil { return err } read, err := json.Marshal(b.Read) if err != nil { return err } var sources any if len(b.Sources) > 0 { raw, err := json.Marshal(b.Sources) if err != nil { return err } sources = raw } var module *string if b.Module != "" { module = &b.Module } var asked *time.Time if !b.Asked.IsZero() { asked = &b.Asked } _, err = i.store.Pool().Exec(ctx, `insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest, built_against, built_contexts, asked, source_fingerprint, build_sources) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) on conflict (id) do nothing`, b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made, b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint, sources) if err != nil { return err } return i.RecordMirrored(ctx, b.ID, "", b.Mirrored) } // RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257): // by the build that said so, or by a person who says why. The first record of a copy stands. // // **A copy a build says it holds is held again.** The builder answers a copy only once the store holds // it whole, copying again what was let go of; so a copy the records say was collected, and a build now // says it copied, is no longer collected. Left collected, it would stay in the store and out of every // sweep for ever. func (i *Inventory) RecordMirrored(ctx context.Context, build, why string, references []string) error { var by *string if build != "" { by = &build } for _, reference := range references { if reference == "" { continue } if _, err := i.store.Pool().Exec(ctx, `insert into artifact_mirrored (reference, build, why) values ($1, $2, $3) on conflict (reference) do nothing`, reference, by, why); err != nil { return err } if build != "" { if _, err := i.store.Pool().Exec(ctx, `delete from artifact_collected where reference = $1`, reference); err != nil { return err } } } return nil } // Builds is what has happened lately, newest first. // // For one module when named, or across the mesh when not. Both are asked: *what happened just // now* after something goes wrong, and *what has happened to this* when deciding whether to // trust it. func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Build, error) { if limit <= 0 { limit = 20 } query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at from build order by at desc limit $1` args := []any{limit} if module != "" { query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at from build where module = $2 order by at desc limit $1` args = append(args, module) } rows, err := i.store.Pool().Query(ctx, query, args...) if err != nil { return nil, err } defer rows.Close() var out []Build for rows.Next() { var b Build var made []byte var asked *time.Time if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &asked, &b.At); err != nil { return nil, err } if asked != nil { b.Asked = *asked } if err := json.Unmarshal(made, &b.Made); err != nil { return nil, err } out = append(out, b) } return out, rows.Err() } // BuildByID is one build's record, by the id its request carried — what a plan matches its outcome // by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the // repository, path, ref and commit from. False when no outcome with that id was recorded. func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) { var b Build var made []byte var asked *time.Time err := i.store.Pool().QueryRow(ctx, `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, coalesce(source_path,''), asked, at from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path, &asked, &b.At) if errors.Is(err, pgx.ErrNoRows) { return Build{}, false, nil } if err != nil { return Build{}, false, err } if asked != nil { b.Asked = *asked } if err := json.Unmarshal(made, &b.Made); err != nil { return Build{}, false, err } return b, true, nil } // Held is every artifact this mesh has built, keyed "/". // // **The successful build of each module asked last wins**, which is the same rule the rest of the // mesh uses for what a module currently is — asked last, not finished last (novox/hq // 04-ISSUES/219): an older request that finishes later stood on older bases. A module rebuilt to something broken and then rebuilt again // is at the second one; a module whose last build failed is at the last one that worked, because a // failure published nothing and the thing it published before is still what exists. // // Only successes, and only builds that knew what they were building: a build that failed before it // could read a manifest has no module to be the artifact of. func (i *Inventory) Held(ctx context.Context) (map[string]string, error) { // **A rebuild of an unchanged source holds what the first build of it made** (novox/hq issue 280): // the mesh registers that build's artifacts, so a module standing on it is handed the same base // and is unchanged too, rather than moving for a digest an image rebuild could not help changing. made, err := i.heldMade(ctx) if err != nil { return nil, err } held := map[string]string{} for module, artifacts := range made { for _, artifact := range artifacts { if artifact.Name == "" || artifact.Reference == "" { continue } held[module+"/"+artifact.Name] = artifact.Reference } } return held, nil } // BuiltAgainst is what each module's newest successful build stood on, as recorded — the build // edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a // module standing on nothing looks like: an edge the mesh has not derived is not an edge. func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) { rows, err := i.store.Pool().Query(ctx, `select distinct on (module) module, built_against from build where module is not null and module <> '' and failed = '' order by module, `+newestRequestFirst) if err != nil { return nil, err } defer rows.Close() against := map[string][]string{} for rows.Next() { var module string var raw []byte if err := rows.Scan(&module, &raw); err != nil { return nil, err } if len(raw) == 0 { continue } var refs []string if err := json.Unmarshal(raw, &refs); err != nil { continue } if len(refs) > 0 { against[module] = refs } } return against, rows.Err() } // ReadRepositories is what each module's newest successful build read source from besides its own // repository, by module name. // // The mirror of BuiltAgainst, and derived the same way and for the same reason: a merge into a // repository a module only packages is a change to that module, and the manifest the mesh keeps // carries nothing that would say so (novox/hq 04-ISSUES/131). // // **With the build source that build said** (novox/hq ADR 0267): a context's entry carries the paths the // build read there, and the module's own repository an entry marked Own with its paths. A build that said // none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read // as before: every file of a context, and its own directory or root. func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) { // The newest build of each module whatever its outcome: a failed one newer than the newest that // worked leaves that one's build source stale — the merge it was asked for may have changed the closure // (novox/hq ADR 0267), so its module is read whole until a build works again. newest := map[string]struct { at time.Time failed bool }{} tried, err := i.store.Pool().Query(ctx, `select distinct on (module) module, coalesce(asked, at), failed <> '' from build where module is not null and module <> '' order by module, `+newestRequestFirst) if err != nil { return nil, err } for tried.Next() { var module string var at time.Time var failed bool if err := tried.Scan(&module, &at, &failed); err != nil { tried.Close() return nil, err } newest[module] = struct { at time.Time failed bool }{at, failed} } tried.Close() if err := tried.Err(); err != nil { return nil, err } rows, err := i.store.Pool().Query(ctx, `select distinct on (module) module, built_contexts, build_sources, coalesce(asked, at) from build where module is not null and module <> '' and failed = '' order by module, `+newestRequestFirst) if err != nil { return nil, err } defer rows.Close() read := map[string][]ReadRepository{} for rows.Next() { var module string var raw, rawSources []byte var built time.Time if err := rows.Scan(&module, &raw, &rawSources, &built); err != nil { return nil, err } var of []ReadRepository if len(raw) > 0 { if err := json.Unmarshal(raw, &of); err != nil { of = nil } } var sources []BuildSource if len(rawSources) > 0 { if err := json.Unmarshal(rawSources, &sources); err != nil { sources = nil } } whole := "" if n, known := newest[module]; known && n.failed && n.at.After(built) && len(sources) > 0 { sources, whole = nil, "a newer build of it failed" } of = WithBuildSources(of, sources) if whole != "" { of = append(of, ReadRepository{Own: true, Whole: whole}) } if len(of) > 0 { for k := range of { of[k].Built, of[k].Looked = built, newest[module].at } read[module] = of } } return read, rows.Err() } // WithBuildSources lays a build's build sources over the repositories it read: a context's paths on its // entry, and the module's own as an entry of its own. A context the build read that its sources do not // name stays whole; a source naming a context the build did not say it read is dropped, since nothing // moves a module through a repository it is not recorded as reading. func WithBuildSources(read []ReadRepository, sources []BuildSource) []ReadRepository { out := make([]ReadRepository, 0, len(read)+1) for _, r := range read { r.Paths, r.Own = nil, false for _, s := range sources { if s.Repository != "" && s.Repository == r.Repository && s.Ref == r.Ref && len(s.Paths) > 0 { r.Paths = append([]string(nil), s.Paths...) } } out = append(out, r) } for _, s := range sources { if s.Repository == "" && len(s.Paths) > 0 { out = append(out, ReadRepository{Ref: s.Ref, Paths: append([]string(nil), s.Paths...), Own: true}) } } return out } // manifestOrNil keeps the difference between "declared nothing" and "predates this being kept". // // A build recorded before the mesh kept manifests has no manifest, and that is not the same as one // whose manifest was empty. A replay can then say which it is holding instead of inventing an // empty declaration for a module that certainly had one. func manifestOrNil(raw []byte) any { if len(raw) == 0 { return nil } return raw } // Announceable is every build worth telling a catalogue about, oldest first. // // **Oldest first, because a graph is built in the order things happened.** Registering a module // that stands on a base before the base itself would make the edge point at a version the // catalogue has not seen, and the shape of a fresh mesh guarantees that order matters: the base is // always first and always the one that was missed. // // Only builds that succeeded and know what they built. A failure produced no module-version, and // announcing one would put something in the graph that was never made — the same rule the builder // follows when it decides whether to announce at all. // // One row per module and commit: a module built twice at the same commit is one fact, and the // row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219). func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) { rows, err := i.store.Pool().Query(ctx, `select distinct on (module, commit_hash) id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest, built_against, at from build where failed = '' and module is not null and module <> '' and commit_hash <> '' order by module, commit_hash, `+newestRequestFirst) if err != nil { return nil, err } defer rows.Close() var out []Build for rows.Next() { var b Build var made []byte var manifest, against []byte if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil { return nil, err } if err := json.Unmarshal(made, &b.Made); err != nil { return nil, err } // Null rather than empty is a build recorded before the mesh kept these, and saying so is // the point of keeping them nullable: the replay carries nothing rather than an empty // declaration for a module that certainly had one. if len(manifest) > 0 { b.Manifest = manifest } if len(against) > 0 { if err := json.Unmarshal(against, &b.Against); err != nil { return nil, err } } out = append(out, b) } if err := rows.Err(); err != nil { return nil, err } // Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and // the order that matters to a catalogue is a different one. sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) }) return out, nil }