package inventory import ( "context" "strings" "time" "github.com/novox/mesh-controller/internal/chain" ) // What has been built. // // A build result was answered to whoever asked and kept nowhere, so "when did this last build", // "why did it fail" and "which machine built what is running" had no answer. Failures are recorded // too: one that leaves no trace is indistinguishable from a build nobody asked for, and the // difference is the whole of whether somebody should be looking at something. // Build is one attempt, whichever way it went. type Build struct { ID string // ManifestDigest names the manifest in the manifests bucket, when a chain keeps it apart from the record (issue 1121). ManifestDigest string Repository string Ref string // Module is empty for a build that failed before it knew what it was building. Module string Commit string // On is the machine that did it. On string // Path is where inside the repository the module lives (novox/hq ADR 0069). Path string // Manifest is the declaration the builder resolved, as it announced it. // // **Kept because the catalogue may not have been listening.** The announcement carries this // and the catalogue turns it into the module's requires/provides edges. On a fresh mesh the // modules built before the catalogue exists are exactly the ones it most needs, so the mesh // has to be able to say afterwards what they declared (novox/hq 04-ISSUES/050). Manifest []byte // Against is every artifact this build stood on, as references rather than module names — // what makes a build edge derived rather than declared (ADR 0009). Against []string // Mirrored is every base this build copied into the artifact store (novox/hq ADR 0257), said // whether the build worked or not: the copy is the mesh's, recorded in artifact_mirrored. Mirrored []string // Read is every repository this build read source from besides the module's own (novox/hq // 04-ISSUES/131), at the ref it read. Read []ReadRepository // Sources are what the build was made from, as files, per repository (novox/hq ADR 0267): said by the // builder for a build of a trunk commit; nil where it said none. Sources []BuildSource // SourceFingerprint is what the build was made from, hashed, as its builder said it (novox/hq // issue 280); empty from a builder that predates it, or where the source does not pin the build. SourceFingerprint string // Failed is the builder's own words, empty when it worked. Failed string Made []Artifact // Asked is when the build was requested, zero when that is not known (an id of another shape, // or a build recorded before the mesh kept it). **What orders one build of a module against // another** (novox/hq 04-ISSUES/219): builds in flight together finish in any order, and the // one asked last stood on the newest bases. Asked time.Time // At is when the outcome was recorded — when it finished, not when it was asked. At time.Time // ReuseKey is the build's reuse key (novox/hq ADR 0350 §1), and Published what it published: an artifact by its // digest, or a package release by name and version (ADR 0408 §1). Kept on the bus; a build kept in the table has neither. ReuseKey string Published *chain.Published } // AskedOrAt is when the build was asked, or when it was recorded when that is not known — the // order the mesh had before it kept the request time. func (b Build) AskedOrAt() time.Time { if !b.Asked.IsZero() { return b.Asked } return b.At } // newestRequestFirst is the ordering every "what a module currently is" question uses: the newest // request wins, whenever it finished (novox/hq 04-ISSUES/219). A build whose request time is not // known is placed at the moment it was recorded, which is the rule that held before. const newestRequestFirst = `coalesce(asked, at) desc, at desc` // ReadRepository is a repository a build read source from besides the module's own. // // Paths and Own are never stored in a build's `built_contexts` (a controller that predates them would read // an own entry as a context, and every module of a repository as packaging every other): ReadRepositories // lays them over what it reads, from the build's build sources (novox/hq ADR 0267). type ReadRepository struct { Repository string `json:"repository"` Ref string `json:"ref,omitempty"` // Paths are the build source the build read in this repository (builder.SourceHolds): a changed file // outside them is no change to the module. Empty is the whole repository, as before. Paths []string `json:"paths,omitempty"` // Own is the module's own repository, whose Paths narrow what its own directory — or, for a module // built from its repository's root, the whole repository — would otherwise be. Own bool `json:"own,omitempty"` // Built is when the build these were read from was asked, and Looked when the module's newest build of // any outcome was: what the planner judges a build source's age, and a merge's news, by. Never stored. Built time.Time `json:"-"` Looked time.Time `json:"-"` // LookedAt are the merge commits a plan that built the module, or is building it, answered: a merge // of one of them is history for the module whatever the clocks say. Never stored. LookedAt []string `json:"-"` // Whole is why the module is read whole though a build source may have been recorded — a newer build // failed, or a plan has not built it yet — said on an Own entry with no Paths. Never stored. Whole string `json:"-"` } // BuildSource is the build source a build read in one repository (novox/hq ADR 0267): Repository and Ref // a context's, empty for the module's own. type BuildSource struct { Repository string `json:"repository,omitempty"` Ref string `json:"ref,omitempty"` Paths []string `json:"paths"` } // Artifact is one thing a build published. type Artifact struct { Name string `json:"name"` Kind string `json:"kind"` Reference string `json:"reference"` } // Worked reports whether this build produced something. func (b Build) Worked() bool { return b.Failed == "" } // RecordBuild keeps what a builder said. // // Idempotent on the correlation id, because a result can arrive twice: once as the answer to // whoever asked and once on the exchange when nobody was. Recording both would show one build as // two, and which of the two is real is not a question anybody could answer afterwards. func (i *Inventory) RecordBuild(ctx context.Context, b Build) error { c, err := i.theChain(ctx) if err != nil { return err } return i.chainRecordBuild(ctx, c, b) } // RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257): // by the build that said so, or by a person who says why. The first record of a copy stands. // // **A copy a build says it holds is held again.** The builder answers a copy only once the store holds // it whole, copying again what was let go of; so a copy the records say was collected, and a build now // says it copied, is no longer collected. Left collected, it would stay in the store and out of every // sweep for ever. func (i *Inventory) RecordMirrored(ctx context.Context, build, why string, references []string) error { var by *string if build != "" { by = &build } for _, reference := range references { if reference == "" { continue } if _, err := i.store.Pool().Exec(ctx, `insert into artifact_mirrored (reference, build, why) values ($1, $2, $3) on conflict (reference) do nothing`, reference, by, why); err != nil { return err } if build != "" { if _, err := i.store.Pool().Exec(ctx, `delete from artifact_collected where reference = $1`, reference); err != nil { return err } } } return nil } // Builds is what has happened lately, newest first. // // For one module when named, or across the mesh when not. Both are asked: *what happened just // now* after something goes wrong, and *what has happened to this* when deciding whether to // trust it. func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Build, error) { if limit <= 0 { limit = 20 } c, err := i.theChain(ctx) if err != nil { return nil, err } return i.chainBuildList(ctx, c, module, limit) } // BuildByID is one build's record, by the id its request carried — what a plan matches its outcome // by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the // repository, path, ref and commit from. False when no outcome with that id was recorded. func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) { c, err := i.theChain(ctx) if err != nil { return Build{}, false, err } return i.chainBuildByID(ctx, c, id) } // Held is every artifact this mesh has built, keyed "/". // // **The successful build of each module asked last wins**, which is the same rule the rest of the // mesh uses for what a module currently is — asked last, not finished last (novox/hq // 04-ISSUES/219): an older request that finishes later stood on older bases. A module rebuilt to something broken and then rebuilt again // is at the second one; a module whose last build failed is at the last one that worked, because a // failure published nothing and the thing it published before is still what exists. // // Only successes, and only builds that knew what they were building: a build that failed before it // could read a manifest has no module to be the artifact of. func (i *Inventory) Held(ctx context.Context) (map[string]string, error) { // **A rebuild of an unchanged source holds what the first build of it made** (novox/hq issue 280): // the mesh registers that build's artifacts, so a module standing on it is handed the same base // and is unchanged too, rather than moving for a digest an image rebuild could not help changing. made, err := i.heldMade(ctx) if err != nil { return nil, err } held := map[string]string{} for module, artifacts := range made { for _, artifact := range artifacts { if artifact.Name == "" || artifact.Reference == "" { continue } held[module+"/"+artifact.Name] = artifact.Reference } } return held, nil } // RegisteredHeld is Held narrowed to the build each module has registered (novox/hq issue 721): an artifact is kept // only when its reference's digest is in the manifest the module table holds for it, which is what its machines are // sent. After a gate failed and the mesh put the earlier build back (RestoreModule), the newest successful build is // not what runs, and its artifacts are left out here, so nothing is compared with them. func (i *Inventory) RegisteredHeld(ctx context.Context) (map[string]string, error) { held, err := i.Held(ctx) if err != nil { return nil, err } rows, err := i.store.Pool().Query(ctx, `select name, coalesce(manifest::text, '') from module`) if err != nil { return nil, err } defer rows.Close() registered := map[string]string{} for rows.Next() { var name, manifest string if err := rows.Scan(&name, &manifest); err != nil { return nil, err } registered[name] = manifest } if err := rows.Err(); err != nil { return nil, err } return registeredOnly(held, registered), nil } // registeredOnly keeps of held each "/" whose reference's digest the module's registered manifest // names. func registeredOnly(held, registered map[string]string) map[string]string { out := map[string]string{} for key, ref := range held { module, _, ok := strings.Cut(key, "/") at := strings.LastIndex(ref, "@") if !ok || at < 0 || len(ref) <= at+1 { continue } if strings.Contains(registered[module], ref[at+1:]) { out[key] = ref } } return out } // BuiltAgainst is what each module's newest successful build stood on, as recorded — the build // edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a // module standing on nothing looks like: an edge the mesh has not derived is not an edge. func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) { c, err := i.theChain(ctx) if err != nil { return nil, err } return i.chainBuiltAgainst(ctx, c) } // ReadRepositories is what each module's newest successful build read source from besides its own // repository, by module name. // // The mirror of BuiltAgainst, and derived the same way and for the same reason: a merge into a // repository a module only packages is a change to that module, and the manifest the mesh keeps // carries nothing that would say so (novox/hq 04-ISSUES/131). // // **With the build source that build said** (novox/hq ADR 0267): a context's entry carries the paths the // build read there, and the module's own repository an entry marked Own with its paths. A build that said // none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read // as before: every file of a context, and its own directory or root. func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) { c, err := i.theChain(ctx) if err != nil { return nil, err } return i.chainReadRepositories(ctx, c) } // WithBuildSources lays a build's build sources over the repositories it read: a context's paths on its // entry, and the module's own as an entry of its own. A context the build read that its sources do not // name stays whole; a source naming a context the build did not say it read is dropped, since nothing // moves a module through a repository it is not recorded as reading. func WithBuildSources(read []ReadRepository, sources []BuildSource) []ReadRepository { out := make([]ReadRepository, 0, len(read)+1) for _, r := range read { r.Paths, r.Own = nil, false for _, s := range sources { if s.Repository != "" && s.Repository == r.Repository && s.Ref == r.Ref && len(s.Paths) > 0 { r.Paths = append([]string(nil), s.Paths...) } } out = append(out, r) } for _, s := range sources { if s.Repository == "" && len(s.Paths) > 0 { out = append(out, ReadRepository{Ref: s.Ref, Paths: append([]string(nil), s.Paths...), Own: true}) } } return out } // OwnRepositoryAsOwn reads a context that names the module's own registered repository, with the paths its // build read there, as the module's own build source (novox/hq ADR 0420 §4): a change to the program it // compiles moves the module, as one in its own directory does. A context of that repository that said // no paths stays a context read, whole. func OwnRepositoryAsOwn(read []ReadRepository, repository string) []ReadRepository { if repository == "" { return read } out := make([]ReadRepository, 0, len(read)) var moved []ReadRepository for _, r := range read { if !r.Own && len(r.Paths) > 0 && r.Repository != "" && namesRepository(repository, r.Repository) { moved = append(moved, r) continue } out = append(out, r) } if len(moved) == 0 { return read } // One entry of its own: ownSource reads the first that has paths. at := -1 for k := range out { if out[k].Own && len(out[k].Paths) > 0 { at = k break } } if at < 0 { out = append(out, ReadRepository{Own: true}) at = len(out) - 1 } for _, r := range moved { out[at].Paths = append(append([]string(nil), out[at].Paths...), r.Paths...) if out[at].Ref == "" { out[at].Ref = r.Ref } } return out } // namesRepository is whether a context's repository, as owner/name, is the registered one, which may be // spelled as the URL it was cloned from. func namesRepository(registered, named string) bool { trim := func(s string) string { return strings.TrimSuffix(strings.TrimRight(strings.ToLower(strings.TrimSpace(s)), "/"), ".git") } reg, name := trim(registered), trim(named) return reg == name || strings.HasSuffix(reg, "/"+name) } // Announceable is every build worth telling a catalogue about, oldest first. // // **Oldest first, because a graph is built in the order things happened.** Registering a module // that stands on a base before the base itself would make the edge point at a version the // catalogue has not seen, and the shape of a fresh mesh guarantees that order matters: the base is // always first and always the one that was missed. // // Only builds that succeeded and know what they built. A failure produced no module-version, and // announcing one would put something in the graph that was never made — the same rule the builder // follows when it decides whether to announce at all. // // One row per module and commit: a module built twice at the same commit is one fact, and the // row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219). func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) { c, err := i.theChain(ctx) if err != nil { return nil, err } return i.chainAnnounceable(ctx, c) }