package inventory import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "sort" "strings" "time" "github.com/novox/mesh-controller/internal/chain" ) // Builds on the bus (novox/hq ADR 0408, issue 1090): once the kind `builds` is moved (chain.go) every build is a key of // the builds bucket, under its id, and every read below is made from the bucket's records. The tables are read when the // kind is not moved, by the methods these stand beside. // toChainBuild is a build as the bucket keeps it. func toChainBuild(b Build) chain.Build { c := chain.Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On, Path: b.Path, Against: b.Against, Mirrored: b.Mirrored, SourceFingerprint: b.SourceFingerprint, Failed: b.Failed, Asked: b.Asked.UTC(), At: b.At.UTC(), ReuseKey: b.ReuseKey, Published: b.Published} if len(b.Manifest) > 0 { c.Manifest = json.RawMessage(b.Manifest) } for _, r := range b.Read { c.Read = append(c.Read, chain.ReadRepository{Repository: r.Repository, Ref: r.Ref, Paths: r.Paths, Own: r.Own}) } for _, s := range b.Sources { c.Sources = append(c.Sources, chain.Source{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) } for _, a := range b.Made { c.Made = append(c.Made, chain.Artifact{Name: a.Name, Kind: a.Kind, Reference: a.Reference}) } if c.Published == nil { c.Published = publishedOf(b.Made) } return c } // fromChainBuild is a build as the mesh reads it. func fromChainBuild(c chain.Build) Build { b := Build{ID: c.ID, Repository: c.Repository, Ref: c.Ref, Module: c.Module, Commit: c.Commit, On: c.On, Path: c.Path, Manifest: []byte(c.Manifest), Against: c.Against, Mirrored: c.Mirrored, SourceFingerprint: c.SourceFingerprint, Failed: c.Failed, Asked: c.Asked, At: c.At, ReuseKey: c.ReuseKey, Published: c.Published} if len(b.Manifest) == 0 { b.Manifest = nil } for _, r := range c.Read { b.Read = append(b.Read, ReadRepository{Repository: r.Repository, Ref: r.Ref, Paths: r.Paths, Own: r.Own}) } for _, s := range c.Sources { b.Sources = append(b.Sources, BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) } for _, a := range c.Made { b.Made = append(b.Made, Artifact{Name: a.Name, Kind: a.Kind, Reference: a.Reference}) } return b } // publishedOf is what a build published, read from what it made: a package by name and version, and an artifact by the // digest of the first thing it made that has one (`@sha256:…`). Nil for a build that published nothing. func publishedOf(made []Artifact) *chain.Published { var out chain.Published for _, a := range made { switch { case a.Kind == "package": if name, version, ok := strings.Cut(a.Reference, "@"); ok && name != "" && version != "" && out.Package == nil { out.Package = &chain.Package{Name: name, Version: version} } case out.Artifact == "": if _, digest, ok := strings.Cut(a.Reference, "@"); ok && strings.HasPrefix(digest, "sha256:") { out.Artifact = digest } } } if out.Artifact == "" && out.Package == nil { return nil } return &out } // chainBuildKey is a build's key. An id the bus takes no spelling of is refused by the write, loudly. func chainBuildKey(id string) string { return id } // chainRecordBuild keeps a build under its id. **Idempotent on the id**, as the table's insert was: a result arrives twice, // and the first record stands. A request nobody has answered is the record in state asked (issue 1091); the outcome // takes its place under the same key, and keeps the request it answers. func (i *Inventory) chainRecordBuild(ctx context.Context, c *chain.Chain, b Build) error { if b.At.IsZero() { b.At = time.Now().UTC() } key := chainBuildKey(b.ID) for try := 0; try < casTries; try++ { next := toChainBuild(b) cur, found, err := c.Builds.Get(ctx, key) if err != nil { return fmt.Errorf("build %s is not kept on the bus: %w", b.ID, err) } switch { case !found: _, err = c.Builds.Create(ctx, key, next) if errors.Is(err, chain.ErrExists) { continue } case cur.Value.State == chain.BuildAsked: next.Request = cur.Value.Request _, err = c.Builds.Update(ctx, key, next, cur.Revision) if errors.Is(err, chain.ErrMoved) { continue } default: // An outcome stands: the first record is the record. return i.RecordMirrored(ctx, b.ID, "", b.Mirrored) } if err != nil { return fmt.Errorf("build %s is not kept on the bus: %w", b.ID, err) } return i.RecordMirrored(ctx, b.ID, "", b.Mirrored) } return fmt.Errorf("build %s is not kept on the bus: it kept moving under the write", b.ID) } // casTries is how often a write that lost a compare-and-set reads the key again and tries once more. const casTries = 8 // chainBuilds is every build of the bucket, whole, in no order. func chainBuilds(ctx context.Context, c *chain.Chain) ([]Build, error) { all, err := c.Builds.All(ctx) if err != nil { return nil, fmt.Errorf("the builds cannot be read from the bus: %w", err) } out := make([]Build, 0, len(all)) for _, k := range all { // A request nobody has answered is no build: it has no outcome to read. if k.Value.State == chain.BuildAsked { continue } out = append(out, fromChainBuild(k.Value)) } return out, nil } // newestRequestOrder sorts builds as newestRequestFirst does: the newest request first, then the newest record. func newestRequestOrder(builds []Build) { sort.SliceStable(builds, func(a, b int) bool { x, y := builds[a].AskedOrAt(), builds[b].AskedOrAt() if !x.Equal(y) { return x.After(y) } if !builds[a].At.Equal(builds[b].At) { return builds[a].At.After(builds[b].At) } return builds[a].ID > builds[b].ID }) } // byModule sorts builds by module, then newest request first. func byModule(builds []Build) { newestRequestOrder(builds) sort.SliceStable(builds, func(a, b int) bool { return builds[a].Module < builds[b].Module }) } // worked is the builds that made something for a module: succeeded, and knew what they built. func worked(all []Build) []Build { var out []Build for _, b := range all { if b.Failed == "" && b.Module != "" { out = append(out, b) } } return out } func (i *Inventory) chainBuildList(ctx context.Context, c *chain.Chain, module string, limit int) ([]Build, error) { all, err := chainBuilds(ctx, c) if err != nil { return nil, err } var out []Build for _, b := range all { if module == "" || b.Module == module { out = append(out, Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On, Failed: b.Failed, Made: b.Made, Asked: b.Asked, At: b.At, ReuseKey: b.ReuseKey, Published: b.Published}) } } sort.SliceStable(out, func(a, b int) bool { if !out[a].At.Equal(out[b].At) { return out[a].At.After(out[b].At) } return out[a].ID > out[b].ID }) if len(out) > limit { out = out[:limit] } return out, nil } func (i *Inventory) chainBuildByID(ctx context.Context, c *chain.Chain, id string) (Build, bool, error) { k, ok, err := c.Builds.Get(ctx, chainBuildKey(id)) if err != nil || !ok || k.Value.State == chain.BuildAsked { // A request nobody has answered has no outcome to read. return Build{}, false, err } b := fromChainBuild(k.Value) return Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On, Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At}, true, nil } func (i *Inventory) chainBuiltAgainst(ctx context.Context, c *chain.Chain) (map[string][]string, error) { all, err := chainBuilds(ctx, c) if err != nil { return nil, err } ok := worked(all) byModule(ok) out := map[string][]string{} seen := map[string]bool{} for _, b := range ok { if seen[b.Module] { continue } seen[b.Module] = true if len(b.Against) > 0 { out[b.Module] = b.Against } } return out, nil } func (i *Inventory) chainReadRepositories(ctx context.Context, c *chain.Chain) (map[string][]ReadRepository, error) { all, err := chainBuilds(ctx, c) if err != nil { return nil, err } // The newest build of each module whatever its outcome: a failed one newer than the newest that worked leaves that // one's build source stale. type newestOf struct { at time.Time failed bool } newest := map[string]newestOf{} tried := make([]Build, 0, len(all)) for _, b := range all { if b.Module != "" { tried = append(tried, b) } } byModule(tried) for _, b := range tried { if _, seen := newest[b.Module]; !seen { newest[b.Module] = newestOf{b.AskedOrAt(), b.Failed != ""} } } owns, err := i.moduleSources(ctx) if err != nil { return nil, err } ok := worked(all) byModule(ok) read := map[string][]ReadRepository{} seen := map[string]bool{} for _, b := range ok { if seen[b.Module] { continue } seen[b.Module] = true built := b.AskedOrAt() of := append([]ReadRepository(nil), b.Read...) sources := b.Sources whole := "" if n, known := newest[b.Module]; known && n.failed && n.at.After(built) && len(sources) > 0 { sources, whole = nil, "a newer build of it failed" } of = WithBuildSources(of, sources) if own, has := owns[b.Module]; has { of = OwnRepositoryAsOwn(of, own) } if whole != "" { of = append(of, ReadRepository{Own: true, Whole: whole}) } if len(of) > 0 { for k := range of { of[k].Built, of[k].Looked = built, newest[b.Module].at } read[b.Module] = of } } return read, nil } // moduleSources is the repository each registered module is registered from. func (i *Inventory) moduleSources(ctx context.Context) (map[string]string, error) { rows, err := i.store.Pool().Query(ctx, `select name, source from module where source is not null`) if err != nil { return nil, err } defer rows.Close() out := map[string]string{} for rows.Next() { var name, source string if err := rows.Scan(&name, &source); err != nil { return nil, err } out[name] = source } return out, rows.Err() } func (i *Inventory) chainAnnounceable(ctx context.Context, c *chain.Chain) ([]Build, error) { all, err := chainBuilds(ctx, c) if err != nil { return nil, err } var ok []Build for _, b := range worked(all) { if b.Commit != "" { ok = append(ok, b) } } newestRequestOrder(ok) type key struct{ module, commit string } seen := map[key]bool{} var out []Build for _, b := range ok { k := key{b.Module, b.Commit} if seen[k] { continue } seen[k] = true out = append(out, Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On, Failed: b.Failed, Made: b.Made, Path: b.Path, Manifest: b.Manifest, Against: b.Against, At: b.At}) } sort.SliceStable(out, func(a, b int) bool { return out[a].At.Before(out[b].At) }) return out, nil } // failedAtAGate is the builds that failed their gate. The gate's verdicts stay in their table (issue 1097). func (i *Inventory) failedAtAGate(ctx context.Context) (map[string]bool, error) { rows, err := i.store.Pool().Query(ctx, `select build from build_gate where verdict = 'failed'`) if err != nil { return nil, err } defer rows.Close() out := map[string]bool{} for rows.Next() { var id string if err := rows.Scan(&id); err != nil { return nil, err } out[id] = true } return out, rows.Err() } func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module string) (map[string][]sourceRow, []string, error) { all, err := chainBuilds(ctx, c) if err != nil { return nil, nil, err } failed, err := i.failedAtAGate(ctx) if err != nil { return nil, nil, err } ok := worked(all) byModule(ok) out := map[string][]sourceRow{} var order []string for _, b := range ok { if module != "" && b.Module != module { continue } made, err := json.Marshal(b.Made) if err != nil { return nil, nil, err } if _, seen := out[b.Module]; !seen { order = append(order, b.Module) } out[b.Module] = append(out[b.Module], sourceRow{id: b.ID, commit: b.Commit, fingerprint: b.SourceFingerprint, made: made, manifest: b.Manifest, failedGate: failed[b.ID]}) } return out, order, nil } // canonical is a JSON document encoded the one way (objects' keys sorted, compact), so the same content hashes alike // however the bucket's record spells it. func canonicalJSON(raw []byte) []byte { if len(bytes.TrimSpace(raw)) == 0 { return nil } out, err := canonical(raw) if err != nil { return raw } return out } // chainFingerprint is what a build would put on a machine, hashed: its artifacts and its manifest. func chainFingerprint(b Build) string { made, _ := json.Marshal(b.Made) sum := sha256.Sum256(append(append(canonicalJSON(made), 0), canonicalJSON(b.Manifest)...)) return hex.EncodeToString(sum[:]) } func (i *Inventory) chainBuildFingerprints(ctx context.Context, c *chain.Chain, module string) (map[string]string, error) { all, err := chainBuilds(ctx, c) if err != nil { return nil, err } var ok []Build for _, b := range all { if b.Module == module && b.Failed == "" && b.Commit != "" { ok = append(ok, b) } } sortByRecorded(ok) if len(ok) > 50 { ok = ok[:50] } out := map[string]string{} for _, b := range ok { if _, seen := out[b.Commit]; !seen { out[b.Commit] = chainFingerprint(b) } } return out, nil } func (i *Inventory) chainFingerprints(ctx context.Context, c *chain.Chain) (map[string]map[string]string, error) { all, err := chainBuilds(ctx, c) if err != nil { return nil, err } var ok []Build for _, b := range all { if b.Module != "" && b.Failed == "" && b.Commit != "" { ok = append(ok, b) } } sortByRecorded(ok) out := map[string]map[string]string{} for _, b := range ok { if out[b.Module] == nil { out[b.Module] = map[string]string{} } if _, seen := out[b.Module][b.Commit]; !seen { out[b.Module][b.Commit] = chainFingerprint(b) } } return out, nil } // sortByRecorded is newest record first, as `order by at desc`. func sortByRecorded(builds []Build) { sort.SliceStable(builds, func(a, b int) bool { if !builds[a].At.Equal(builds[b].At) { return builds[a].At.After(builds[b].At) } return builds[a].ID > builds[b].ID }) } func (i *Inventory) chainBuildOf(ctx context.Context, c *chain.Chain, module, commit string) (string, error) { all, err := chainBuilds(ctx, c) if err != nil { return "", err } var ok []Build for _, b := range all { if b.Module == module && b.Commit == commit && b.Failed == "" { ok = append(ok, b) } } sortByRecorded(ok) if len(ok) == 0 { return "", nil } return ok[0].ID, nil } // chainManifestBuild is the newest successful build of a module from a commit (or one it abbreviates) that kept a manifest. func (i *Inventory) chainManifestBuild(ctx context.Context, c *chain.Chain, module, commit string) (string, []byte, bool, error) { all, err := chainBuilds(ctx, c) if err != nil { return "", nil, false, err } var ok []Build for _, b := range all { if b.Module == module && b.Failed == "" && len(b.Manifest) > 0 && string(bytes.TrimSpace(b.Manifest)) != "null" && (b.Commit == commit || strings.HasPrefix(b.Commit, commit)) { ok = append(ok, b) } } sortByRecorded(ok) if len(ok) == 0 { return "", nil, false, nil } return ok[0].ID, ok[0].Manifest, true, nil } // chainManifestOf is the manifest a build was recorded with. func (i *Inventory) chainManifestOf(ctx context.Context, c *chain.Chain, id string) ([]byte, error) { k, ok, err := c.Builds.Get(ctx, chainBuildKey(id)) if err != nil || !ok { return nil, err } return []byte(k.Value.Manifest), nil } // chainRecentMade is the artifacts and bases of the KeptBuilds newest successful builds of each registered module. func (i *Inventory) chainRecentMade(ctx context.Context, c *chain.Chain) ([]Build, error) { all, err := chainBuilds(ctx, c) if err != nil { return nil, err } rows, err := i.store.Pool().Query(ctx, `select name from module`) if err != nil { return nil, err } registered := map[string]bool{} for rows.Next() { var name string if err := rows.Scan(&name); err != nil { rows.Close() return nil, err } registered[name] = true } rows.Close() if err := rows.Err(); err != nil { return nil, err } var ok []Build for _, b := range worked(all) { if registered[b.Module] { ok = append(ok, b) } } sortByRecorded(ok) back := map[string]int{} var out []Build for _, b := range ok { back[b.Module]++ if back[b.Module] <= KeptBuilds { out = append(out, b) } } return out, nil } // chainEveryReferenceMade is every successful build's artifacts, oldest build first. func (i *Inventory) chainEveryReferenceMade(ctx context.Context, c *chain.Chain) ([]Build, error) { all, err := chainBuilds(ctx, c) if err != nil { return nil, err } ok := worked(all) sort.SliceStable(ok, func(a, b int) bool { if !ok[a].At.Equal(ok[b].At) { return ok[a].At.Before(ok[b].At) } return ok[a].ID < ok[b].ID }) return ok, nil }