package inventory import ( "context" "encoding/json" "errors" "fmt" "time" "github.com/jackc/pgx/v5" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/chain" ) // **The move** (novox/hq ADR 0408 §8, issue 981 note 7 Q2): the serving controller writes the rows of a kind into its // bucket at its start, holding the lease and after the buckets exist, and records the kind in chain_moved. Idempotent: a // key that exists is already moved and is never overwritten, so a bucket value written since is not laid over by the row // it came from. It counts the rows against the keys, logs the largest value, and **refuses** (a Refusal, which the // controller raises as an urgent condition) on a count that differs or a value over the bus's largest, with nothing // switched: a kind is recorded moved only once all of it is in the bucket. // Refusal is a move that stopped without switching a kind, and why. It is permanent until the cause is put right; any // other error of a move (the bus away, a write not granted) is tried again. type Refusal struct { Kind, Why string } func (r *Refusal) Error() string { return fmt.Sprintf("the move of %s to the bus is refused: %s", r.Kind, r.Why) } // MoveReport is what a move of one kind did. type MoveReport struct { Kind string // Rows is how many records the tables held, Written how many keys this move made, and Present how many already // held a value (moved by an earlier start). Rows, Written, Present int // Largest is the biggest value, in bytes. Largest int // Already is a kind an earlier start moved: nothing was done. Already bool } // mover is one kind of the chain: its rows, read from its tables, as the keys and values they become. type mover struct { kind string // rows reads the tables: each record's key and its value, whole. rows func(ctx context.Context) ([]moveRow, error) // create writes one record under its key; chain.ErrExists when the key holds a value. create func(ctx context.Context, c *chain.Chain, row moveRow) error // keys are the keys the bucket holds. keys func(ctx context.Context, c *chain.Chain) ([]string, error) // count is how many rows the tables hold now. count func(ctx context.Context, tx pgx.Tx) (int, error) // lock keeps the tables from taking a row while the switch is recorded. lock string } type moveRow struct { key string value any size int } func (i *Inventory) movers() []mover { return []mover{{ kind: KindBuilds, rows: func(ctx context.Context) ([]moveRow, error) { builds, err := i.allBuildRows(ctx) if err != nil { return nil, err } out := make([]moveRow, 0, len(builds)) for _, b := range builds { v := toChainBuild(b) body, err := json.Marshal(v) if err != nil { return nil, err } out = append(out, moveRow{key: chainBuildKey(b.ID), value: v, size: len(body)}) } return out, nil }, create: func(ctx context.Context, c *chain.Chain, row moveRow) error { _, err := c.Builds.Create(ctx, row.key, row.value.(chain.Build)) return err }, keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Builds.Keys(ctx) }, count: func(ctx context.Context, tx pgx.Tx) (int, error) { var n int err := tx.QueryRow(ctx, `select count(*) from build`).Scan(&n) return n, err }, lock: `lock table build in share mode`, }} } // MoveChain moves every kind not moved yet, in order, and says what it did. It needs the chain attached and the lease // held (the chain's writes ask). It stops at the first kind that fails. func (i *Inventory) MoveChain(ctx context.Context, say func(format string, args ...any)) ([]MoveReport, error) { if say == nil { say = func(string, ...any) {} } if i.attached() == nil { return nil, ErrNoChain } var out []MoveReport for _, m := range i.movers() { r, err := i.moveKind(ctx, m, say) if err != nil { return out, err } out = append(out, r) } return out, nil } // movePasses is how often a move reads the tables again when they took a row while it wrote: the controller before this // one may still have been writing as the lease changed hands. const movePasses = 5 func (i *Inventory) moveKind(ctx context.Context, m mover, say func(string, ...any)) (MoveReport, error) { report := MoveReport{Kind: m.kind} moved, err := i.ChainMoved(ctx) if err != nil { return report, err } if _, done := moved[m.kind]; done { report.Already = true return report, nil } for pass := 1; pass <= movePasses; pass++ { rows, err := m.rows(ctx) if err != nil { return report, fmt.Errorf("the %s cannot be read from the tables: %w", m.kind, err) } report.Rows, report.Largest = len(rows), 0 // Every value measured before any is written: a value over the bus's largest stops the move with nothing // written (ADR 0408 §6), never cut. for _, r := range rows { report.Largest = max(report.Largest, r.size) if r.size > broker.ChainValueBytes { return report, &Refusal{m.kind, fmt.Sprintf("the value of %s is %d bytes and the bus holds at most %d; "+ "nothing is cut, so the record must be made smaller first", r.key, r.size, broker.ChainValueBytes)} } } report.Written, report.Present = 0, 0 for _, r := range rows { switch err := m.create(ctx, i.attached(), r); { case err == nil: report.Written++ case errors.Is(err, chain.ErrExists): report.Present++ default: return report, fmt.Errorf("%s %s is not written to the bus: %w", m.kind, r.key, err) } } // Rows against keys: every row's key is in the bucket. keys, err := m.keys(ctx, i.attached()) if err != nil { return report, fmt.Errorf("the keys of the %s cannot be read back: %w", m.kind, err) } have := make(map[string]bool, len(keys)) for _, k := range keys { have[k] = true } for _, r := range rows { if !have[r.key] { return report, &Refusal{m.kind, fmt.Sprintf("the tables hold %d records and the bucket does not hold %s", len(rows), r.key)} } } settled, err := i.switchKind(ctx, m, report) if err != nil { return report, err } if settled { say("moved the %s to the bus: %d records, %d written now and %d already there, the largest %d bytes", m.kind, report.Rows, report.Written, report.Present, report.Largest) return report, nil } say("the %s took a row while they were moved, so they are read again (pass %d of %d)", m.kind, pass, movePasses) } return report, &Refusal{m.kind, fmt.Sprintf("the tables kept taking rows through %d passes; another controller is still writing them", movePasses)} } // switchKind records the kind as moved, in one transaction that holds the tables against writes and finds their count // as the move read it: false when they took a row since. func (i *Inventory) switchKind(ctx context.Context, m mover, r MoveReport) (bool, error) { tx, err := i.store.Pool().Begin(ctx) if err != nil { return false, err } defer func() { _ = tx.Rollback(ctx) }() if _, err := tx.Exec(ctx, m.lock); err != nil { return false, err } n, err := m.count(ctx, tx) if err != nil { return false, err } if n != r.Rows { return false, nil } counts, _ := json.Marshal(map[string]int{m.kind: r.Rows}) largest, _ := json.Marshal(map[string]int{m.kind: r.Largest}) if _, err := tx.Exec(ctx, `insert into chain_moved (counts, largest) values ($1, $2) on conflict (only_one) do update set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest, moved_at = now()`, counts, largest); err != nil { return false, err } if err := tx.Commit(ctx); err != nil { return false, err } i.moved.mu.Lock() if i.moved.kind == nil { i.moved.kind = map[string]bool{} } i.moved.kind[m.kind] = true i.moved.mu.Unlock() return true, nil } // allBuildRows is every build of the table, whole. func (i *Inventory) allBuildRows(ctx context.Context) ([]Build, error) { rows, err := i.store.Pool().Query(ctx, `select id, repository, ref, coalesce(module, ''), commit_hash, built_on, failed, made, source_path, manifest, built_against, built_contexts, build_sources, source_fingerprint, asked, at from build order by at, id`) if err != nil { return nil, err } defer rows.Close() var out []Build for rows.Next() { var b Build var made, manifest, against, read, sources []byte var asked *time.Time if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path, &manifest, &against, &read, &sources, &b.SourceFingerprint, &asked, &b.At); err != nil { return nil, err } if asked != nil { b.Asked = *asked } b.Manifest = manifest for _, part := range []struct { raw []byte to any who string }{{made, &b.Made, "made"}, {against, &b.Against, "built_against"}, {read, &b.Read, "built_contexts"}, {sources, &b.Sources, "build_sources"}} { if len(part.raw) == 0 { continue } if err := json.Unmarshal(part.raw, part.to); err != nil { return nil, fmt.Errorf("build %s: %s is not readable: %w", b.ID, part.who, err) } } out = append(out, b) } return out, rows.Err() }