Take a walk's modules from the builder's done keys and ask no build but the builder's own: a key building or unreferenced holds the tier and says so, a failed key fails the walk in its words (issue 1851, A6, A7)
This commit is contained in:
@@ -0,0 +1,268 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A walk takes its modules from the builder's done keys (novox/hq ADR 0503 §6, ADR 0501 §1, ADR 0498 §8; issue 1851, A6
|
||||
// and A7).
|
||||
//
|
||||
// From R4e a walk asks build-agent for no module but the builder's own (ADR 0453 §5). A module of its tier is marked
|
||||
// taken from a key (PlanModule.Key) and settled by the look: its fingerprint is read from the indexer's trunk value
|
||||
// written for the commit the walk carries of its repository; the builder's key of that fingerprint, done with every
|
||||
// artifact referenced, is taken in (recorded and registered, off the plans' hold) and the module is built; a key not done
|
||||
// yet, or waiting for a reference, holds the tier and says so; a failed key fails the walk in the builder's own words.
|
||||
// **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules).
|
||||
|
||||
// isBuildersOwn says a module is the builder's own: the one module build-agent still builds, asked on a merge, until R4g
|
||||
// (ADR 0453 §5, ADR 0498 §7).
|
||||
func isBuildersOwn(module string) bool { return buildersOwn(module) }
|
||||
|
||||
// buildersOwn is isBuildersOwn's rule: a variable only so a test of the one ask left (its bus waits, its plan key) may
|
||||
// name modules of its own as the builder's.
|
||||
var buildersOwn = func(module string) bool { return module == sdkdelivery.BuilderModule }
|
||||
|
||||
// planKeyReads is what a walk reads of the indexer and the builder (builderkeys.Reader, or a test's).
|
||||
type planKeyReads interface {
|
||||
ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error)
|
||||
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
|
||||
}
|
||||
|
||||
// openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus.
|
||||
var openPlanKeys = func() (planKeyReads, func(), error) {
|
||||
r, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return r, closeKeys, nil
|
||||
}
|
||||
|
||||
// takeFromKey marks a module of a walk's tier as taken from the builder's key: asked of nobody (ADR 0501 §6).
|
||||
func takeFromKey(p *inventory.Plan, state *inventory.PlanModule, e inventory.Entry, now time.Time) {
|
||||
state.BusWaitFrom = nil
|
||||
state.State = "asked"
|
||||
state.AskedAt = &now
|
||||
state.Key = true
|
||||
state.Build, state.Fingerprint = "", ""
|
||||
state.KeyAt = p.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref))
|
||||
state.Why = "waits for the builder's key"
|
||||
state.Commit, state.BuiltAt, state.Made = "", nil, ""
|
||||
fmt.Printf(" tier %d: %s is taken from the builder's key, asking no build\n", p.Tier, e.Manifest.Module)
|
||||
}
|
||||
|
||||
// keyTakes are the take-ins of keys a look started and has not seen end, by key build id, and what each said when it
|
||||
// waits: kept in this process, so a look starts each once and a settle says why it waits.
|
||||
var keyTakes = struct {
|
||||
sync.Mutex
|
||||
running map[string]bool
|
||||
waits map[string]string
|
||||
// taken are the keys whose take-in ended registered, or found the module already at a later build (superseded).
|
||||
taken map[string]bool
|
||||
}{running: map[string]bool{}, waits: map[string]string{}, taken: map[string]bool{}}
|
||||
|
||||
// settleFromKeys settles every module of the tier taken from a key: built from its key's record, failed with the key's
|
||||
// words or the take-in's refusal, or still waiting, saying for what. It says whether it changed the plan. It reads only
|
||||
// the buckets and the records; a done key not recorded yet is taken in off the plans' hold, and its end asks the look.
|
||||
func settleFromKeys(ctx context.Context, open *stores, keys planKeyReads, p *inventory.Plan, tier []string) (bool, error) {
|
||||
return settleKeys(ctx, open.inventory, keys, p, tier, func(module string, trunk sdkdelivery.Module, id string) {
|
||||
go takeKeyOffTheHold(context.WithoutCancel(ctx), open, module, trunk, id)
|
||||
})
|
||||
}
|
||||
|
||||
// keyRecords is what settling from keys reads of the records (*inventory.Inventory, or a test's).
|
||||
type keyRecords interface {
|
||||
BuildByID(ctx context.Context, id string) (inventory.Build, bool, error)
|
||||
RefusedOf(ctx context.Context, ids []string) (map[string]string, error)
|
||||
SourceOf(ctx context.Context, module string) (inventory.Source, error)
|
||||
}
|
||||
|
||||
// settleKeys is settleFromKeys on the records handed in, starting a take-in with take.
|
||||
func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inventory.Plan, tier []string,
|
||||
take func(module string, trunk sdkdelivery.Module, id string)) (bool, error) {
|
||||
changed := false
|
||||
set := func(s *inventory.PlanModule, why string) {
|
||||
if s.Why != why {
|
||||
s.Why = why
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil || !s.Key || s.State != "asked" {
|
||||
continue
|
||||
}
|
||||
trunk, err := keys.ModuleAt(ctx, m, s.KeyAt)
|
||||
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||
set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt)))
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if trunk.FP2 == "" {
|
||||
why := trunk.None2
|
||||
if why == "" {
|
||||
why = "the indexer wrote no fingerprint of the new form"
|
||||
}
|
||||
set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why))
|
||||
continue
|
||||
}
|
||||
if s.Fingerprint != trunk.FP2 {
|
||||
s.Fingerprint, changed = trunk.FP2, true
|
||||
}
|
||||
k, err := keys.Build(ctx, m, trunk.FP2)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
switch {
|
||||
case k.Failed():
|
||||
// **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`,
|
||||
// `version taken:`), and a rebuild of it is a new revision the next walk takes.
|
||||
s.State, s.Why = "failed", k.Build.Error
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(k.Build.Error))
|
||||
changed = true
|
||||
continue
|
||||
case !k.Done() || len(k.Unreferenced()) > 0:
|
||||
set(s, "waits: "+k.Says())
|
||||
continue
|
||||
}
|
||||
id := keyBuildID(k)
|
||||
refused, err := inv.RefusedOf(ctx, []string{id})
|
||||
if err != nil {
|
||||
return changed, fmt.Errorf("the refused registrations cannot be read: %w", err)
|
||||
}
|
||||
for kept, why := range refusalsKeptFor([]string{id}) {
|
||||
refused[kept] = why
|
||||
}
|
||||
if why, ok := refused[id]; ok {
|
||||
s.State, s.Why, s.Build = "failed", why, id
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(why))
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
b, found, err := inv.BuildByID(ctx, id)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if found && b.Worked() {
|
||||
keyTakes.Lock()
|
||||
taken := keyTakes.taken[id]
|
||||
keyTakes.Unlock()
|
||||
registered := taken
|
||||
if !registered {
|
||||
if registered, err = registeredFrom(ctx, inv, m, b); err != nil {
|
||||
return changed, err
|
||||
}
|
||||
}
|
||||
if registered {
|
||||
at := b.At
|
||||
s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, b.Commit, id
|
||||
s.Made = fmt.Sprintf("taken from the builder's key %s (revision %d), no build asked", short(trunk.FP2), k.Revision)
|
||||
s.Why = ""
|
||||
fmt.Printf("%s: %s taken from the builder's key %s\n", p.ID, m, short(trunk.FP2))
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
keyTakes.Lock()
|
||||
running, waits := keyTakes.running[id], keyTakes.waits[id]
|
||||
if !running {
|
||||
keyTakes.running[id] = true
|
||||
}
|
||||
keyTakes.Unlock()
|
||||
if waits != "" {
|
||||
set(s, firstLine(waits))
|
||||
} else {
|
||||
set(s, "taking in the builder's done key "+short(trunk.FP2))
|
||||
}
|
||||
if !running {
|
||||
take(m, trunk, id)
|
||||
}
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// registeredFrom says the module's registration stands on the build: the catalogue holds the module at the build's
|
||||
// commit, or the build was registered as one with an unchanged source (no move).
|
||||
func registeredFrom(ctx context.Context, inv keyRecords, module string, b inventory.Build) (bool, error) {
|
||||
src, err := inv.SourceOf(ctx, module)
|
||||
if errors.Is(err, inventory.ErrNoSuchModule) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return sameCommitID(src.BuiltFrom, b.Commit), nil
|
||||
}
|
||||
|
||||
func sameCommitID(a, b string) bool {
|
||||
if a == "" || b == "" {
|
||||
return false
|
||||
}
|
||||
n := min(len(a), len(b))
|
||||
return n >= 7 && a[:n] == b[:n]
|
||||
}
|
||||
|
||||
func orNow(commit string) string {
|
||||
if commit == "" {
|
||||
return "its value now"
|
||||
}
|
||||
return short(commit)
|
||||
}
|
||||
|
||||
// takeKeyOffTheHold takes a walk's module in from its done key, off the plans' hold: a registration may sweep the
|
||||
// artifact store, and the plans are held by one look at a time. Its end asks the look, which settles the module from the
|
||||
// record; a refusal is kept by the key's build id, so the look fails the walk with it; a wait is said by the look.
|
||||
func takeKeyOffTheHold(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) {
|
||||
defer func() {
|
||||
keyTakes.Lock()
|
||||
delete(keyTakes.running, id)
|
||||
keyTakes.Unlock()
|
||||
askToLook()
|
||||
}()
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
keepKeyWait(id, "waits: the builder's keys cannot be read: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer closeKeys()
|
||||
_, _, err = takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk,
|
||||
Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
switch {
|
||||
case err == nil, errors.Is(err, inventory.ErrSuperseded):
|
||||
// Superseded is no failure: the module is already at what a later merge's key built, which a walk that carries
|
||||
// this merge is answered by, as a build-agent outcome superseded was.
|
||||
keepKeyWait(id, "")
|
||||
keyTakes.Lock()
|
||||
keyTakes.taken[id] = true
|
||||
keyTakes.Unlock()
|
||||
case errors.Is(err, errKeyWaits):
|
||||
keepKeyWait(id, err.Error())
|
||||
default:
|
||||
why := err.Error()
|
||||
if rerr := open.inventory.RecordRefused(ctx, id, module, why); rerr != nil {
|
||||
fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, rerr)
|
||||
}
|
||||
keepRefusalFor(id, why, time.Now())
|
||||
}
|
||||
}
|
||||
|
||||
func keepKeyWait(id, why string) {
|
||||
keyTakes.Lock()
|
||||
defer keyTakes.Unlock()
|
||||
if why == "" {
|
||||
delete(keyTakes.waits, id)
|
||||
return
|
||||
}
|
||||
keyTakes.waits[id] = why
|
||||
}
|
||||
Reference in New Issue
Block a user