jschoubben
1414ed7f7b
A second read that failed, or that turned true in between, refused the ask and failed the whole plan.
337 lines
13 KiB
Go
337 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
|
|
"github.com/novox/mesh-controller/internal/builder"
|
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// A walk takes its modules from the builder's done keys (novox/hq ADR 0503 §6, ADR 0501 §1, ADR 0498 §8; issue 1851, A6
|
|
// and A7).
|
|
//
|
|
// From R4e a walk asks build-agent for no module but the builder's own (ADR 0453 §5). A module of its tier is marked
|
|
// taken from a key (PlanModule.Key) and settled by the look: its fingerprint is read from the indexer's trunk value
|
|
// written for the commit the walk carries of its repository; the builder's key of that fingerprint, done with every
|
|
// artifact referenced, is taken in (recorded and registered, off the plans' hold) and the module is built; a key not done
|
|
// yet, or waiting for a reference, holds the tier and says so; a failed key fails the walk in the builder's own words.
|
|
// **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules).
|
|
|
|
// isBuildersOwn says a module is build-agent's own: asked of build-agent on a merge and taken from its build record, never
|
|
// from the builder's key, until R4g (ADR 0453 §5, ADR 0498 §7, ADR 0520 §1).
|
|
func isBuildersOwn(module string) bool { return buildersOwn(module) }
|
|
|
|
// buildAgentsOwn are those modules, named (ADR 0520 §1): the builder's; build-agent itself, whose image's context is another
|
|
// repository's; and mesh-tools, the toolchain image build-agent's builds stand on. The builder writes no key of any of them,
|
|
// and a walk waiting for one would hold its tier for ever. mesh-tools-go is not among them: it builds from a Dockerfile in
|
|
// its own directory on a pinned base, and has its key from mesh-build-image.
|
|
//
|
|
// **And the parts that write references** (ADR 0522 §6, issue 1886): the build modules push and answer each reference,
|
|
// and the builder writes it into the key, so none of them is ever taken from a key they write themselves. Until R4g
|
|
// build-agent is the way they are delivered whatever the builder says.
|
|
var buildAgentsOwn = map[string]bool{sdkdelivery.BuilderModule: true, builder.AgentModule: true, "mesh-tools": true,
|
|
"mesh-build-go": true, "mesh-build-node": true, "mesh-build-image": true}
|
|
|
|
// buildersOwn is isBuildersOwn's rule: a variable only so a test of the asks left (their bus waits, their plan key) may
|
|
// name modules of its own as build-agent's.
|
|
var buildersOwn = func(module string) bool { return buildAgentsOwn[module] }
|
|
|
|
// planKeyReads is what a walk reads of the indexer and the builder (builderkeys.Reader, or a test's).
|
|
type planKeyReads interface {
|
|
ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error)
|
|
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
|
|
// SaysReferences is whether a running builder says it writes references (builderkeys.Reader.SaysReferences).
|
|
SaysReferences(ctx context.Context) (bool, error)
|
|
}
|
|
|
|
// builderSayKey carries, on an ask's context, the builder's say the walk read before asking (withBuilderSay).
|
|
type builderSayKey struct{}
|
|
|
|
// withBuilderSay hands the say a walk read to the ask's guard, so the guard never reads it a second time.
|
|
func withBuilderSay(ctx context.Context, says bool) context.Context {
|
|
return context.WithValue(ctx, builderSayKey{}, says)
|
|
}
|
|
|
|
// builderSayOf is the say a walk handed the ask, and whether it handed one.
|
|
func builderSayOf(ctx context.Context) (says, handed bool) {
|
|
says, handed = ctx.Value(builderSayKey{}).(bool)
|
|
return says, handed
|
|
}
|
|
|
|
// builderSaysReferences is whether a running builder instance says, in its own bucket of instances, that the keys it
|
|
// writes name references (issue 1886). Until one does, a walk asks build-agent for every module, as before R4e: the
|
|
// keys the running builder writes could never be delivered from. A variable so a test needs no bus.
|
|
var builderSaysReferences = func(ctx context.Context) (bool, error) {
|
|
keys, closeKeys, err := openBuilderKeys()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
defer closeKeys()
|
|
return keys.SaysReferences(ctx)
|
|
}
|
|
|
|
// openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus.
|
|
var openPlanKeys = func() (planKeyReads, func(), error) {
|
|
r, closeKeys, err := openBuilderKeys()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return r, closeKeys, nil
|
|
}
|
|
|
|
// takeFromKey marks a module of a walk's tier as taken from the builder's key: asked of nobody (ADR 0501 §6).
|
|
func takeFromKey(p *inventory.Plan, state *inventory.PlanModule, e inventory.Entry, now time.Time) {
|
|
state.BusWaitFrom = nil
|
|
state.State = "asked"
|
|
state.AskedAt = &now
|
|
state.Key = true
|
|
state.Build, state.Fingerprint = "", ""
|
|
state.KeyAt = p.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref))
|
|
state.Why = "waits for the builder's key"
|
|
state.Commit, state.BuiltAt, state.Made = "", nil, ""
|
|
fmt.Printf(" tier %d: %s is taken from the builder's key, asking no build\n", p.Tier, e.Manifest.Module)
|
|
}
|
|
|
|
// keyTakes are the take-ins of keys a look started and has not seen end, by key build id, and what each said when it
|
|
// waits: kept in this process, so a look starts each once and a settle says why it waits.
|
|
var keyTakes = struct {
|
|
sync.Mutex
|
|
running map[string]bool
|
|
waits map[string]string
|
|
// taken are the keys whose take-in ended registered, or found the module already at a later build (superseded).
|
|
taken map[string]bool
|
|
}{running: map[string]bool{}, waits: map[string]string{}, taken: map[string]bool{}}
|
|
|
|
// settleFromKeys settles every module of the tier taken from a key: built from its key's record, failed with the key's
|
|
// words or the take-in's refusal, or still waiting, saying for what. It says whether it changed the plan. It reads only
|
|
// the buckets and the records; a done key not recorded yet is taken in off the plans' hold, and its end asks the look.
|
|
func settleFromKeys(ctx context.Context, open *stores, keys planKeyReads, p *inventory.Plan, tier []string) (bool, error) {
|
|
return settleKeys(ctx, open.inventory, keys, p, tier, func(module string, trunk sdkdelivery.Module, id string) {
|
|
inBackground(func() { takeKeyOffTheHold(context.WithoutCancel(ctx), open, module, trunk, id) })
|
|
})
|
|
}
|
|
|
|
// keyRecords is what settling from keys reads of the records (*inventory.Inventory, or a test's).
|
|
type keyRecords interface {
|
|
BuildByID(ctx context.Context, id string) (inventory.Build, bool, error)
|
|
RefusedOf(ctx context.Context, ids []string) (map[string]string, error)
|
|
SourceOf(ctx context.Context, module string) (inventory.Source, error)
|
|
}
|
|
|
|
// settleKeys is settleFromKeys on the records handed in, starting a take-in with take.
|
|
func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inventory.Plan, tier []string,
|
|
take func(module string, trunk sdkdelivery.Module, id string)) (bool, error) {
|
|
changed := false
|
|
set := func(s *inventory.PlanModule, why string) {
|
|
if s.Why != why {
|
|
s.Why = why
|
|
changed = true
|
|
}
|
|
}
|
|
says, err := keys.SaysReferences(ctx)
|
|
if err != nil {
|
|
return changed, fmt.Errorf("whether the builder writes references cannot be read: %w", err)
|
|
}
|
|
// **A walk never waits on a key the running builder cannot reference** (issue 1886): while no builder says it writes
|
|
// references, a module that waits on its key, or whose key failed, is handed back unasked, and the look asks it of
|
|
// build-agent. A done key with every reference is still taken.
|
|
toAgent := func(s *inventory.PlanModule, why string) {
|
|
s.State, s.Key, s.AskedAt, s.KeyAt, s.Build, s.Fingerprint = "", false, nil, "", "", ""
|
|
s.Why = "the builder does not say it writes references, so build-agent is asked (" + why + ")"
|
|
changed = true
|
|
}
|
|
for _, m := range tier {
|
|
s := p.Modules[m]
|
|
if s == nil || !s.Key || s.State != "asked" {
|
|
continue
|
|
}
|
|
trunk, err := keys.ModuleAt(ctx, m, s.KeyAt)
|
|
if errors.Is(err, builderkeys.ErrNoValue) {
|
|
if !says {
|
|
toAgent(s, "the indexer holds no value of it")
|
|
continue
|
|
}
|
|
set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt)))
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return changed, err
|
|
}
|
|
if trunk.FP2 == "" {
|
|
why := trunk.None2
|
|
if why == "" {
|
|
why = "the indexer wrote no fingerprint of the new form"
|
|
}
|
|
if !says {
|
|
toAgent(s, why)
|
|
continue
|
|
}
|
|
set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why))
|
|
continue
|
|
}
|
|
if s.Fingerprint != trunk.FP2 {
|
|
s.Fingerprint, changed = trunk.FP2, true
|
|
}
|
|
k, err := keys.Build(ctx, m, trunk.FP2)
|
|
if err != nil {
|
|
return changed, err
|
|
}
|
|
switch {
|
|
case !says && (k.Failed() || !k.Done() || len(k.Unreferenced()) > 0):
|
|
toAgent(s, k.Says())
|
|
continue
|
|
case k.Failed():
|
|
// **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`,
|
|
// `version taken:`), and a rebuild of it is a new revision the next walk takes.
|
|
s.State, s.Why = "failed", k.Build.Error
|
|
p.State = inventory.PlanFailed
|
|
p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(k.Build.Error))
|
|
changed = true
|
|
continue
|
|
case !k.Done() || len(k.Unreferenced()) > 0:
|
|
set(s, "waits: "+k.Says())
|
|
continue
|
|
}
|
|
id := keyBuildID(k)
|
|
refused, err := inv.RefusedOf(ctx, []string{id})
|
|
if err != nil {
|
|
return changed, fmt.Errorf("the refused registrations cannot be read: %w", err)
|
|
}
|
|
for kept, why := range refusalsKeptFor([]string{id}) {
|
|
refused[kept] = why
|
|
}
|
|
if why, ok := refused[id]; ok {
|
|
s.State, s.Why, s.Build = "failed", why, id
|
|
p.State = inventory.PlanFailed
|
|
p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(why))
|
|
changed = true
|
|
continue
|
|
}
|
|
b, found, err := inv.BuildByID(ctx, id)
|
|
if err != nil {
|
|
return changed, err
|
|
}
|
|
if found && b.Worked() {
|
|
keyTakes.Lock()
|
|
taken := keyTakes.taken[id]
|
|
keyTakes.Unlock()
|
|
registered := taken
|
|
if !registered {
|
|
if registered, err = registeredAt(ctx, inv, m, trunk.Commit); err != nil {
|
|
return changed, err
|
|
}
|
|
}
|
|
if registered {
|
|
at := b.At
|
|
s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, trunk.Commit, id
|
|
s.Made = fmt.Sprintf("taken from the builder's key %s (revision %d), no build asked", short(trunk.FP2), k.Revision)
|
|
s.Why = ""
|
|
fmt.Printf("%s: %s taken from the builder's key %s\n", p.ID, m, short(trunk.FP2))
|
|
changed = true
|
|
continue
|
|
}
|
|
}
|
|
keyTakes.Lock()
|
|
running, waits := keyTakes.running[id], keyTakes.waits[id]
|
|
if !running {
|
|
keyTakes.running[id] = true
|
|
}
|
|
keyTakes.Unlock()
|
|
if waits != "" {
|
|
set(s, firstLine(waits))
|
|
} else {
|
|
set(s, "taking in the builder's done key "+short(trunk.FP2))
|
|
}
|
|
if !running {
|
|
take(m, trunk, id)
|
|
}
|
|
}
|
|
return changed, nil
|
|
}
|
|
|
|
// registeredAt says the catalogue holds the module registered at the commit the walk takes it at: its key's take-in
|
|
// registered it there, whether with the key's artifacts or, unchanged, with the build it stands on (no move). A key
|
|
// recorded at an earlier commit (the same fingerprint at an older merge) is taken in again, to register it here.
|
|
func registeredAt(ctx context.Context, inv keyRecords, module, commit string) (bool, error) {
|
|
src, err := inv.SourceOf(ctx, module)
|
|
if errors.Is(err, inventory.ErrNoSuchModule) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return sameCommitID(src.BuiltFrom, commit), nil
|
|
}
|
|
|
|
func sameCommitID(a, b string) bool {
|
|
if a == "" || b == "" {
|
|
return false
|
|
}
|
|
if a == b {
|
|
return true
|
|
}
|
|
n := min(len(a), len(b))
|
|
return n >= 7 && a[:n] == b[:n]
|
|
}
|
|
|
|
func orNow(commit string) string {
|
|
if commit == "" {
|
|
return "its value now"
|
|
}
|
|
return short(commit)
|
|
}
|
|
|
|
// takeKeyOffTheHold takes a walk's module in from its done key, off the plans' hold: a registration may sweep the
|
|
// artifact store, and the plans are held by one look at a time. Its end asks the look, which settles the module from the
|
|
// record; a refusal is kept by the key's build id, so the look fails the walk with it; a wait is said by the look.
|
|
func takeKeyOffTheHold(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) {
|
|
defer func() {
|
|
keyTakes.Lock()
|
|
delete(keyTakes.running, id)
|
|
keyTakes.Unlock()
|
|
askToLook()
|
|
}()
|
|
keys, closeKeys, err := openBuilderKeys()
|
|
if err != nil {
|
|
keepKeyWait(id, "waits: the builder's keys cannot be read: "+err.Error())
|
|
return
|
|
}
|
|
defer closeKeys()
|
|
_, _, err = takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk,
|
|
Ask: registrationAsk{Merge: trunk.Commit}})
|
|
switch {
|
|
case err == nil, errors.Is(err, inventory.ErrSuperseded):
|
|
// Superseded is no failure: the module is already at what a later merge's key built, which a walk that carries
|
|
// this merge is answered by, as a build-agent outcome superseded was.
|
|
keepKeyWait(id, "")
|
|
keyTakes.Lock()
|
|
keyTakes.taken[id] = true
|
|
keyTakes.Unlock()
|
|
case errors.Is(err, errKeyWaits):
|
|
keepKeyWait(id, err.Error())
|
|
default:
|
|
why := err.Error()
|
|
if rerr := open.inventory.RecordRefused(ctx, id, module, why); rerr != nil {
|
|
fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, rerr)
|
|
}
|
|
keepRefusalFor(id, why, time.Now())
|
|
}
|
|
}
|
|
|
|
func keepKeyWait(id, why string) {
|
|
keyTakes.Lock()
|
|
defer keyTakes.Unlock()
|
|
if why == "" {
|
|
delete(keyTakes.waits, id)
|
|
return
|
|
}
|
|
keyTakes.waits[id] = why
|
|
}
|