jschoubben
4eeba8fb20
A found ended or pending key is skipped and a pulled-in key is created only where no key exists. RecordSentWith and RecordSentUnder set sent_builds = null in their own update of the node row, so a machine delivery 2 has sent is never copied afterwards. These two updates and the copy's own null are the only writes to the column.
363 lines
13 KiB
Go
363 lines
13 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"sort"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"github.com/novox/mesh-controller/internal/chain"
|
|
)
|
|
|
|
// The links of the chain (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): a send that puts a build on a machine
|
|
// writes, in the assignment of the module there, the build, what it published, its commit and the rollout that sent it, and
|
|
// since when; the rollout lists the assignments it changed with the build before and after (the controller's send, in
|
|
// release_plan.go). One read of an assignment then names the commit, the build and the rollout that put it there, and one read
|
|
// of the rollout lists its assignments.
|
|
|
|
type rolloutKey struct{}
|
|
|
|
// RolloutModule is what a rollout sends of one module: the commit and the build of it.
|
|
type RolloutModule struct{ Commit, Build string }
|
|
|
|
type rolloutOf struct {
|
|
id string
|
|
modules map[string]RolloutModule
|
|
}
|
|
|
|
// WithRollout is a context for a send a rollout makes: what RecordSent writes in the assignments of the modules it carries
|
|
// names this rollout and the build it sent. A module the send carries beside these, that another walk is moving, is named
|
|
// by the build the mesh holds of its commit and by no rollout.
|
|
func WithRollout(ctx context.Context, id string, modules map[string]RolloutModule) context.Context {
|
|
return context.WithValue(ctx, rolloutKey{}, rolloutOf{id: id, modules: modules})
|
|
}
|
|
|
|
// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the
|
|
// commit. Empty when the assignments are not on the bus, and for a module not sent yet.
|
|
func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) {
|
|
c, err := i.theChain(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
all, err := c.Assignments.All(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := map[string]chain.Ref{}
|
|
for _, k := range all {
|
|
a := k.Value
|
|
if a.Node == node && a.Stands() && (a.Build != "" || a.Commit != "") {
|
|
out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links.
|
|
func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) {
|
|
c, err := i.theChain(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
all, err := c.Assignments.All(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []chain.Assignment
|
|
for _, k := range all {
|
|
if a := k.Value; a.Node == node && !a.Assigned.IsZero() && a.Stands() {
|
|
out = append(out, a)
|
|
}
|
|
}
|
|
sort.Slice(out, func(a, b int) bool { return out[a].Module < out[b].Module })
|
|
return out, nil
|
|
}
|
|
|
|
// recordRuns writes what a send put on a machine into the assignments, once the send is recorded, and it is how the mesh
|
|
// knows what a machine was sent (SentBuilds). **A send that names its builds** writes, for each module it carried a build of
|
|
// — by the commit — the build, what it published, the commit, the rollout and since when, on the module's key: filled where
|
|
// the key names nothing, replaced when the commit moved, never blanked by a push by hand; a module nothing assigns, carried
|
|
// because another requires it, gets a standing key marked pulled-in; and every other key of the machine loses what it named,
|
|
// for the machine is not sent it any more. **A send that names none** (a declaration by hand) makes what the machine runs
|
|
// unknown: every key of it says so (sent-unknown) and names no commit. **An error here fails the record of the send**, for
|
|
// what a machine was sent is read from these keys and from nowhere else.
|
|
func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error {
|
|
c, err := i.theChain(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("what the machine runs cannot be written: %w", err)
|
|
}
|
|
var node string
|
|
if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) {
|
|
return nil
|
|
} else if err != nil {
|
|
return err
|
|
}
|
|
rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf)
|
|
all, err := c.Assignments.All(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("the assignments of %s cannot be read: %w", node, err)
|
|
}
|
|
var failed []error
|
|
for _, k := range sortedValues(all) {
|
|
a := k.Value
|
|
if a.Node != node || !a.Stands() {
|
|
continue
|
|
}
|
|
if builds[a.Module] != "" {
|
|
continue // carried, and its commit is written below
|
|
}
|
|
if err := i.forgetRun(ctx, c, k, builds == nil, at); err != nil {
|
|
failed = append(failed, fmt.Errorf("%s on %s: %w", a.Module, node, err))
|
|
}
|
|
}
|
|
modules := make([]string, 0, len(builds))
|
|
for m := range builds {
|
|
modules = append(modules, m)
|
|
}
|
|
sort.Strings(modules)
|
|
for _, m := range modules {
|
|
if builds[m] == "" {
|
|
continue
|
|
}
|
|
if err := i.recordRun(ctx, c, node, m, builds[m], rollout, at); err != nil {
|
|
failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err))
|
|
}
|
|
}
|
|
return errors.Join(failed...)
|
|
}
|
|
|
|
func sortedValues(all map[string]chain.Kept[chain.Assignment]) []chain.Kept[chain.Assignment] {
|
|
out := make([]chain.Kept[chain.Assignment], 0, len(all))
|
|
for key, k := range all {
|
|
k.Key = key
|
|
out = append(out, k)
|
|
}
|
|
sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key })
|
|
return out
|
|
}
|
|
|
|
// forgetRun writes that a machine is no longer sent what a key named: its links are cleared, and for a send by hand the key
|
|
// says what the machine runs is unknown.
|
|
func (i *Inventory) forgetRun(ctx context.Context, c *chain.Chain, k chain.Kept[chain.Assignment], hand bool, at time.Time) error {
|
|
for try := 0; try < casTries; try++ {
|
|
next := k.Value
|
|
next.Build, next.Artifact, next.Package, next.Commit, next.Rollout, next.Since = "", "", nil, "", "", time.Time{}
|
|
next.SentUnknown = time.Time{}
|
|
if hand {
|
|
next.SentUnknown = at.UTC()
|
|
}
|
|
if next.Build == k.Value.Build && next.Commit == k.Value.Commit && next.Rollout == k.Value.Rollout &&
|
|
next.Artifact == k.Value.Artifact && next.Package == k.Value.Package && next.Since.Equal(k.Value.Since) &&
|
|
next.SentUnknown.Equal(k.Value.SentUnknown) {
|
|
return nil
|
|
}
|
|
if _, err := c.Assignments.Update(ctx, k.Key, next, k.Revision); errors.Is(err, chain.ErrMoved) {
|
|
cur, found, gerr := c.Assignments.Get(ctx, k.Key)
|
|
if gerr != nil {
|
|
return gerr
|
|
}
|
|
if !found || !cur.Value.Stands() {
|
|
return nil
|
|
}
|
|
k = chain.Kept[chain.Assignment]{Key: k.Key, Value: cur.Value, Revision: cur.Revision}
|
|
continue
|
|
} else {
|
|
return err
|
|
}
|
|
}
|
|
return fmt.Errorf("the assignment kept moving under the write")
|
|
}
|
|
|
|
func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error {
|
|
key := assignmentKey(node, module)
|
|
var build, rolloutID string
|
|
if sent, ours := rollout.modules[module]; ours && sent.Commit == commit {
|
|
build, rolloutID = sent.Build, rollout.id
|
|
}
|
|
var published *chain.Published
|
|
resolved := false
|
|
for try := 0; try < casTries; try++ {
|
|
cur, found, err := c.Assignments.Get(ctx, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if found && cur.Value.State != "" {
|
|
return nil // a wait for a build: the module is not on the machine by it
|
|
}
|
|
standing := found && cur.Value.Stands()
|
|
same := standing && cur.Value.Commit == commit
|
|
if same && cur.Value.Build != "" && cur.Value.SentUnknown.IsZero() && (rolloutID == "" || cur.Value.Rollout != "") {
|
|
return nil // a send that carries what the machine runs already says nothing new
|
|
}
|
|
if !resolved {
|
|
// The build of the commit is looked up once, and only for a module that moved or names no build.
|
|
if build == "" {
|
|
if build, err = i.BuildOf(ctx, module, commit); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if build != "" {
|
|
if b, found, err := i.BuildByID(ctx, build); err != nil {
|
|
return err
|
|
} else if found {
|
|
published = publishedOf(b.Made)
|
|
}
|
|
}
|
|
resolved = true
|
|
}
|
|
next := cur.Value
|
|
switch {
|
|
case !standing:
|
|
// Carried and assigned to nobody here: a standing key, no time assigned, marked pulled in. An ended one is
|
|
// written over, the ended one staying in history.
|
|
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Commit: commit, Build: build, Rollout: rolloutID,
|
|
Since: at.UTC()}
|
|
case same:
|
|
// Already running it: fill what is missing, blank nothing.
|
|
if next.Build == "" {
|
|
next.Build = build
|
|
}
|
|
if next.Rollout == "" {
|
|
next.Rollout = rolloutID
|
|
}
|
|
if next.Since.IsZero() {
|
|
next.Since = at.UTC()
|
|
}
|
|
default:
|
|
next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC()
|
|
next.Artifact, next.Package = "", nil
|
|
}
|
|
next.SentUnknown = time.Time{}
|
|
if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build {
|
|
next.Artifact, next.Package = published.Artifact, published.Package
|
|
}
|
|
if standing && next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout &&
|
|
next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) &&
|
|
next.Since.Equal(cur.Value.Since) && next.SentUnknown.Equal(cur.Value.SentUnknown) {
|
|
return nil
|
|
}
|
|
if !found {
|
|
if _, err := c.Assignments.Create(ctx, key, next); errors.Is(err, chain.ErrExists) {
|
|
continue
|
|
} else {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
|
continue
|
|
} else {
|
|
return err
|
|
}
|
|
}
|
|
return fmt.Errorf("the assignment kept moving under the write")
|
|
}
|
|
|
|
// CopySentBuilds copies what delivery 1 recorded in node.sent_builds onto the keys of the assignments, once, at the start of
|
|
// the controller that holds the lease and has attached the chain (issue 1109, review round 1): delivery 1 wrote no commit onto
|
|
// the key of a module an assignment pulled in, so without it those modules would be missing from SentBuilds although the
|
|
// machine is known to have been sent. A key that names a commit is never overwritten, so it is idempotent; once a node's keys
|
|
// are written its column is emptied, so a later restart cannot lay a stale commit over a key a later send cleared or ended.
|
|
// It answers how many keys it wrote; an error is raised by the caller as a condition.
|
|
func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) {
|
|
c, err := i.theChain(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
rows, err := i.store.Pool().Query(ctx, `select name, sent_builds from node where sent_builds is not null`)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
type kept struct {
|
|
name string
|
|
sent map[string]string
|
|
}
|
|
var all []kept
|
|
for rows.Next() {
|
|
var name string
|
|
var raw []byte
|
|
if err := rows.Scan(&name, &raw); err != nil {
|
|
rows.Close()
|
|
return 0, err
|
|
}
|
|
sent := map[string]string{}
|
|
if err := json.Unmarshal(raw, &sent); err != nil {
|
|
rows.Close()
|
|
return 0, fmt.Errorf("what %s was sent, in the store, cannot be read: %w", name, err)
|
|
}
|
|
all = append(all, kept{name, sent})
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
now := time.Now().UTC()
|
|
for _, k := range all {
|
|
if err := i.copySentBuildsOf(ctx, c, k.name, k.sent, now); err != nil {
|
|
return 0, fmt.Errorf("what %s was sent cannot be copied onto its assignments: %w", k.name, err)
|
|
}
|
|
if _, err := i.store.Pool().Exec(ctx, `update node set sent_builds = null where name = $1`, k.name); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
return len(all), nil
|
|
}
|
|
|
|
// copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that
|
|
// names no commit gets it (and the build, when one is known; no rollout), and where no key exists a standing pulled-in key is
|
|
// made. A key with a commit, an ended key, a hand send's unknown and a wait for a build are left as they are.
|
|
func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error {
|
|
for _, module := range slices.Sorted(maps.Keys(sent)) {
|
|
commit := sent[module]
|
|
if commit == "" {
|
|
continue
|
|
}
|
|
key := assignmentKey(node, module)
|
|
build, err := i.chainBuildOf(ctx, c, module, commit)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for try := 0; ; try++ {
|
|
if try == casTries {
|
|
return fmt.Errorf("the assignment %s kept moving under the write", key)
|
|
}
|
|
cur, found, err := c.Assignments.Get(ctx, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if found && (cur.Value.State != "" || !cur.Value.Ended.IsZero() || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) {
|
|
break // a wait, an ended key (never revived), one that names a commit, or a hand send's unknown
|
|
}
|
|
next := cur.Value
|
|
if !found {
|
|
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at}
|
|
}
|
|
next.Commit, next.Build = commit, build
|
|
if next.Since.IsZero() {
|
|
next.Since = at
|
|
}
|
|
if !found {
|
|
_, err = c.Assignments.Create(ctx, key, next)
|
|
if errors.Is(err, chain.ErrExists) {
|
|
continue
|
|
}
|
|
} else {
|
|
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
|
|
if errors.Is(err, chain.ErrMoved) {
|
|
continue
|
|
}
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
}
|