Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0149a3d7de | ||
|
|
b56c1d00a3 | ||
|
|
e16cb4207b | ||
|
|
a47c63799e | ||
|
|
703badcf5a | ||
|
|
cb00bbbfcf | ||
|
|
0e4feb0d3a | ||
|
|
275ddd76e8 | ||
|
|
e0fb241263 | ||
|
|
f89d870308 | ||
|
|
36c4ce605e | ||
|
|
fc55785956 |
@@ -219,10 +219,10 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
}
|
||||
if p.Batch() {
|
||||
if err := keepBatch(ctx, inv, &p, now, ""); err != nil {
|
||||
return notNow(err)
|
||||
return unsaved(err)
|
||||
}
|
||||
} else if err := nameMerges(ctx, inv, &p); err != nil {
|
||||
return notNow(err)
|
||||
return unsaved(err)
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s) after a later merge of its branch was put in %s (%s): that %s "+
|
||||
"answers it\n", repository, m.Base, m.Commit, p.ID, p.State, walkOrBatch(p))
|
||||
@@ -239,7 +239,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
}
|
||||
holdFor(&batch, m.Head, waits, holding, own, now)
|
||||
if err := keepBatch(ctx, inv, &batch, now, ""); err != nil {
|
||||
return notNow(err)
|
||||
return unsaved(err)
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID,
|
||||
batchWords(batch, now))
|
||||
@@ -820,6 +820,12 @@ func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan
|
||||
// walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits.
|
||||
func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
inv := open.inventory
|
||||
// A merge found in two records is the half-done move of a crash: put right before anything is decided from it.
|
||||
if repaired, err := inv.RepairMerges(ctx); err != nil {
|
||||
return err
|
||||
} else if repaired > 0 {
|
||||
fmt.Printf("%d merge(s) found in two plans, the half of a move a restart interrupted, are kept by the plan they were moved to\n", repaired)
|
||||
}
|
||||
if err := searchAfterFailures(ctx, inv); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -152,6 +152,8 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
Path: result.Path,
|
||||
// What it was made from (novox/hq issue 280): two builds with one are one build.
|
||||
SourceFingerprint: result.SourceFingerprint,
|
||||
// And its reuse key (novox/hq ADR 0350 §1), which the chain names (ADR 0408 §1).
|
||||
ReuseKey: result.ReuseKey,
|
||||
}
|
||||
// When it was asked, which is what orders it against another build of the same module
|
||||
// (novox/hq 04-ISSUES/219) — not when it was heard.
|
||||
@@ -254,6 +256,18 @@ func buildsCommand(ctx context.Context, args []string) error {
|
||||
for _, made := range b.Made {
|
||||
fmt.Printf(" %-10s %s\n", made.Kind, made.Reference)
|
||||
}
|
||||
// What it published and under what reuse key (novox/hq ADR 0408 §1).
|
||||
if p := b.Published; p != nil {
|
||||
if p.Artifact != "" {
|
||||
fmt.Printf(" published %s\n", p.Artifact)
|
||||
}
|
||||
if p.Package != nil {
|
||||
fmt.Printf(" published %s@%s\n", p.Package.Name, p.Package.Version)
|
||||
}
|
||||
}
|
||||
if b.ReuseKey != "" {
|
||||
fmt.Printf(" reuse key %s\n", b.ReuseKey)
|
||||
}
|
||||
if !b.Worked() {
|
||||
// The builder's own first line. The whole failure is often a build log, and printing
|
||||
// it here would bury every other row.
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
|
||||
// bus's key-value store, written by the controller that holds the lease. The tables of the controller's store are
|
||||
// copied into them once, by the serving controller at its start, and a kind is read from its bucket only once the move
|
||||
// has recorded it (inventory.ChainReads).
|
||||
|
||||
// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
|
||||
func chainHolds(ctx context.Context) error {
|
||||
_, err := theLease.epoch(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// openTheChain opens the chain's buckets on the bus: the serving controller's connection, or one dialled for a command.
|
||||
func openTheChain(ctx context.Context) (*chain.Chain, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Not closed: a borrowed connection closes nothing, and a dialled one lasts as long as the command.
|
||||
return chain.OnTheBus(ctx, js.Conn(), chainHolds)
|
||||
}
|
||||
|
||||
// kindChainMoveRefused is the condition of a move that stopped without switching a kind.
|
||||
const kindChainMoveRefused = "chain-move-refused"
|
||||
|
||||
// chainMoveRetry is how long a move that failed for want of the bus waits before it tries again.
|
||||
const chainMoveRetry = 30 * time.Second
|
||||
|
||||
// movingTheChain copies the tables into the buckets and records it, at the serving controller's start: after the buckets
|
||||
// exist and the lease is held, before anything reads a kind that has not moved. **The first try is made here, before the
|
||||
// controller serves**, so what it serves is read from one place. A move that fails for want of the bus is tried again
|
||||
// every half minute while the controller serves from the tables; one that is refused — a count that differs, a value
|
||||
// over the bus's largest — is raised as an urgent condition, and the tables stay what is read: nothing is half-switched.
|
||||
func movingTheChain(ctx context.Context, inv *inventory.Inventory) {
|
||||
say := func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }
|
||||
// The first try is made here, before the controller serves; the tries after it, while it serves.
|
||||
if movedOrStopped(ctx, inv, say) {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(chainMoveRetry):
|
||||
}
|
||||
if movedOrStopped(ctx, inv, say) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// movedOrStopped makes one try at the move and says whether there is nothing more to try: moved, refused, or the controller
|
||||
// ending. False for a failure of the kind that is tried again (the bus away, a write not granted yet, the lease not held).
|
||||
func movedOrStopped(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) bool {
|
||||
_, err := inv.MoveChain(ctx, say)
|
||||
var refusal *inventory.Refusal
|
||||
switch {
|
||||
case err == nil:
|
||||
go clearChainMove(ctx)
|
||||
go askAgainSoon(ctx, inv)
|
||||
return true
|
||||
case errors.As(err, &refusal):
|
||||
say("%v", err)
|
||||
go raiseChainMove(ctx, refusal)
|
||||
return true
|
||||
case ctx.Err() != nil:
|
||||
return true
|
||||
}
|
||||
say("the move to the bus did not finish, and the tables are read until it does: %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// chainMoveObservation is the urgent condition for a refused move.
|
||||
func chainMoveObservation(r *inventory.Refusal) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "chain." + r.Kind, Token: kindChainMoveRefused,
|
||||
Kind: kindChainMoveRefused, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Summary: "the controller's " + r.Kind + " were not moved to the bus: " + r.Why, Source: "chain move"}
|
||||
}
|
||||
|
||||
// whenKept runs f once the serving controller keeps conditions: the move runs before they are kept.
|
||||
func whenKept(ctx context.Context, f func(k *conditions.Keeper)) {
|
||||
for conditionsFrom == nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
f(conditionsFrom)
|
||||
}
|
||||
|
||||
func raiseChainMove(ctx context.Context, r *inventory.Refusal) {
|
||||
whenKept(ctx, func(k *conditions.Keeper) {
|
||||
if _, err := k.Observe(ctx, chainMoveObservation(r)); err != nil {
|
||||
fmt.Printf("chain: the condition for the refused move could not be raised: %v\n", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func clearChainMove(ctx context.Context) {
|
||||
whenKept(ctx, func(k *conditions.Keeper) {
|
||||
for _, kind := range []string{inventory.KindBuilds, inventory.KindRollouts, inventory.KindAssignments} {
|
||||
key := chainMoveObservation(&inventory.Refusal{Kind: kind}).Key()
|
||||
if _, err := k.Clear(ctx, key, "the "+kind+" are on the bus"); err != nil {
|
||||
fmt.Printf("chain: the condition for the move of the %s could not be cleared: %v\n", kind, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// attachTheChain gives the inventory of the serving controller the chain on its connection, then moves what has not moved.
|
||||
func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
|
||||
bus, ok := server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
return errors.New("the chain lives on the bus, and this control plane is not on it")
|
||||
}
|
||||
c, err := chain.OnTheBus(ctx, bus.Conn, chainHolds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inv.UseChain(c)
|
||||
movingTheChain(ctx, inv)
|
||||
return nil
|
||||
}
|
||||
|
||||
// askAgain asks once more for each build request the move found never handed over (novox/hq issue 981 note 7 Q5): the
|
||||
// requests a merge could not ask for, which a person's hand `build` would otherwise have to repeat. **At most once**: a
|
||||
// request is claimed before it is asked and settled after, so a crash between loses the ask, which is where it stood.
|
||||
func askAgain(ctx context.Context, inv *inventory.Inventory) {
|
||||
pending, err := inv.RequestsToAskAgain(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("chain: the requests never handed over cannot be read: %v\n", err)
|
||||
return
|
||||
}
|
||||
for _, r := range pending {
|
||||
if claimed, err := inv.ClaimToAskAgain(ctx, r.ID); err != nil || !claimed {
|
||||
if err != nil {
|
||||
fmt.Printf("chain: %s cannot be claimed to be asked again: %v\n", r.ID, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
id, err := askABuild(ctx, buildSource{Repository: r.Repository, Seat: r.Seat}, r.Path, r.Ref)
|
||||
if err != nil {
|
||||
if serr := inv.AskedAgainAs(ctx, r.ID, "not asked again: "+err.Error()); serr != nil {
|
||||
fmt.Printf("chain: %s was not asked again, and that could not be kept: %v\n", r.ID, serr)
|
||||
}
|
||||
fmt.Printf("chain: %s was never handed over and cannot be asked again: %v\n", r.ID, err)
|
||||
continue
|
||||
}
|
||||
recordBuildRequest(ctx, inv, inventory.BuildRequest{ID: id, Repository: r.Repository, Seat: r.Seat, Path: r.Path,
|
||||
Ref: r.Ref, Commit: r.Commit, For: r.For})
|
||||
if err := inv.AskedAgainAs(ctx, r.ID, id); err != nil {
|
||||
fmt.Printf("chain: %s was asked again as %s, and that could not be kept: %v\n", r.ID, id, err)
|
||||
}
|
||||
fmt.Printf("chain: %s, never handed over, is asked again as %s\n", r.ID, id)
|
||||
}
|
||||
}
|
||||
|
||||
// askAgainSoon waits for the controller to be serving, which an ask needs, and then asks again.
|
||||
func askAgainSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Minute):
|
||||
askAgain(ctx, inv)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **Acceptance 10 of design 47: one read of an assignment names the commit, the build and the rollout that put it there,
|
||||
// and one read of the rollout lists the assignments it changed** (novox/hq ADR 0408, issue 1095). It enters by the live
|
||||
// trigger — a merge heard, its batch cut into a plan when the window closed, its build answered through the build seat's
|
||||
// path, the plan advanced — and writes no key and no row by hand. Only the machine's end of the send is the test's: the send
|
||||
// is recorded as the controller records one it made, and the gate's facts are the store's.
|
||||
func TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var sent [][]string
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
digest := fmt.Sprintf("d-%s-%d", node, len(sent))
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest,
|
||||
map[string]string{"notes": current["notes"].Commit}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
// A merge of notes is heard, and its window closes: the batch is cut into the walk.
|
||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(time.Second+90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
||||
}
|
||||
rollout := ws[0].ID
|
||||
|
||||
// The walk asks its first tier (at the cut, or at its next step); the build seat answers, and the controller takes the
|
||||
// outcome in as it does.
|
||||
asking, err := inv.PlanByID(ctx, rollout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ {
|
||||
advancePlans(ctx, open)
|
||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
buildID := asking.Modules["notes"].Build
|
||||
if buildID == "" || len(*asked) != 1 || !strings.HasSuffix((*asked)[0], "#"+buildID) {
|
||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("a", 64)
|
||||
result := outcome(buildID, "modules/notes", "c-notes", "notes", "")
|
||||
result.ReuseKey = "rk-notes"
|
||||
result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}}
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The walk sends the tier to its first machine.
|
||||
for try := 0; try < 3 && len(sent) == 0; try++ {
|
||||
advancePlans(ctx, open)
|
||||
}
|
||||
if !slices.Equal(slices.Concat(sent...), []string{"anchor"}) {
|
||||
t.Fatalf("sent %v", sent)
|
||||
}
|
||||
|
||||
// One read of the assignment.
|
||||
on, err := inv.AssignmentsOn(ctx, "anchor")
|
||||
a, ok := assignmentOf(on, "notes")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("the assignments of anchor: %+v %v", on, err)
|
||||
}
|
||||
// Beside it a module pulled in by the assignment, which no send of this walk put anywhere.
|
||||
if wireguard, pulled := assignmentOf(on, "mesh-wireguard"); pulled && (wireguard.Build != "" || wireguard.Rollout != "") {
|
||||
t.Fatalf("a module this rollout did not send names a build or a rollout: %+v", wireguard)
|
||||
}
|
||||
if a.Module != "notes" || a.Commit != "c-notes" || a.Build != buildID || a.Rollout != rollout || a.Artifact != digest ||
|
||||
a.Since.IsZero() || a.Assigned.IsZero() || !a.Stands() {
|
||||
t.Fatalf("the assignment names %+v, not commit c-notes, build %s, artifact %s, rollout %s", a, buildID, digest, rollout)
|
||||
}
|
||||
// One read of the rollout.
|
||||
p, err := inv.PlanByID(ctx, rollout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantChange := chain.Change{Node: "anchor", Module: "notes", After: chain.Ref{Build: buildID, Commit: "c-notes"}}
|
||||
if !slices.Contains(p.Changed, wantChange) || !slices.Contains(p.Builds, buildID) || len(p.Steps) != 1 ||
|
||||
p.Steps[0].Node != "anchor" || p.Steps[0].Sent == nil {
|
||||
t.Fatalf("the rollout lists changed %+v, builds %v, steps %+v; want %+v", p.Changed, p.Builds, p.Steps, wantChange)
|
||||
}
|
||||
// And one read of the build names the same commit, with what it published and its reuse key.
|
||||
built, found, err := inv.BuildByID(ctx, buildID)
|
||||
if err != nil || !found || built.Commit != "c-notes" || built.Module != "notes" {
|
||||
t.Fatalf("the build: %+v %v %v", built, found, err)
|
||||
}
|
||||
lately, err := inv.Builds(ctx, "notes", 1)
|
||||
if err != nil || len(lately) != 1 || lately[0].ReuseKey != "rk-notes" || lately[0].Published == nil ||
|
||||
lately[0].Published.Artifact != digest {
|
||||
t.Fatalf("what the build published: %+v %v", lately, err)
|
||||
}
|
||||
// The verbs say the links.
|
||||
if lines := changedLines(p); len(lines) != 2 || !strings.Contains(lines[1], "anchor/notes") ||
|
||||
!strings.Contains(lines[1], buildID) {
|
||||
t.Fatalf("`plans` says %v", lines)
|
||||
}
|
||||
if lines := runLines(on); len(lines) != 2 || !strings.Contains(lines[1], buildID) || !strings.Contains(lines[1], rollout) {
|
||||
t.Fatalf("`node show` says %v", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine sent again what it runs says nothing new, and a push by hand with no rollout fills what is missing and blanks
|
||||
// nothing (issue 1095).
|
||||
func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-1", Module: "notes", Commit: "c1", Repository: "novox/notes"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
node := nodeID(t, open, "anchor")
|
||||
inRollout := inventory.WithRollout(ctx, "plan-1", map[string]inventory.RolloutModule{"notes": {Commit: "c1", Build: "build-1"}})
|
||||
if err := inv.RecordSent(inRollout, node, "d1", map[string]string{"notes": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent again by hand, with no rollout: the assignment still names the rollout that put the build there.
|
||||
if err := inv.RecordSent(ctx, node, "d2", map[string]string{"notes": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
on, _ := inv.AssignmentsOn(ctx, "anchor")
|
||||
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
|
||||
t.Fatalf("the assignment names %+v", on)
|
||||
}
|
||||
// A declaration sent by hand with no builds known says nothing of them.
|
||||
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c1" {
|
||||
t.Fatalf("a send of unknown builds changed %+v", on)
|
||||
}
|
||||
// A module sent a commit no build is kept of is named by its commit alone.
|
||||
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c2" || runOf(on, "notes").Build != "" ||
|
||||
runOf(on, "notes").Rollout != "" {
|
||||
t.Fatalf("the assignment after a push of c2 by hand names %+v", on)
|
||||
}
|
||||
}
|
||||
|
||||
// assignmentOf is one module's assignment among those of a machine: assigning a module assigns what it requires beside it.
|
||||
func assignmentOf(on []chain.Assignment, module string) (chain.Assignment, bool) {
|
||||
for _, a := range on {
|
||||
if a.Module == module {
|
||||
return a, true
|
||||
}
|
||||
}
|
||||
return chain.Assignment{}, false
|
||||
}
|
||||
|
||||
func runOf(on []chain.Assignment, module string) chain.Assignment {
|
||||
a, _ := assignmentOf(on, module)
|
||||
return a
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a
|
||||
// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of
|
||||
// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the
|
||||
// build before and after.
|
||||
|
||||
// rolloutModules is what a plan sends of each module: the commit and the build of it.
|
||||
func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule {
|
||||
out := make(map[string]inventory.RolloutModule, len(p.Modules))
|
||||
for m, s := range p.Modules {
|
||||
if s != nil && s.Commit != "" {
|
||||
out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout.
|
||||
func sendingAs(ctx context.Context, p *inventory.Plan) context.Context {
|
||||
return inventory.WithRollout(ctx, p.ID, rolloutModules(p))
|
||||
}
|
||||
|
||||
// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit,
|
||||
// else the commit the machine was last sent for it. Read before the send, which writes the new ones.
|
||||
func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref {
|
||||
was := map[string]chain.Ref{}
|
||||
if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known {
|
||||
for m, commit := range sent {
|
||||
was[m] = chain.Ref{Commit: commit}
|
||||
}
|
||||
}
|
||||
if ran, err := inv.AssignmentRuns(ctx, node); err == nil {
|
||||
for m, r := range ran {
|
||||
was[m] = r
|
||||
}
|
||||
}
|
||||
return was
|
||||
}
|
||||
|
||||
// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and
|
||||
// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the
|
||||
// module is replaced, so a send made again says it once.
|
||||
func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) {
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}}
|
||||
if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 {
|
||||
change.Before = p.Changed[at].Before
|
||||
p.Changed[at] = change
|
||||
} else {
|
||||
p.Changed = append(p.Changed, change)
|
||||
}
|
||||
if s.Build != "" && !slices.Contains(p.Builds, s.Build) {
|
||||
p.Builds = append(p.Builds, s.Build)
|
||||
}
|
||||
}
|
||||
if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) {
|
||||
sent := at.UTC()
|
||||
p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent})
|
||||
}
|
||||
}
|
||||
|
||||
// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none.
|
||||
func refSaid(r chain.Ref) string {
|
||||
switch {
|
||||
case r.Build == "" && r.Commit == "":
|
||||
return "nothing"
|
||||
case r.Build == "":
|
||||
return "commit " + short(r.Commit)
|
||||
}
|
||||
return r.Build + " (" + short(r.Commit) + ")"
|
||||
}
|
||||
|
||||
// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after.
|
||||
func changedLines(p inventory.Plan) []string {
|
||||
if len(p.Changed) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := []string{" changed:"}
|
||||
for _, c := range p.Changed {
|
||||
out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the
|
||||
// rollout that put it there and since when.
|
||||
func runLines(assigned []chain.Assignment) []string {
|
||||
var out []string
|
||||
for _, a := range assigned {
|
||||
if a.Build == "" && a.Commit == "" {
|
||||
continue
|
||||
}
|
||||
line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit}))
|
||||
if a.Rollout != "" {
|
||||
line += ", sent by " + a.Rollout
|
||||
}
|
||||
if !a.Since.IsZero() {
|
||||
line += " at " + a.Since.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]string{" what they run, as the chain names it:"}, out...)
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one
|
||||
// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the
|
||||
// rollout carried once each, and one step for a machine and a tier.
|
||||
func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{
|
||||
"notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}}
|
||||
was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}}
|
||||
|
||||
noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at)
|
||||
noteChanges(&p, "laptop", []string{"notes"}, nil, at)
|
||||
// The send noted again, after what was sent had become what the machine runs: the first before stands.
|
||||
noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute))
|
||||
|
||||
want := []chain.Change{
|
||||
{Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
{Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}},
|
||||
{Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
}
|
||||
if len(p.Changed) != len(want) {
|
||||
t.Fatalf("changed %+v", p.Changed)
|
||||
}
|
||||
for k := range want {
|
||||
if p.Changed[k] != want[k] {
|
||||
t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k])
|
||||
}
|
||||
}
|
||||
if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" {
|
||||
t.Errorf("builds %v", p.Builds)
|
||||
}
|
||||
if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" {
|
||||
t.Errorf("steps %+v", p.Steps)
|
||||
}
|
||||
lines := changedLines(p)
|
||||
if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") ||
|
||||
!strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") {
|
||||
t.Errorf("changed lines %q", lines)
|
||||
}
|
||||
if got := changedLines(inventory.Plan{}); got != nil {
|
||||
t.Errorf("a plan that changed nothing says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
lines := runLines([]chain.Assignment{
|
||||
{Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at},
|
||||
{Node: "anchor", Module: "web", Commit: "c3"},
|
||||
{Node: "anchor", Module: "never-sent"},
|
||||
})
|
||||
if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") ||
|
||||
!strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") {
|
||||
t.Fatalf("lines %q", lines)
|
||||
}
|
||||
if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil {
|
||||
t.Fatal("a module never sent is said to run something")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// refusing is a bucket that refuses to write while it is told to.
|
||||
type refusing struct {
|
||||
chain.Backend
|
||||
refuse atomic.Bool
|
||||
}
|
||||
|
||||
func (r *refusing) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if r.refuse.Load() {
|
||||
return 0, nats.ErrTimeout
|
||||
}
|
||||
return r.Backend.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
func (r *refusing) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if r.refuse.Load() {
|
||||
return 0, nats.ErrTimeout
|
||||
}
|
||||
return r.Backend.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
// **A merge whose plan cannot be saved is held, recorded nowhere, and recorded when it is handed over again** (ADR 0407
|
||||
// decision 3, issue 1092): hearMerge answers ErrTryAgain — the bus leaves the announcement unacknowledged and hands it
|
||||
// over again — and the merge is in no record until the plan holding it is written.
|
||||
func TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
rollouts := &refusing{Backend: chain.NewMemory()}
|
||||
open.inventory.UseChain(chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil }))
|
||||
merge := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
|
||||
rollouts.refuse.Store(true)
|
||||
err := (following{open: open}).hearMerge(ctx, merge, t0.Add(time.Second))
|
||||
if !errors.Is(err, link.ErrTryAgain) {
|
||||
t.Fatalf("a merge whose plan could not be saved answered %v, which the bus takes as done", err)
|
||||
}
|
||||
if _, known, err := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); err != nil || known {
|
||||
t.Fatalf("the merge is kept though its plan was not: %v %v", known, err)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 0 {
|
||||
t.Fatalf("a batch was kept though its write failed: %+v", bs)
|
||||
}
|
||||
|
||||
rollouts.refuse.Store(false)
|
||||
hear(t, open, merge, t0.Add(10*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 {
|
||||
t.Fatalf("the merge handed over again made %d batch(es)", len(bs))
|
||||
}
|
||||
if m, known, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); !known || m.Plan != bs[0].ID {
|
||||
t.Fatalf("the merge is %+v %v, and the batch %s", m, known, bs[0].ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A merge whose plan could not be saved is held, never taken** (novox/hq ADR 0407 decision 3, issue 1092): the merge is
|
||||
// carried until its plan is saved, and the bus acknowledges the announcement only when the controller answers without
|
||||
// ErrTryAgain (link.Server.sourceMoved), so a failed save of any kind must answer it.
|
||||
func TestAFailureToSaveAMergesPlanIsHeldAndNotTaken(t *testing.T) {
|
||||
cause := errors.New("the plan for novox/mesh-catalog is not written: the plan was written by somebody else since it was read")
|
||||
held := unsaved(cause)
|
||||
if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) {
|
||||
t.Fatalf("a failed save answers %v", held)
|
||||
}
|
||||
if unsaved(nil) != nil {
|
||||
t.Fatal("a save that worked answers an error")
|
||||
}
|
||||
// Not the same as notNow, which holds a merge only for a store that is away.
|
||||
if errors.Is(notNow(cause), link.ErrTryAgain) {
|
||||
t.Fatal("notNow holds every failure; the premise of this test is gone")
|
||||
}
|
||||
}
|
||||
@@ -929,6 +929,12 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
}
|
||||
if len(assigned) > 0 {
|
||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||
// And what each runs, as its assignment names it (novox/hq ADR 0408, issue 1095).
|
||||
if on, err := inv.AssignmentsOn(ctx, name); err == nil {
|
||||
for _, line := range runLines(on) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
|
||||
// reason its condition says. Not computable is said, never read as "all applied".
|
||||
|
||||
@@ -172,6 +172,11 @@ func serve(ctx context.Context) (err error) {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
// The chain on the bus, and the move of what the tables still hold into it (novox/hq ADR 0408 §8): before anything
|
||||
// reads builds, rollouts or assignments.
|
||||
if err := attachTheChain(ctx, inv, server); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
|
||||
// somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was
|
||||
|
||||
@@ -2,12 +2,15 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/recordmirror"
|
||||
)
|
||||
@@ -41,7 +44,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
||||
return e
|
||||
}
|
||||
sources := map[string]func(context.Context) (map[string][]byte, error){
|
||||
recordmirror.Assignments.Name: inv.AssignmentsMirror,
|
||||
recordmirror.Assignments.Name: inv.AssignmentsMirrorToWrite,
|
||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||
}
|
||||
@@ -49,10 +52,18 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
||||
failing := map[string]string{}
|
||||
tick := time.NewTicker(mirrorEvery)
|
||||
defer tick.Stop()
|
||||
events := &assignmentEvents{}
|
||||
for {
|
||||
if holds() {
|
||||
// The assignments, once they are keys of the bus, are written by the verbs that change them (novox/hq ADR
|
||||
// 0408): this tick keeps the provisioner's plan in their values and says what changed; the mirror of the store
|
||||
// writes them only until the move.
|
||||
onTheBus := keepingAssignments(ctx, inv, announce, epoch, events, failing)
|
||||
for _, m := range recordmirror.Mirrors() {
|
||||
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
|
||||
if m.Name == recordmirror.Assignments.Name && onTheBus {
|
||||
continue
|
||||
}
|
||||
why := mirrorOnce(ctx, inv, api, m, writers, sources[m.Name], announce, epoch, holds)
|
||||
if why != failing[m.Name] {
|
||||
if why != "" {
|
||||
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
|
||||
@@ -72,7 +83,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
||||
}
|
||||
|
||||
// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
|
||||
func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
|
||||
func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
|
||||
source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
|
||||
holds func() bool) string {
|
||||
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
@@ -84,6 +95,9 @@ func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mir
|
||||
return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
|
||||
}
|
||||
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
|
||||
if m.Name == recordmirror.Assignments.Name {
|
||||
w.Holds = assignmentsHolds(inv, holds)
|
||||
}
|
||||
writers[m.Name] = w
|
||||
}
|
||||
want, err := source(reading)
|
||||
@@ -104,3 +118,88 @@ func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mir
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// assignmentEvents is the assignments as the bucket held them at the last look, to say what changed since.
|
||||
type assignmentEvents struct {
|
||||
known map[string]inventory.KeptValue
|
||||
}
|
||||
|
||||
// keepingAssignments is the tick's work for the assignments once they are on the bus: the provisioner's plan written into
|
||||
// the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last
|
||||
// look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet.
|
||||
func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64,
|
||||
events *assignmentEvents, failing map[string]string) bool {
|
||||
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
const name = "assignments"
|
||||
if _, on, err := inv.KeptAssignments(reading); err != nil || !on {
|
||||
if err != nil {
|
||||
// Not known to be off the bus: the store's mirror is not written over what the bus may hold.
|
||||
if failing[name] == "" {
|
||||
failing[name] = err.Error()
|
||||
fmt.Printf("the assignments cannot be read from the bus: %v\n", err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
if _, _, err := inv.KeepAssignmentPlans(reading); err != nil {
|
||||
if failing[name] != err.Error() {
|
||||
failing[name] = err.Error()
|
||||
fmt.Printf("the provisioner's plans are not kept in the assignments: %v\n", err)
|
||||
}
|
||||
} else if failing[name] != "" {
|
||||
fmt.Printf("the provisioner's plans are kept in the assignments again\n")
|
||||
failing[name] = ""
|
||||
}
|
||||
now, _, err := inv.KeptAssignments(reading)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
if events.known != nil {
|
||||
keys := make([]string, 0, len(now)+len(events.known))
|
||||
for key := range now {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
for key := range events.known {
|
||||
if _, still := now[key]; !still {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
was, had := events.known[key]
|
||||
is, has := now[key]
|
||||
e := recordmirror.Event{Mirror: recordmirror.Assignments.Name, Key: key, Revision: is.Revision, Epoch: epoch(),
|
||||
At: time.Now().UTC()}
|
||||
switch {
|
||||
case has && !had:
|
||||
e.Change = recordmirror.Added
|
||||
case has && had && was.Revision != is.Revision:
|
||||
e.Change, e.Fields = recordmirror.Changed, recordmirror.ChangedFields(was.Value, is.Value)
|
||||
case !has && had:
|
||||
e.Change, e.Revision = recordmirror.Removed, 0
|
||||
default:
|
||||
continue
|
||||
}
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for try := 0; try < 2; try++ {
|
||||
if err = announce.PublishSeatEvent(reading, broker.ControllerSeat, recordmirror.Assignments.Event, body); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
events.known = now
|
||||
return true
|
||||
}
|
||||
|
||||
// assignmentsHolds is the assignments mirror's leave to write: the lease's, and the assignments neither being switched to
|
||||
// the bus nor on it. Asked by the writer before every write and purge, so a pass that read its source before the switch
|
||||
// writes nothing after it (ADR 0408 §4).
|
||||
func assignmentsHolds(inv *inventory.Inventory, holds func() bool) func() bool {
|
||||
return func() bool { return holds() && inv.AssignmentsMirrorMayWrite() }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/recordmirror"
|
||||
)
|
||||
|
||||
// **A writer of the store's mirror of the assignments, made before the assignments are switched to the bus, writes and
|
||||
// purges nothing after** (ADR 0408 §4, issue 1093): its leave to write is asked before every write, and false once they are
|
||||
// switching or moved.
|
||||
func TestAMirrorWriterMadeBeforeTheSwitchWritesAndPurgesNothingAfter(t *testing.T) {
|
||||
inv := inventory.NewForTest()
|
||||
w := &recordmirror.Writer{Mirror: recordmirror.Assignments, Holds: assignmentsHolds(inv, func() bool { return true })}
|
||||
if !w.Holds() {
|
||||
t.Fatal("the writer may not write before the switch")
|
||||
}
|
||||
inv.BeginSwitchForTest(inventory.KindAssignments)
|
||||
if w.Holds() {
|
||||
t.Fatal("the writer may write while the assignments are switching")
|
||||
}
|
||||
// Refused before it reads the bucket (it has none here) or lays or ends a key.
|
||||
r, err := w.Sync(t.Context(), map[string][]byte{"anchor.notes": []byte(`{}`)})
|
||||
if !errors.Is(err, recordmirror.ErrNotHeld) || r.Written != 0 || r.Removed != 0 {
|
||||
t.Fatalf("the writer wrote after the switch: %+v %v", r, err)
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -1054,7 +1055,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
}
|
||||
sort.Strings(machines)
|
||||
sent, err := sendRollout(withScope(ctx, scope), open, machines)
|
||||
was := map[string]map[string]chain.Ref{}
|
||||
for _, n := range machines {
|
||||
was[n] = ranBefore(ctx, inv, n)
|
||||
}
|
||||
sent, err := sendRollout(withScope(sendingAs(ctx, p), scope), open, machines)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("sending %s to %s after tier %d: %w", strings.Join(rest, ", "),
|
||||
strings.Join(machines, ", "), p.Tier, err)
|
||||
@@ -1076,6 +1081,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
}
|
||||
noteSends(ctx, p.Tier, []string{n}, carried, now)
|
||||
noteChanges(p, n, carried, was[n], now)
|
||||
}
|
||||
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
||||
strings.Join(sent, ", "))
|
||||
@@ -1324,6 +1330,10 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
s := p.Modules[m]
|
||||
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
|
||||
}
|
||||
// What the assignments name before the send, for the rollout to say what it changed; and the send is made as this
|
||||
// rollout, so what is recorded of it names it (novox/hq ADR 0408, issue 1095).
|
||||
was := ranBefore(ctx, inv, node)
|
||||
ctx = sendingAs(ctx, p)
|
||||
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
||||
// all of it there.
|
||||
carried, sent, grants, err := gatedSend(ctx, open, node, owns)
|
||||
@@ -1350,6 +1360,9 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
}
|
||||
}
|
||||
noteSends(ctx, p.Tier, sent, modules, now)
|
||||
for _, n := range sent {
|
||||
noteChanges(p, n, modules, was, now)
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "),
|
||||
strings.Join(sent, ", "))
|
||||
@@ -2016,6 +2029,10 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
// The assignments it changed, each with the build and commit before and after (novox/hq ADR 0408, issue 1095).
|
||||
for _, line := range changedLines(p) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if *whatIf != "" {
|
||||
|
||||
@@ -132,6 +132,9 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
|
||||
}
|
||||
// And where config values are read once the move of config has run (issue 664): mesh-config_resolved.
|
||||
inv.ReadsConfigFrom(readConfigValues)
|
||||
// And the chain, opened when a kind that moved to the bus is first read (novox/hq ADR 0408): a command that reads none
|
||||
// has no use of a connection.
|
||||
inv.UseChainOpener(openTheChain)
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -303,6 +303,16 @@ func notNow(err error) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// unsaved marks the failure to save the plan a merge was added to, whatever it was: the merge is carried until that save
|
||||
// writes it (inventory.AddMerge), so the announcement is held and handed over again, never taken (novox/hq ADR 0407
|
||||
// decision 3, ADR 0408, issue 1092). The bus acknowledges it only after the save succeeded.
|
||||
func unsaved(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%w: the merge is not kept until its plan is saved: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
|
||||
// SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which
|
||||
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
|
||||
@@ -92,9 +92,47 @@ const (
|
||||
MirrorPredefined = "predefined"
|
||||
)
|
||||
|
||||
// ControllerMirrors are the local names of the controller's record mirrors, which a module may read.
|
||||
// The chain's buckets (novox/hq ADR 0408, issue 1089): builds, rollouts and assignments are one chain kept on the bus
|
||||
// alone, one bucket each, one key per record, written by the lease holder by compare-and-set. The assignments bucket is
|
||||
// AssignmentsBucket above, which keeps its name and its readers (ADR 0408 §3). Nothing is removed from them (§4), they keep
|
||||
// ChainHistory earlier revisions of every key (§5), and a value is never cut (§6).
|
||||
var (
|
||||
BuildsBucket = BucketName(ControllerSeat, ChainBuilds)
|
||||
RolloutsBucket = BucketName(ControllerSeat, ChainRollouts)
|
||||
)
|
||||
|
||||
// **The assignments bucket keeps the marker age (MirrorMarkerTTL) it was made with as a mirror.** A stream that allows
|
||||
// messages an age cannot be made to disallow them (the server refuses the update: "message TTL status can not be
|
||||
// disabled", nats-server stream.go), so asserting the bucket without the age would fail the controller's start on a
|
||||
// mesh that has the bucket. The chain never purges or removes a key (ADR 0408 §4), so no marker is ever written and the
|
||||
// age ends nothing; it is kept only so the stream's configuration stays one the server accepts.
|
||||
//
|
||||
// The chain buckets' local names, as a reader names them in `reads` after `mesh-controller.`.
|
||||
const (
|
||||
ChainBuilds = "builds"
|
||||
ChainRollouts = "rollouts"
|
||||
)
|
||||
|
||||
// The chain buckets' bounds. ChainHistory is the bus's largest history (ADR 0408 §5). A value is at most ChainValueBytes,
|
||||
// just under the server's largest message of a mebibyte, which also carries the subject and the headers (ADR 0408 §6):
|
||||
// the controller writes none larger and the move refuses to. The buckets' size is the arithmetic of the survey of
|
||||
// issue 981: 8585 builds of 1 to 40 KB, 1049 rollouts of about 2 KB a revision and 64 revisions, so 512 MiB for the two
|
||||
// that grow, and the assignments, 285 of under a kilobyte, 64 MiB.
|
||||
const (
|
||||
ChainHistory = 64
|
||||
ChainValueBytes = 1<<20 - 8<<10
|
||||
ChainBuildsBytes = 512 << 20
|
||||
ChainRolloutsBytes = 512 << 20
|
||||
ChainAssignmentBytes = 64 << 20
|
||||
)
|
||||
|
||||
// ChainBuckets are the three buckets of the chain: builds, rollouts and assignments.
|
||||
func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, AssignmentsBucket} }
|
||||
|
||||
// ControllerMirrors are the local names of the controller's buckets a module may read: the record mirrors and the
|
||||
// chain's buckets.
|
||||
func ControllerMirrors() []string {
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined}
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts}
|
||||
}
|
||||
|
||||
// The mirrors' caps (design 54 §5): one value per key, no age, 64 KiB a value, 64 MiB a bucket.
|
||||
@@ -151,14 +189,14 @@ func IsControllerBucket(bucket string) bool {
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
|
||||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
|
||||
bucket == DeliveryMergesBucket
|
||||
bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
|
||||
DeliveryMergesBucket}
|
||||
DeliveryMergesBucket, BuildsBucket, RolloutsBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
@@ -315,11 +353,36 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", DeliveryMergesBucket, err)
|
||||
}
|
||||
// **The chain: no age, no removal, history 64, no cut** (ADR 0408 §4 to §6). A key is never removed, so no purge
|
||||
// marker needs an age and a key that expired would be a record the bus forgot.
|
||||
for _, c := range []struct {
|
||||
bucket, why string
|
||||
bytes int64
|
||||
}{
|
||||
{BuildsBucket, "every build, one key each by its id, with its repository, commit, reuse key and what it " +
|
||||
"published (novox/hq ADR 0408, design 47): written by the controller alone, never removed, read through `reads`",
|
||||
ChainBuildsBytes},
|
||||
{RolloutsBucket, "every rollout, one key each by its id: the commit it fixed per repository, its tiers, the " +
|
||||
"builds it carried and the assignments it changed (novox/hq ADR 0408, design 47): written by the " +
|
||||
"controller alone, never removed, read through `reads`", ChainRolloutsBytes},
|
||||
} {
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, chainBucket(c.bucket, c.why, c.bytes, 0)); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", c.bucket, err)
|
||||
}
|
||||
}
|
||||
// **The assignments keep the marker age they had as a mirror.** A stream that allows messages an age cannot be
|
||||
// made to disallow them (the server refuses: "message TTL status can not be disabled"), so the bucket that was
|
||||
// the assignments mirror is asserted with the age it was made with. Nothing is purged from it, so no marker is ever
|
||||
// written and the age ends nothing.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, chainBucket(AssignmentsBucket,
|
||||
"which module is assigned to which node, keyed <node>.<module>, with the build, commit and rollout it runs "+
|
||||
"there (novox/hq ADR 0408, design 47): written by the controller alone, never removed, read through `reads`",
|
||||
ChainAssignmentBytes, MirrorMarkerTTL)); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", AssignmentsBucket, err)
|
||||
}
|
||||
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
|
||||
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
|
||||
for _, m := range []struct{ bucket, why string }{
|
||||
{AssignmentsBucket, "which module is assigned to which node, keyed <node>.<module> (novox/hq ADR 0289, " +
|
||||
"design 54): written by the controller alone from its store when an assignment changes, read through `reads`"},
|
||||
{DeclaredConfigBucket, "each registered module's config declarations as its manifest holds them, keyed by " +
|
||||
"the module (novox/hq ADR 0348, design 57 F3): written by the controller alone from its store when a " +
|
||||
"registration changes them, read through `reads`"},
|
||||
@@ -342,6 +405,20 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// chainBucket is the options of a bucket of the chain: sixty-four revisions of each key, no age, a value up to
|
||||
// ChainValueBytes, on disk.
|
||||
func chainBucket(bucket, why string, bytes int64, markers time.Duration) jetstream.KeyValueConfig {
|
||||
return jetstream.KeyValueConfig{
|
||||
Bucket: bucket,
|
||||
Description: why,
|
||||
History: ChainHistory,
|
||||
MaxValueSize: ChainValueBytes,
|
||||
MaxBytes: bytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
LimitMarkerTTL: markers,
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureLeaseBucket creates the lease's bucket if absent and brings its options to match (to-be 45
|
||||
// §6). **Before the lease is taken, by any candidate**: it is the lease's own precondition, and asserting
|
||||
// a bucket that exists changes nothing. File storage, so its revisions — the epochs — outlive a restart of
|
||||
|
||||
@@ -259,3 +259,52 @@ func TestAMergeTheOldConsumerHadNotAcknowledgedIsTheMergesConsumers(t *testing.T
|
||||
t.Fatalf("the merges consumer holds %v (%v); want the merge the old consumer never acknowledged", merges, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **The chain's buckets keep sixty-four revisions, the assignments' that were a mirror's included** (ADR 0408 §3 to §5,
|
||||
// issue 1089): a bucket made as a mirror — one revision, a marker age — is brought to the chain's options without the
|
||||
// server refusing the update, and a key written three times keeps three values.
|
||||
func TestTheChainBucketsKeepSixtyFourRevisionsOnTheBus(t *testing.T) {
|
||||
js, err := Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The bucket as the mirror made it.
|
||||
if _, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: AssignmentsBucket, History: 1,
|
||||
MaxValueSize: MirrorValueBytes, MaxBytes: MirrorBytes, Storage: jetstream.FileStorage,
|
||||
LimitMarkerTTL: MirrorMarkerTTL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatalf("a mirror's bucket is not brought to the chain's options: %v", err)
|
||||
}
|
||||
for _, name := range ChainBuckets() {
|
||||
kv, err := api.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := status.(*jetstream.KeyValueBucketStatus).StreamInfo().Config
|
||||
if status.History() != ChainHistory || status.TTL() != 0 || cfg.MaxMsgSize < 0 || cfg.MaxMsgSize != ChainValueBytes {
|
||||
t.Errorf("%s keeps %d revisions, age %s, a value of %d bytes", name, status.History(), status.TTL(), cfg.MaxMsgSize)
|
||||
}
|
||||
for _, value := range []string{"one", "two", "three"} {
|
||||
if _, err := kv.Put(ctx, "k", []byte(value)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
history, err := kv.History(ctx, "k")
|
||||
if err != nil || len(history) != 3 {
|
||||
t.Errorf("%s keeps %d values of a key written three times: %v", name, len(history), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a
|
||||
@@ -118,7 +120,7 @@ func TestTheMirrorsListAndTheWritersTableAgree(t *testing.T) {
|
||||
module := Principal{Kind: KindModule, Module: ControllerSeat, Node: "one"}
|
||||
rowed := map[string]bool{}
|
||||
for _, row := range WritersTable {
|
||||
if !strings.HasSuffix(row.State, " mirror") {
|
||||
if !strings.HasSuffix(row.State, " mirror") && !strings.HasSuffix(row.State, " of the chain") {
|
||||
continue
|
||||
}
|
||||
for _, subject := range row.Subjects {
|
||||
@@ -254,3 +256,52 @@ func TestTheControllerMayAskMeshConfigInItsOwnName(t *testing.T) {
|
||||
// **Delivering this change loses no merge** (review of issue 973): a merge published under the old filter of the
|
||||
// `controller` consumer and never acknowledged by it is delivered by the merges consumer once the assertion has
|
||||
// made it, because it starts where `controller` has acknowledged up to, not from now.
|
||||
|
||||
// **The chain's buckets keep sixty-four revisions and no age** (ADR 0408 §4 to §6, issue 1089): the bus's largest
|
||||
// history, no age that could forget a key, a value up to just under the server's largest message, on disk.
|
||||
func TestTheChainBucketsKeepSixtyFourRevisionsAndNoAge(t *testing.T) {
|
||||
if want := []string{"mesh-controller_builds", "mesh-controller_rollouts", "mesh-controller_assignments"}; !slices.Equal(ChainBuckets(), want) {
|
||||
t.Fatalf("the chain's buckets are %v, not %v", ChainBuckets(), want)
|
||||
}
|
||||
for _, bucket := range ChainBuckets() {
|
||||
cfg := chainBucket(bucket, "why", ChainBuildsBytes, 0)
|
||||
if cfg.History != 64 || cfg.TTL != 0 || cfg.Storage != jetstream.FileStorage {
|
||||
t.Errorf("%s keeps %d revisions, age %v, on %v", bucket, cfg.History, cfg.TTL, cfg.Storage)
|
||||
}
|
||||
if cfg.MaxValueSize <= MirrorValueBytes || cfg.MaxValueSize >= 1<<20 {
|
||||
t.Errorf("%s takes a value of %d bytes, not just under a mebibyte", bucket, cfg.MaxValueSize)
|
||||
}
|
||||
if cfg.Replicas > 1 {
|
||||
t.Errorf("%s is replicated %d times", bucket, cfg.Replicas)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Every bucket of the chain is the controller's, listed for a reader, and has a writer row** (ADR 0408 §3).
|
||||
func TestEveryChainBucketIsOnTheControllersListAndHasAWriterRow(t *testing.T) {
|
||||
controller := Principal{Kind: KindController}
|
||||
module := Principal{Kind: KindModule, Module: ControllerSeat, Node: "one"}
|
||||
for _, bucket := range ChainBuckets() {
|
||||
if !IsControllerBucket(bucket) || !slices.Contains(ControllerBuckets(), bucket) {
|
||||
t.Errorf("%s is not on the controller's list of buckets", bucket)
|
||||
}
|
||||
local := strings.TrimPrefix(bucket, ControllerSeat+"_")
|
||||
if !slices.Contains(ControllerMirrors(), local) {
|
||||
t.Errorf("a module cannot name %s in `reads`", bucket)
|
||||
}
|
||||
rows := 0
|
||||
for _, row := range WritersTable {
|
||||
for _, subject := range row.Subjects {
|
||||
if subject == "$KV."+bucket+".>" {
|
||||
rows++
|
||||
if row.Writes == nil || !row.Writes(controller, subject) || row.Writes(module, subject) {
|
||||
t.Errorf("%q is not written by the controller alone", row.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Errorf("%s has %d writer rows", bucket, rows)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -403,6 +403,16 @@ var DegradableGrants = []DegradableGrant{
|
||||
Reason: "refused: the controller then keeps no merge mesh-delivery announced, so the build of a moved module is " +
|
||||
"refused as it is without the operator's Approve — failing closed (ADR 0420 §7, issue 1025)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + BuildsBucket + ".>",
|
||||
Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
|
||||
"first push, and writes the chain only after that, holding the lease (ADR 0408 §2 and §8, issue 1089)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + RolloutsBucket + ".>",
|
||||
Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
|
||||
"first push, and writes the chain only after that, holding the lease (ADR 0408 §2 and §8, issue 1089)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + RolloutSilentBucket + ".>",
|
||||
Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " +
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -193,9 +193,18 @@ var WritersTable = []WriterRow{
|
||||
// The record mirrors (novox/hq ADR 0289 §9, design 54 §2): rows of this table, so a grant that lets
|
||||
// anything else write them is refused at composition. Built so far: the assignments and each registered
|
||||
// module's declared config (design 57 F3), and the list of predefined keys kept (design 57 K3).
|
||||
{State: "the assignments mirror", Writer: "controller (lease holder), from its store, only on a change",
|
||||
KeptIn: "key-value " + AssignmentsBucket, Others: "read through `reads` as mesh-controller." + MirrorAssignments,
|
||||
{State: "the assignments mirror", Writer: "controller (lease holder): the verbs that assign, and the sends that record what runs; " +
|
||||
"a mirror of its store only until the assignments are moved to the bus (novox/hq ADR 0408)",
|
||||
KeptIn: "key-value " + AssignmentsBucket + ", the record once moved", Others: "read through `reads` as mesh-controller." + MirrorAssignments,
|
||||
Subjects: kvOf(AssignmentsBucket), Writes: isController},
|
||||
// The chain (novox/hq ADR 0408): builds and rollouts beside the assignments above, which keep their bucket. Not
|
||||
// mirrors of the store but the record, written by the lease holder by compare-and-set and removed by nobody.
|
||||
{State: "the builds of the chain", Writer: "controller (lease holder), compare-and-set on the key's revision",
|
||||
KeptIn: "key-value " + BuildsBucket + ", sixty-four revisions of each key, none removed",
|
||||
Others: "read through `reads` as mesh-controller." + ChainBuilds, Subjects: kvOf(BuildsBucket), Writes: isController},
|
||||
{State: "the rollouts of the chain", Writer: "controller (lease holder), compare-and-set on the key's revision",
|
||||
KeptIn: "key-value " + RolloutsBucket + ", sixty-four revisions of each key, none removed",
|
||||
Others: "read through `reads` as mesh-controller." + ChainRollouts, Subjects: kvOf(RolloutsBucket), Writes: isController},
|
||||
{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
|
||||
KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
|
||||
Subjects: kvOf(DeclaredConfigBucket), Writes: isController},
|
||||
|
||||
@@ -38,6 +38,9 @@ var designRows = []string{
|
||||
"a message a consumer gave up on",
|
||||
// The record mirrors, rows of to-be 45 §1's table by design 54 §2 (ADR 0289 §9).
|
||||
"the assignments mirror",
|
||||
// The chain beside the assignments, the record itself and no mirror of one (ADR 0408, design 47, issue 1089).
|
||||
"the builds of the chain",
|
||||
"the rollouts of the chain",
|
||||
"the declared-config mirror",
|
||||
"the predefined-keys mirror",
|
||||
// The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6).
|
||||
|
||||
@@ -130,7 +130,7 @@ func StateProblems(m Manifest) []string {
|
||||
// controller's account alone (the writers table); declared as a module's state, they would be asserted a
|
||||
// second way and granted to the module's own principal as well. Named here so a read of one is a read of
|
||||
// kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing.
|
||||
var ControllerMirrors = []string{"assignments", "declared-config", "predefined"}
|
||||
var ControllerMirrors = []string{"assignments", "declared-config", "predefined", "builds", "rollouts"}
|
||||
|
||||
// StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no
|
||||
// such state. An absent owner says nothing — a module may be installed long before the one whose
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Entry is one key's value at a revision. Caught is set on the entry a watch sends once, after the values the bucket
|
||||
// held when it began, and carries no key.
|
||||
type Entry struct {
|
||||
Key string
|
||||
// Deleted marks the end of a key (Remove); it holds no value.
|
||||
Deleted bool
|
||||
Value []byte
|
||||
Revision uint64
|
||||
At time.Time
|
||||
Caught bool
|
||||
}
|
||||
|
||||
// Backend is one bucket of the chain. **It has no delete and no purge**: no key of the chain is ever removed (ADR
|
||||
// 0408 §4), so nothing that holds a Backend can remove one.
|
||||
type Backend interface {
|
||||
// Get is one key's value and revision; false when it holds none.
|
||||
Get(ctx context.Context, key string) (Entry, bool, error)
|
||||
// Create writes a key that holds nothing and fails with ErrExists when it holds something.
|
||||
Create(ctx context.Context, key string, value []byte) (uint64, error)
|
||||
// Update writes a key at the revision it was read at and fails with ErrMoved when it moved.
|
||||
Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error)
|
||||
// Keys is every key. An error is an error: never an empty bucket.
|
||||
Keys(ctx context.Context) ([]string, error)
|
||||
// Watch sends every key's current value, then a Caught entry, then each later write, until the context ends.
|
||||
Watch(ctx context.Context) (<-chan Entry, error)
|
||||
// History is a key's earlier values and its current one, oldest first, at most the bucket keeps.
|
||||
History(ctx context.Context, key string) ([]Entry, error)
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrMoved is a compare-and-set that lost: somebody wrote the key since it was read.
|
||||
ErrMoved = errors.New("the key was written by somebody else since it was read")
|
||||
// ErrExists is a create of a key that holds a value.
|
||||
ErrExists = errors.New("the key already holds a value")
|
||||
// ErrNotHeld is a write refused because this controller does not hold the lease.
|
||||
ErrNotHeld = errors.New("this controller does not hold the lease, so it writes nothing of the chain")
|
||||
// ErrTooLarge is a value over the largest the bus holds: never cut (ADR 0408 §6), so refused.
|
||||
ErrTooLarge = errors.New("the value is larger than the bus holds, and a value of the chain is never cut")
|
||||
)
|
||||
|
||||
// Memory is a Backend held in this process, for tests and for nothing else. It keeps what the bucket keeps: every
|
||||
// earlier value, up to History of them, and a revision that rises with every write of the bucket.
|
||||
type Memory struct {
|
||||
// Keeps is how many values of a key it keeps; zero keeps sixty-four.
|
||||
Keeps int
|
||||
|
||||
mu sync.Mutex
|
||||
values map[string][]Entry
|
||||
revision uint64
|
||||
watchers []*watcher
|
||||
// Now is the clock the entries are dated by; nil is time.Now.
|
||||
Now func() time.Time
|
||||
// writes counts the creates and updates taken.
|
||||
writes int
|
||||
// fail is what every call answers, when set.
|
||||
fail error
|
||||
}
|
||||
|
||||
// NewMemory is an empty bucket.
|
||||
func NewMemory() *Memory { return &Memory{values: map[string][]Entry{}} }
|
||||
|
||||
// SetFail makes every call answer err from now on, or none when err is nil: a bus that is away.
|
||||
func (m *Memory) SetFail(err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.fail = err
|
||||
}
|
||||
|
||||
// Writes is how many writes the bucket took.
|
||||
func (m *Memory) Writes() int {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.writes
|
||||
}
|
||||
|
||||
func (m *Memory) keep() int {
|
||||
if m.Keeps > 0 {
|
||||
return m.Keeps
|
||||
}
|
||||
return 64
|
||||
}
|
||||
|
||||
func (m *Memory) now() time.Time {
|
||||
if m.Now != nil {
|
||||
return m.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// last is a key's newest entry, false when it holds none.
|
||||
func (m *Memory) last(key string) (Entry, bool) {
|
||||
h := m.values[key]
|
||||
if len(h) == 0 {
|
||||
return Entry{}, false
|
||||
}
|
||||
return h[len(h)-1], true
|
||||
}
|
||||
|
||||
func (m *Memory) Get(_ context.Context, key string) (Entry, bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.fail != nil {
|
||||
return Entry{}, false, m.fail
|
||||
}
|
||||
e, ok := m.last(key)
|
||||
if !ok {
|
||||
return Entry{}, false, nil
|
||||
}
|
||||
return clone(e), true, nil
|
||||
}
|
||||
|
||||
func (m *Memory) Create(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.fail != nil {
|
||||
return 0, m.fail
|
||||
}
|
||||
if _, held := m.last(key); held {
|
||||
return 0, ErrExists
|
||||
}
|
||||
return m.write(key, value), nil
|
||||
}
|
||||
|
||||
func (m *Memory) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.fail != nil {
|
||||
return 0, m.fail
|
||||
}
|
||||
e, held := m.last(key)
|
||||
if !held || e.Revision != revision {
|
||||
return 0, ErrMoved
|
||||
}
|
||||
return m.write(key, value), nil
|
||||
}
|
||||
|
||||
// write is called with the lock held.
|
||||
func (m *Memory) write(key string, value []byte) uint64 {
|
||||
m.revision++
|
||||
m.writes++
|
||||
e := Entry{Key: key, Value: append([]byte(nil), value...), Revision: m.revision, At: m.now()}
|
||||
h := append(m.values[key], e)
|
||||
if len(h) > m.keep() {
|
||||
h = h[len(h)-m.keep():]
|
||||
}
|
||||
m.values[key] = h
|
||||
for _, w := range m.watchers {
|
||||
w.push(clone(e))
|
||||
}
|
||||
return e.Revision
|
||||
}
|
||||
|
||||
func (m *Memory) Keys(_ context.Context) ([]string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.fail != nil {
|
||||
return nil, m.fail
|
||||
}
|
||||
out := make([]string, 0, len(m.values))
|
||||
for k := range m.values {
|
||||
if _, held := m.last(k); held {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// watcher is one watch of a Memory: an unbounded queue, so a watch that is slow is behind and never misses a write.
|
||||
type watcher struct {
|
||||
mu sync.Mutex
|
||||
q []Entry
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
func (w *watcher) push(e Entry) {
|
||||
w.mu.Lock()
|
||||
w.q = append(w.q, e)
|
||||
w.mu.Unlock()
|
||||
select {
|
||||
case w.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (w *watcher) pop() (Entry, bool) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if len(w.q) == 0 {
|
||||
return Entry{}, false
|
||||
}
|
||||
e := w.q[0]
|
||||
w.q = w.q[1:]
|
||||
return e, true
|
||||
}
|
||||
|
||||
func (m *Memory) Watch(ctx context.Context) (<-chan Entry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.fail != nil {
|
||||
return nil, m.fail
|
||||
}
|
||||
w := &watcher{wake: make(chan struct{}, 1)}
|
||||
keys := make([]string, 0, len(m.values))
|
||||
for k := range m.values {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if e, held := m.last(k); held {
|
||||
w.q = append(w.q, clone(e))
|
||||
}
|
||||
}
|
||||
w.q = append(w.q, Entry{Caught: true})
|
||||
m.watchers = append(m.watchers, w)
|
||||
out := make(chan Entry)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer func() {
|
||||
m.mu.Lock()
|
||||
for i, x := range m.watchers {
|
||||
if x == w {
|
||||
m.watchers = append(m.watchers[:i], m.watchers[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}()
|
||||
for {
|
||||
e, ok := w.pop()
|
||||
if !ok {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-w.wake:
|
||||
}
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case out <- e:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Memory) History(_ context.Context, key string) ([]Entry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.fail != nil {
|
||||
return nil, m.fail
|
||||
}
|
||||
out := make([]Entry, 0, len(m.values[key]))
|
||||
for _, e := range m.values[key] {
|
||||
out = append(out, clone(e))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func clone(e Entry) Entry {
|
||||
e.Value = append([]byte(nil), e.Value...)
|
||||
return e
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// OnTheBus opens the chain's three buckets on a connection. The controller asserts them at its start
|
||||
// (broker.EnsureControllerBuckets), so one older than this has not, and the error says so.
|
||||
func OnTheBus(ctx context.Context, conn *nats.Conn, holds Holds) (*Chain, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
open := func(bucket string) (Backend, error) {
|
||||
kv, err := api.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s is not on the bus — the controller asserts it at its start, so one "+
|
||||
"older than this has not: %w", bucket, err)
|
||||
}
|
||||
return busBucket{kv}, nil
|
||||
}
|
||||
builds, err := open(broker.BuildsBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rollouts, err := open(broker.RolloutsBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
assignments, err := open(broker.AssignmentsBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(builds, rollouts, assignments, holds), nil
|
||||
}
|
||||
|
||||
// busBucket is a Backend on a key-value bucket. It calls no delete and no purge: the type has none to call (ADR 0408 §4).
|
||||
type busBucket struct{ kv jetstream.KeyValue }
|
||||
|
||||
func entryOf(e jetstream.KeyValueEntry) Entry {
|
||||
return Entry{Key: e.Key(), Value: e.Value(), Revision: e.Revision(), At: e.Created()}
|
||||
}
|
||||
|
||||
func (b busBucket) Get(ctx context.Context, key string) (Entry, bool, error) {
|
||||
e, err := b.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return Entry{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Entry{}, false, err
|
||||
}
|
||||
return entryOf(e), true, nil
|
||||
}
|
||||
|
||||
func (b busBucket) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
rev, err := b.kv.Create(ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return 0, ErrExists
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (b busBucket) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
rev, err := b.kv.Update(ctx, key, value, revision)
|
||||
var apiErr *jetstream.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence {
|
||||
return 0, ErrMoved
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (b busBucket) Keys(ctx context.Context) ([]string, error) {
|
||||
keys, err := b.kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
func (b busBucket) Watch(ctx context.Context) (<-chan Entry, error) {
|
||||
w, err := b.kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(chan Entry)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
v := Entry{Caught: true}
|
||||
if e != nil {
|
||||
v = entryOf(e)
|
||||
}
|
||||
select {
|
||||
case out <- v:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (b busBucket) History(ctx context.Context, key string) ([]Entry, error) {
|
||||
all, err := b.kv.History(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Entry, 0, len(all))
|
||||
for _, e := range all {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
continue // a marker an earlier mirror left; nothing of the chain makes one
|
||||
}
|
||||
out = append(out, entryOf(e))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Bucket is one kind of record of the chain, typed. Reads need nothing; **every write asks first whether this
|
||||
// controller holds the lease** and writes nothing when it does not (ADR 0408 §2).
|
||||
type Bucket[T any] struct {
|
||||
b Backend
|
||||
holds Holds
|
||||
}
|
||||
|
||||
// Holds says whether this controller may write now: nil when it holds the lease, and why not when it does not. The
|
||||
// serving controller answers from its lease, and a command from the holder's (the epoch gate).
|
||||
type Holds func(ctx context.Context) error
|
||||
|
||||
// Kept is a record as the bucket holds it: with its revision, the one a write must find.
|
||||
type Kept[T any] struct {
|
||||
Key string
|
||||
Value T
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// validKey is what the bus takes as a key: letters, digits and - _ / = . with no dot at either end.
|
||||
var validKey = regexp.MustCompile(`^[-/_=.a-zA-Z0-9]+$`)
|
||||
|
||||
// ErrBadKey is a key the bus does not take: refused loudly, never rewritten into another.
|
||||
var ErrBadKey = errors.New("the bus takes no key of that spelling")
|
||||
|
||||
// ValidKey says the bus takes the key: letters, digits and - _ / = . with no dot at either end.
|
||||
func ValidKey(key string) bool {
|
||||
return validKey.MatchString(key) && !strings.HasPrefix(key, ".") && !strings.HasSuffix(key, ".")
|
||||
}
|
||||
|
||||
func (s Bucket[T]) guard(ctx context.Context, key string, value []byte) error {
|
||||
if s.holds == nil {
|
||||
return ErrNotHeld
|
||||
}
|
||||
if err := s.holds(ctx); err != nil {
|
||||
return fmt.Errorf("%w: %v", ErrNotHeld, err)
|
||||
}
|
||||
if !ValidKey(key) {
|
||||
return fmt.Errorf("%w: %q", ErrBadKey, key)
|
||||
}
|
||||
if len(value) > broker.ChainValueBytes {
|
||||
return fmt.Errorf("%w: %d bytes, the most is %d", ErrTooLarge, len(value), broker.ChainValueBytes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decode[T any](e Entry) (Kept[T], error) {
|
||||
var v T
|
||||
if err := json.Unmarshal(e.Value, &v); err != nil {
|
||||
return Kept[T]{}, fmt.Errorf("the value of %s cannot be read: %w", e.Key, err)
|
||||
}
|
||||
return Kept[T]{Key: e.Key, Value: v, Revision: e.Revision}, nil
|
||||
}
|
||||
|
||||
// Get is one record; false when the key holds none.
|
||||
func (s Bucket[T]) Get(ctx context.Context, key string) (Kept[T], bool, error) {
|
||||
e, ok, err := s.b.Get(ctx, key)
|
||||
if err != nil || !ok {
|
||||
return Kept[T]{}, false, err
|
||||
}
|
||||
e.Key = key
|
||||
k, err := decode[T](e)
|
||||
return k, err == nil, err
|
||||
}
|
||||
|
||||
// Create writes a key that holds nothing: ErrExists when it holds something. It returns the revision.
|
||||
func (s Bucket[T]) Create(ctx context.Context, key string, v T) (uint64, error) {
|
||||
body, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := s.guard(ctx, key, body); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.b.Create(ctx, key, body)
|
||||
}
|
||||
|
||||
// Update writes a key at the revision it was read at: ErrMoved when it moved, the key unchanged.
|
||||
func (s Bucket[T]) Update(ctx context.Context, key string, v T, revision uint64) (uint64, error) {
|
||||
body, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := s.guard(ctx, key, body); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.b.Update(ctx, key, body, revision)
|
||||
}
|
||||
|
||||
// All is every record, by key, read in one pass of a watch. An error is an error: never an empty bucket.
|
||||
func (s Bucket[T]) All(ctx context.Context) (map[string]Kept[T], error) {
|
||||
ctx, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
w, err := s.b.Watch(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]Kept[T]{}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("reading the bucket: %w", ctx.Err())
|
||||
case e, open := <-w:
|
||||
if !open {
|
||||
return nil, errors.New("the bucket stopped answering before it was read whole")
|
||||
}
|
||||
if e.Caught {
|
||||
return out, nil
|
||||
}
|
||||
k, err := decode[T](e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[e.Key] = k
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// History is a key's earlier records and its current one, oldest first.
|
||||
func (s Bucket[T]) History(ctx context.Context, key string) ([]Kept[T], error) {
|
||||
all, err := s.b.History(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Kept[T], 0, len(all))
|
||||
for _, e := range all {
|
||||
e.Key = key
|
||||
k, err := decode[T](e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Keys is every key of the bucket.
|
||||
func (s Bucket[T]) Keys(ctx context.Context) ([]string, error) { return s.b.Keys(ctx) }
|
||||
|
||||
// Watch is the bucket's raw entries: its current values, a Caught mark, then each write.
|
||||
func (s Bucket[T]) Watch(ctx context.Context) (<-chan Entry, error) { return s.b.Watch(ctx) }
|
||||
|
||||
// GetRaw is one key's value as the bucket holds it, for a writer that changes a field of a record and must keep the rest
|
||||
// exactly as it is.
|
||||
func (s Bucket[T]) GetRaw(ctx context.Context, key string) (Entry, bool, error) {
|
||||
e, ok, err := s.b.Get(ctx, key)
|
||||
e.Key = key
|
||||
return e, ok, err
|
||||
}
|
||||
|
||||
// CreateRaw writes a key's first value as bytes, with the same guards as Create.
|
||||
func (s Bucket[T]) CreateRaw(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if err := s.guard(ctx, key, value); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.b.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
// UpdateRaw writes a key's value as bytes, at the revision it was read at, with the same guards as Update.
|
||||
func (s Bucket[T]) UpdateRaw(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if err := s.guard(ctx, key, value); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.b.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
// Chain is the three buckets of the chain.
|
||||
type Chain struct {
|
||||
Builds Bucket[Build]
|
||||
Rollouts Bucket[Rollout]
|
||||
Assignments Bucket[Assignment]
|
||||
|
||||
rollouts Backend
|
||||
holds Holds
|
||||
}
|
||||
|
||||
// RolloutsOf is the rollouts bucket typed as the controller's plan, which carries more than the chain's Rollout names:
|
||||
// the same keys and values, read and written as the plan.
|
||||
func RolloutsOf[T any](c *Chain) Bucket[T] { return Bucket[T]{c.rollouts, c.holds} }
|
||||
|
||||
// New is a chain over three backends, written only while holds says this controller may write. A nil holds writes nothing.
|
||||
func New(builds, rollouts, assignments Backend, holds Holds) *Chain {
|
||||
return &Chain{
|
||||
Builds: Bucket[Build]{builds, holds},
|
||||
Rollouts: Bucket[Rollout]{rollouts, holds},
|
||||
Assignments: Bucket[Assignment]{assignments, holds},
|
||||
rollouts: rollouts,
|
||||
holds: holds,
|
||||
}
|
||||
}
|
||||
|
||||
// InMemory is a chain held in this process, for tests, written whenever holds says so; nil always holds.
|
||||
func InMemory(holds Holds) *Chain {
|
||||
if holds == nil {
|
||||
holds = func(context.Context) error { return nil }
|
||||
}
|
||||
return New(NewMemory(), NewMemory(), NewMemory(), holds)
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
func at(h int) time.Time { return time.Date(2026, 10, 18, h, 0, 0, 0, time.UTC) }
|
||||
|
||||
// **A build is kept whole and read back with its reuse key and what it published** (ADR 0408 §1): its module,
|
||||
// repository, commit, reuse key, and an artifact by digest or a package release by name and version.
|
||||
func TestABuildIsKeptWholeAndReadBackWithItsReuseKeyAndWhatItPublished(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
artifact := Build{ID: "b1", Repository: "mesh-catalog", Module: "notes", Commit: "c1", ReuseKey: "rk-1",
|
||||
Published: &Published{Artifact: "sha256:aa"}, Asked: at(1), At: at(2),
|
||||
Made: []Artifact{{Name: "notes", Kind: "image", Reference: "sha256:aa"}}}
|
||||
pkg := Build{ID: "b2", Repository: "mesh-sdk", Module: "sdk", Commit: "c2", ReuseKey: "rk-2",
|
||||
Published: &Published{Package: &Package{Name: "mesh-sdk", Version: "1.4.0"}}}
|
||||
for _, b := range []Build{artifact, pkg} {
|
||||
if _, err := c.Builds.Create(ctx, b.ID, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, ok, err := c.Builds.Get(ctx, "b1")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("read b1: %v %v", ok, err)
|
||||
}
|
||||
if got.Value.Module != "notes" || got.Value.Repository != "mesh-catalog" || got.Value.Commit != "c1" ||
|
||||
got.Value.ReuseKey != "rk-1" || got.Value.Published.Artifact != "sha256:aa" || len(got.Value.Made) != 1 {
|
||||
t.Fatalf("the build came back as %+v", got.Value)
|
||||
}
|
||||
got, _, _ = c.Builds.Get(ctx, "b2")
|
||||
if p := got.Value.Published; p == nil || p.Package == nil || p.Package.Name != "mesh-sdk" || p.Package.Version != "1.4.0" {
|
||||
t.Fatalf("the package came back as %+v", p)
|
||||
}
|
||||
if _, ok, _ := c.Builds.Get(ctx, "nope"); ok {
|
||||
t.Fatal("a key nobody wrote holds a build")
|
||||
}
|
||||
}
|
||||
|
||||
// **A write against a revision that moved is refused and nothing is laid over it** (ADR 0408 §2).
|
||||
func TestAWriteAgainstAMovedRevisionIsRefusedAndNothingIsLaidOver(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes"}); !errors.Is(err, ErrExists) {
|
||||
t.Fatalf("a create of a key that holds a value answered %v", err)
|
||||
}
|
||||
second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c2"}, first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A writer that read the first revision writes now.
|
||||
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "stale"}, first); !errors.Is(err, ErrMoved) {
|
||||
t.Fatalf("a write at a moved revision answered %v", err)
|
||||
}
|
||||
got, _, _ := c.Assignments.Get(ctx, "anchor.notes")
|
||||
if got.Value.Commit != "c2" || got.Revision != second {
|
||||
t.Fatalf("the key that moved was laid over: %+v at %d", got.Value, got.Revision)
|
||||
}
|
||||
}
|
||||
|
||||
// **No key of the chain is ever removed** (ADR 0408 §4): the Backend has no delete and no purge, so nothing that holds
|
||||
// one can remove a key; and an assignment of a module taken off its node is kept, with its earlier values.
|
||||
func TestNoKeyOfTheChainIsEverRemoved(t *testing.T) {
|
||||
var backend Backend = NewMemory()
|
||||
if _, ok := backend.(interface {
|
||||
Delete(context.Context, string, uint64) error
|
||||
}); ok {
|
||||
t.Fatal("the backend can delete")
|
||||
}
|
||||
if _, ok := backend.(interface {
|
||||
Purge(context.Context, string) error
|
||||
}); ok {
|
||||
t.Fatal("the backend can purge")
|
||||
}
|
||||
if _, ok := any(InMemory(nil).Builds).(interface {
|
||||
Delete(context.Context, string) error
|
||||
}); ok {
|
||||
t.Fatal("a bucket of the chain can delete")
|
||||
}
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"})
|
||||
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys, err := c.Assignments.Keys(ctx)
|
||||
if err != nil || len(keys) != 1 || keys[0] != "anchor.notes" {
|
||||
t.Fatalf("keys %v %v", keys, err)
|
||||
}
|
||||
history, err := c.Assignments.History(ctx, "anchor.notes")
|
||||
if err != nil || len(history) != 2 {
|
||||
t.Fatalf("the key keeps %d values: %v", len(history), err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Each key keeps up to sixty-four earlier values** (ADR 0408 §5), oldest first.
|
||||
func TestAKeyKeepsItsEarlierValuesUpToSixtyFour(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
rev, _ := c.Rollouts.Create(ctx, "r1", Rollout{ID: "r1", State: "n0"})
|
||||
for i := 1; i <= 70; i++ {
|
||||
var err error
|
||||
if rev, err = c.Rollouts.Update(ctx, "r1", Rollout{ID: "r1", State: "n" + string(rune('A'+i%26))}, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
history, err := c.Rollouts.History(ctx, "r1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(history) != broker.ChainHistory {
|
||||
t.Fatalf("the key keeps %d values and the bus keeps %d", len(history), broker.ChainHistory)
|
||||
}
|
||||
for i := 1; i < len(history); i++ {
|
||||
if history[i].Revision <= history[i-1].Revision {
|
||||
t.Fatal("the history is not oldest first")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A value over the largest message is refused, not cut, and the key is unchanged** (ADR 0408 §6).
|
||||
func TestAValueOverTheLargestMessageIsRefusedNotCut(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
small := Build{ID: "b1", Repository: "r", Commit: "c"}
|
||||
rev, err := c.Builds.Create(ctx, "b1", small)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
huge := Build{ID: "b1", Repository: "r", Commit: "c", Manifest: json.RawMessage(`"` + strings.Repeat("x", 1<<20) + `"`)}
|
||||
if _, err := c.Builds.Update(ctx, "b1", huge, rev); !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatalf("a value of a mebibyte answered %v", err)
|
||||
}
|
||||
if _, err := c.Builds.Create(ctx, "b2", huge); !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatalf("a created value of a mebibyte answered %v", err)
|
||||
}
|
||||
got, _, _ := c.Builds.Get(ctx, "b1")
|
||||
if got.Revision != rev || len(got.Value.Manifest) != 0 {
|
||||
t.Fatalf("the key was changed: %+v", got)
|
||||
}
|
||||
if _, ok, _ := c.Builds.Get(ctx, "b2"); ok {
|
||||
t.Fatal("a refused create left a key")
|
||||
}
|
||||
// The largest that fits is written whole.
|
||||
fits := Build{ID: "b3", Manifest: json.RawMessage(`"` + strings.Repeat("x", broker.ChainValueBytes-1024) + `"`)}
|
||||
if _, err := c.Builds.Create(ctx, "b3", fits); err != nil {
|
||||
t.Fatalf("a value under the largest was refused: %v", err)
|
||||
}
|
||||
got, _, _ = c.Builds.Get(ctx, "b3")
|
||||
if len(got.Value.Manifest) != len(fits.Manifest) {
|
||||
t.Fatal("a value under the largest was cut")
|
||||
}
|
||||
}
|
||||
|
||||
// **A rollout names every assignment it changed, with the build before and after** (ADR 0408 §1).
|
||||
func TestARolloutNamesEveryAssignmentItChangedWithBuildBeforeAndAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
r := Rollout{ID: "r1", State: "rolling", Commits: []Commit{{Repository: "mesh-catalog", Commit: "c2"}},
|
||||
Tiers: [][]string{{"notes"}}, Links: Links{Builds: []string{"b2"}, Changed: []Change{
|
||||
{Node: "anchor", Module: "notes", Before: Ref{Build: "b1", Commit: "c1"}, After: Ref{Build: "b2", Commit: "c2"}},
|
||||
{Node: "g14", Module: "notes", After: Ref{Build: "b2", Commit: "c2"}}}}}
|
||||
if _, err := c.Rollouts.Create(ctx, "r1", r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := c.Rollouts.Get(ctx, "r1")
|
||||
if len(got.Value.Changed) != 2 || got.Value.Changed[0].Before.Build != "b1" || got.Value.Changed[0].After.Build != "b2" ||
|
||||
got.Value.Changed[1].Before.Build != "" || got.Value.Changed[1].After.Commit != "c2" {
|
||||
t.Fatalf("the rollout came back as %+v", got.Value)
|
||||
}
|
||||
// And in the contract's words: `changed`, with `before` and `after` naming `build` and `commit`.
|
||||
raw, _ := json.Marshal(got.Value)
|
||||
for _, want := range []string{`"changed":[`, `"before":{"build":"b1","commit":"c1"}`, `"after":{"build":"b2","commit":"c2"}`,
|
||||
`"builds":["b2"]`, `"commits":[`, `"tiers":[["notes"]]`} {
|
||||
if !strings.Contains(string(raw), want) {
|
||||
t.Errorf("the rollout's value lacks %s: %s", want, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **An assignment names its build, artifact or package, commit and rollout**, and keeps the fields its readers read
|
||||
// under the names they read them by (ADR 0408 §3).
|
||||
func TestAnAssignmentNamesBuildArtifactCommitAndRollout(t *testing.T) {
|
||||
a := Assignment{Node: "anchor", Module: "notes", Assigned: at(3), RequiredBy: "wiki",
|
||||
Provisioning: json.RawMessage(`{"plan":1}`), Build: "b2", Artifact: "sha256:bb", Commit: "c2", Rollout: "r1", Since: at(4)}
|
||||
raw, _ := json.Marshal(a)
|
||||
for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"assigned":"2026-10-18T03:00:00Z"`,
|
||||
`"required-by":"wiki"`, `"provisioning":{"plan":1}`, `"build":"b2"`, `"artifact":"sha256:bb"`, `"commit":"c2"`,
|
||||
`"rollout":"r1"`, `"since":"2026-10-18T04:00:00Z"`} {
|
||||
if !strings.Contains(string(raw), want) {
|
||||
t.Errorf("the assignment's value lacks %s: %s", want, raw)
|
||||
}
|
||||
}
|
||||
if a.Key() != "anchor.notes" {
|
||||
t.Errorf("key %s", a.Key())
|
||||
}
|
||||
// An assignment of today carries none of the new fields, and says none: a reader of today reads the bytes it did.
|
||||
raw, _ = json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at(3)})
|
||||
if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` {
|
||||
t.Errorf("an assignment without a chain is %s", raw)
|
||||
}
|
||||
// A package in place of an artifact.
|
||||
raw, _ = json.Marshal(Assignment{Node: "a", Module: "m", Package: &Package{Name: "n", Version: "1"}})
|
||||
if !strings.Contains(string(raw), `"package":{"name":"n","version":"1"}`) {
|
||||
t.Errorf("the package is %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// **The writer without the lease writes nothing** (ADR 0408 §2): it reads, and every create and update is refused with
|
||||
// the bucket untouched.
|
||||
func TestTheWriterWithoutTheLeaseWritesNothing(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var away error
|
||||
builds, rollouts, assignments := NewMemory(), NewMemory(), NewMemory()
|
||||
c := New(builds, rollouts, assignments, func(context.Context) error { return away })
|
||||
rev, err := c.Builds.Create(ctx, "b1", Build{ID: "b1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
away = errors.New("another controller holds the lease")
|
||||
if _, err := c.Builds.Create(ctx, "b2", Build{ID: "b2"}); !errors.Is(err, ErrNotHeld) {
|
||||
t.Fatalf("a create without the lease answered %v", err)
|
||||
}
|
||||
if _, err := c.Builds.Update(ctx, "b1", Build{ID: "b1", Failed: "x"}, rev); !errors.Is(err, ErrNotHeld) {
|
||||
t.Fatalf("an update without the lease answered %v", err)
|
||||
}
|
||||
if _, err := c.Rollouts.Create(ctx, "r", Rollout{ID: "r"}); !errors.Is(err, ErrNotHeld) {
|
||||
t.Fatalf("a rollout written without the lease: %v", err)
|
||||
}
|
||||
if _, err := c.Assignments.Create(ctx, "a.b", Assignment{}); !errors.Is(err, ErrNotHeld) {
|
||||
t.Fatalf("an assignment written without the lease: %v", err)
|
||||
}
|
||||
if builds.Writes() != 1 || rollouts.Writes() != 0 || assignments.Writes() != 0 {
|
||||
t.Fatalf("writes %d %d %d", builds.Writes(), rollouts.Writes(), assignments.Writes())
|
||||
}
|
||||
// Reads still answer.
|
||||
if _, ok, err := c.Builds.Get(ctx, "b1"); err != nil || !ok {
|
||||
t.Fatalf("a read without the lease: %v %v", ok, err)
|
||||
}
|
||||
if all, err := c.Builds.All(ctx); err != nil || len(all) != 1 {
|
||||
t.Fatalf("all: %v %v", all, err)
|
||||
}
|
||||
// And a controller with no lease function at all holds nothing.
|
||||
if _, err := New(NewMemory(), NewMemory(), NewMemory(), nil).Builds.Create(ctx, "b", Build{}); !errors.Is(err, ErrNotHeld) {
|
||||
t.Fatalf("a chain with no lease wrote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A bucket that is away is an error, never an empty bucket** (ADR 0227 rule 4).
|
||||
func TestABucketThatIsAwayIsAnErrorNotAnEmptyOne(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m := NewMemory()
|
||||
c := New(m, NewMemory(), NewMemory(), func(context.Context) error { return nil })
|
||||
if _, err := c.Builds.Create(ctx, "b1", Build{ID: "b1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
away := errors.New("the bus is away")
|
||||
m.SetFail(away)
|
||||
if all, err := c.Builds.All(ctx); !errors.Is(err, away) || all != nil {
|
||||
t.Fatalf("all answered %v %v", all, err)
|
||||
}
|
||||
if _, _, err := c.Builds.Get(ctx, "b1"); !errors.Is(err, away) {
|
||||
t.Fatalf("get answered %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A watch hands over every key's value, says it has, and then each write.
|
||||
func TestAWatchHandsOverTheValuesThenEachWrite(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
c := InMemory(nil)
|
||||
if _, err := c.Builds.Create(ctx, "b1", Build{ID: "b1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w, err := c.Builds.Watch(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := func() Entry {
|
||||
select {
|
||||
case e := <-w:
|
||||
return e
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the watch said nothing")
|
||||
return Entry{}
|
||||
}
|
||||
}
|
||||
if e := next(); e.Key != "b1" || e.Caught {
|
||||
t.Fatalf("first %+v", e)
|
||||
}
|
||||
if e := next(); !e.Caught {
|
||||
t.Fatalf("second %+v", e)
|
||||
}
|
||||
if _, err := c.Builds.Create(ctx, "b2", Build{ID: "b2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e := next(); e.Key != "b2" {
|
||||
t.Fatalf("third %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
// **A key the bus does not take is refused, never rewritten into another** (ADR 0408 §2): a write names the key loudly.
|
||||
func TestAKeyTheBusTakesNoSpellingOfIsRefused(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
for _, key := range []string{"", "build one", "build*", ".hidden", "trailing.", "a>b"} {
|
||||
if _, err := c.Builds.Create(ctx, key, Build{ID: key}); !errors.Is(err, ErrBadKey) {
|
||||
t.Errorf("the key %q answered %v", key, err)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"build-1700000000000000000", "anchor.notes", "a/b_c=d"} {
|
||||
if _, err := c.Builds.Create(ctx, key, Build{ID: key}); err != nil {
|
||||
t.Errorf("the key %q answered %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An empty bucket is read as empty, and the read ends.
|
||||
func TestAnEmptyBucketIsReadAsEmpty(t *testing.T) {
|
||||
all, err := InMemory(nil).Assignments.All(context.Background())
|
||||
if err != nil || len(all) != 0 {
|
||||
t.Fatalf("%v %v", all, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **An assignment that has ended still stands in the bucket as a value that says so, and nothing in the chain can remove a
|
||||
// key** (ADR 0408 §4, issue 1093): the value with `ended` is no assignment to a reader, and the earlier value stays in
|
||||
// history.
|
||||
func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), Commit: "c1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
standing, _, _ := c.Assignments.Get(ctx, "anchor.notes")
|
||||
if !standing.Value.Stands() {
|
||||
t.Fatal("a value that has not ended does not stand")
|
||||
}
|
||||
ended := standing.Value
|
||||
ended.Ended = at(2)
|
||||
if _, err := c.Assignments.Update(ctx, "anchor.notes", ended, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, found, _ := c.Assignments.Get(ctx, "anchor.notes")
|
||||
if !found || got.Value.Stands() || got.Value.Commit != "c1" || got.Value.Ended.IsZero() {
|
||||
t.Fatalf("the ended value is %+v (found %v)", got.Value, found)
|
||||
}
|
||||
if history, _ := c.Assignments.History(ctx, "anchor.notes"); len(history) != 2 || !history[0].Value.Stands() {
|
||||
t.Fatalf("the earlier values are %+v", history)
|
||||
}
|
||||
if (Assignment{Node: "a", Module: "m", Assigned: at(1), State: "pending"}).Stands() {
|
||||
t.Fatal("a pending value stands")
|
||||
}
|
||||
raw, _ := json.Marshal(ended)
|
||||
if !strings.Contains(string(raw), `"ended":"2026-10-18T02:00:00Z"`) {
|
||||
t.Fatalf("the ended value is %s", raw)
|
||||
}
|
||||
// And a value that stands says nothing of either, so a reader of today reads the bytes it did.
|
||||
raw, _ = json.Marshal(Assignment{Node: "a", Module: "m", Assigned: at(1)})
|
||||
if strings.Contains(string(raw), "ended") || strings.Contains(string(raw), "state") {
|
||||
t.Fatalf("a standing value says %s", raw)
|
||||
}
|
||||
// No bucket, and no chain, has a removal.
|
||||
type remover interface {
|
||||
Remove(context.Context, string, uint64) error
|
||||
}
|
||||
for name, x := range map[string]any{"builds": c.Builds, "rollouts": c.Rollouts, "assignments": c.Assignments, "chain": c,
|
||||
"a backend": NewMemory()} {
|
||||
if _, has := x.(remover); has {
|
||||
t.Errorf("%s can remove a key", name)
|
||||
}
|
||||
if _, has := x.(interface {
|
||||
RemoveAssignment(context.Context, string, uint64) error
|
||||
}); has {
|
||||
t.Errorf("%s can remove an assignment", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
// Package chain keeps the controller's chain on the bus alone (novox/hq ADR 0408, design 47): module, commit, build,
|
||||
// published artifact or package, rollout and assignment are one chain, a bucket each of the bus's key-value store,
|
||||
// each the only source of its kind, written by the controller that holds the lease by compare-and-set.
|
||||
//
|
||||
// **No key is removed.** A Backend has no delete and no purge (ADR 0408 §4): an assignment of a module taken off its
|
||||
// node is kept, saying so. **A value is never cut** (§6): one over the largest the bus holds is refused, and the key
|
||||
// stays as it was. **Each key keeps its earlier values** (§5), read back with History.
|
||||
package chain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Ref names a build and the commit it was made from: one end of a change.
|
||||
type Ref struct {
|
||||
Build string `json:"build,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
}
|
||||
|
||||
// Package is a release a build published to the package registry.
|
||||
type Package struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// Published is what a build published: an artifact by its digest, or a package release by name and version.
|
||||
type Published struct {
|
||||
Artifact string `json:"artifact,omitempty"`
|
||||
Package *Package `json:"package,omitempty"`
|
||||
}
|
||||
|
||||
// Artifact is one thing a build made.
|
||||
type Artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
|
||||
// ReadRepository is a repository a build read source from besides the module's own.
|
||||
type ReadRepository struct {
|
||||
Repository string `json:"repository"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
Own bool `json:"own,omitempty"`
|
||||
}
|
||||
|
||||
// Source is the build source a build read in one repository.
|
||||
type Source struct {
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
|
||||
// The states of a build record. A record in state asked is a request that no outcome has answered (issue 1091); one
|
||||
// without a state is an outcome, and keeps the Request it was asked with.
|
||||
const BuildAsked = "asked"
|
||||
|
||||
// The states of a request's Reask.
|
||||
const (
|
||||
ReaskPending = "pending"
|
||||
ReaskClaimed = "claimed"
|
||||
)
|
||||
|
||||
// Request is a build as it was asked for (novox/hq issue 325, ADR 0408 §1): kept in the build's own record under its id,
|
||||
// first alone as state asked and then beside the outcome that answered it.
|
||||
type Request struct {
|
||||
// Repository and Seat are the source as the mesh spells it; Path is the module's directory in it, Ref the branch or
|
||||
// commit asked, and Commit the commit that made the ask, where one did.
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Seat string `json:"seat,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
// For says who asked: merge, plan, build (a person), assign.
|
||||
For string `json:"for,omitempty"`
|
||||
// NotAsked is why the ask could not be handed over; OutcomeUnknown why an asker that handed it over stopped waiting.
|
||||
NotAsked string `json:"not-asked,omitempty"`
|
||||
OutcomeUnknown string `json:"outcome-unknown,omitempty"`
|
||||
At time.Time `json:"at,omitzero"`
|
||||
AtTerminal bool `json:"at-terminal,omitempty"`
|
||||
// Reask is, on a request the move found never handed over, "pending" until the controller asks it again, "claimed"
|
||||
// while it does, and the build id it was asked as once it has: published once, never twice (issue 981 note 7 Q5).
|
||||
Reask string `json:"reask,omitempty"`
|
||||
}
|
||||
|
||||
// Build is one attempt, whichever way it went, kept under its id. It names its module, its repository and commit, its
|
||||
// reuse key (novox/hq ADR 0350) and what it published (ADR 0408 §1).
|
||||
type Build struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state,omitempty"`
|
||||
Repository string `json:"repository"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
On string `json:"on,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Manifest json.RawMessage `json:"manifest,omitempty"`
|
||||
Against []string `json:"against,omitempty"`
|
||||
Mirrored []string `json:"mirrored,omitempty"`
|
||||
Read []ReadRepository `json:"read,omitempty"`
|
||||
Sources []Source `json:"sources,omitempty"`
|
||||
SourceFingerprint string `json:"source-fingerprint,omitempty"`
|
||||
Failed string `json:"failed,omitempty"`
|
||||
Made []Artifact `json:"made,omitempty"`
|
||||
Asked time.Time `json:"asked,omitzero"`
|
||||
At time.Time `json:"at,omitzero"`
|
||||
ReuseKey string `json:"reuse-key,omitempty"`
|
||||
Published *Published `json:"published,omitempty"`
|
||||
Request *Request `json:"request,omitempty"`
|
||||
}
|
||||
|
||||
// Commit is one repository's commit a rollout carries.
|
||||
type Commit struct {
|
||||
Repository string `json:"repository"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Commit string `json:"commit"`
|
||||
Merged time.Time `json:"merged,omitzero"`
|
||||
}
|
||||
|
||||
// Change is one assignment a rollout changed: the build and commit it ran before and the ones it runs after.
|
||||
type Change struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Before Ref `json:"before"`
|
||||
After Ref `json:"after"`
|
||||
}
|
||||
|
||||
// Step is one tier sent to one machine.
|
||||
type Step struct {
|
||||
Tier int `json:"tier"`
|
||||
Node string `json:"node"`
|
||||
Sent *time.Time `json:"sent,omitempty"`
|
||||
Reported *time.Time `json:"reported,omitempty"`
|
||||
}
|
||||
|
||||
// Links are the fields a rollout gains on the chain: the builds it carried, the machine steps it has taken and the
|
||||
// assignments it changed, each with the build before and after. The controller's plan embeds them, so the plan's own
|
||||
// fields and these are one value (ADR 0408 §1).
|
||||
type Links struct {
|
||||
Builds []string `json:"builds,omitempty"`
|
||||
Steps []Step `json:"steps,omitempty"`
|
||||
Changed []Change `json:"changed,omitempty"`
|
||||
}
|
||||
|
||||
// HeldMerge is a merge a rollout holds: the columns the controller's table of batched merges had, kept in the record of the
|
||||
// batch or walk that answers it (novox/hq ADR 0276, issue 1092). A merge no plan answers (Waiting) stays in the record
|
||||
// that held it last. Placed is when it entered this record, so one found in two records after a crash between the two
|
||||
// writes is read from the one it entered last.
|
||||
type HeldMerge struct {
|
||||
Repository string `json:"repository"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Commit string `json:"commit"`
|
||||
Merged time.Time `json:"merged,omitzero"`
|
||||
Heard time.Time `json:"heard,omitzero"`
|
||||
Event json.RawMessage `json:"event,omitempty"`
|
||||
Alone bool `json:"alone,omitempty"`
|
||||
Waiting bool `json:"waiting,omitempty"`
|
||||
Placed time.Time `json:"placed,omitzero"`
|
||||
}
|
||||
|
||||
// Rollout is the controller's record of one batch's delivery, kept under the batch's id: the commit it fixed per
|
||||
// repository, its tiers, the builds it carried and the assignments it changed (ADR 0408 §1). The controller's plan
|
||||
// carries more (its modules, its times, its backups) beside these, under the same key.
|
||||
type Rollout struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state,omitempty"`
|
||||
Commits []Commit `json:"commits,omitempty"`
|
||||
Tiers [][]string `json:"tiers,omitempty"`
|
||||
Links
|
||||
}
|
||||
|
||||
// Assignment is one module on one node, kept under `<node>.<module>`. The first five fields are the ones its readers
|
||||
// have always read, with the names they have always had (novox/hq ADR 0408 §3); the rest name the chain: the build, the
|
||||
// artifact (by digest) or the package it runs, its commit, the rollout that put it there and since when.
|
||||
type Assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Assigned time.Time `json:"assigned,omitzero"`
|
||||
RequiredBy string `json:"required-by,omitempty"`
|
||||
Provisioning json.RawMessage `json:"provisioning,omitempty"`
|
||||
|
||||
Build string `json:"build,omitempty"`
|
||||
Artifact string `json:"artifact,omitempty"`
|
||||
Package *Package `json:"package,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Rollout string `json:"rollout,omitempty"`
|
||||
Since time.Time `json:"since,omitzero"`
|
||||
|
||||
// Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed**
|
||||
// (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a
|
||||
// State, as no assignment. State is, in a later delivery, the pending assignment waiting for its build (issue 1094).
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
State string `json:"state,omitempty"`
|
||||
}
|
||||
|
||||
// Stands says the value is an assignment that holds: it has not ended and is not waiting in a state of its own.
|
||||
func (a Assignment) Stands() bool { return a.Ended.IsZero() && a.State == "" }
|
||||
|
||||
// Key is an assignment's key.
|
||||
func (a Assignment) Key() string { return a.Node + "." + a.Module }
|
||||
@@ -66,16 +66,14 @@ func (i *Inventory) Take(ctx context.Context, nodeName, module string) error {
|
||||
// take is Take without the adopted check, for converging, which takes every assigned module in
|
||||
// the same act that makes the node converged.
|
||||
func (i *Inventory) take(ctx context.Context, node Node, module string) error {
|
||||
var assigned bool
|
||||
if err := i.store.Pool().QueryRow(ctx,
|
||||
`select exists (select 1 from assignment where node = $1 and module = $2)`,
|
||||
node.ID, module).Scan(&assigned); err != nil {
|
||||
assigned, err := i.isAssigned(ctx, node.Name, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !assigned {
|
||||
return fmt.Errorf("%w: %s is not on %s; assign it first", ErrNotAssigned, module, node.Name)
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`insert into taken (node, module) values ($1, $2) on conflict do nothing`, node.ID, module)
|
||||
return err
|
||||
}
|
||||
@@ -95,27 +93,24 @@ func (i *Inventory) Converge(ctx context.Context, nodeName string) ([]string, er
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
rows, err := tx.Query(ctx,
|
||||
`insert into taken (node, module)
|
||||
select node, module from assignment where node = $1
|
||||
on conflict do nothing
|
||||
returning module`, node.ID)
|
||||
assigned, err := i.assignedModules(ctx, node.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var took []string
|
||||
for rows.Next() {
|
||||
for _, module := range assigned {
|
||||
var m string
|
||||
if err := rows.Scan(&m); err != nil {
|
||||
rows.Close()
|
||||
err := tx.QueryRow(ctx,
|
||||
`insert into taken (node, module) values ($1, $2) on conflict do nothing returning module`,
|
||||
node.ID, module).Scan(&m)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
continue // taken already
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
took = append(took, m)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`update node set adopted = false, adopted_since = null, converged_at = now(),
|
||||
held = null, reachable = null, firewall = null, adoption_reported = null
|
||||
|
||||
@@ -33,6 +33,11 @@ type BatchedMerge struct {
|
||||
|
||||
// AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan.
|
||||
func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return false, err
|
||||
} else if on {
|
||||
return i.chainAddMerge(ctx, c, m)
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`insert into batched_merge (repository, branch, commit_hash, merged_at, heard_at, event, plan_id, alone)
|
||||
values (lower($1), $2, $3, $4, $5, $6, nullif($7, ''), $8) on conflict do nothing`,
|
||||
@@ -45,6 +50,11 @@ func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error)
|
||||
|
||||
// MergeOf is one kept merge; false when it was never heard.
|
||||
func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return BatchedMerge{}, false, err
|
||||
} else if on {
|
||||
return i.chainMergeOf(ctx, c, repository, commit)
|
||||
}
|
||||
ms, err := i.merges(ctx, `where repository = lower($1) and commit_hash = $2`, repository, commit)
|
||||
if err != nil || len(ms) == 0 {
|
||||
return BatchedMerge{}, false, err
|
||||
@@ -54,12 +64,22 @@ func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (Bat
|
||||
|
||||
// MergesOf is every merge a plan answers, oldest merge first.
|
||||
func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainMergesOf(ctx, c, plan)
|
||||
}
|
||||
return i.merges(ctx, `where plan_id = $1 order by merged_at nulls last, heard_at`, plan)
|
||||
}
|
||||
|
||||
// LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the
|
||||
// newest first.
|
||||
func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainLaterMergesOf(ctx, c, repository, branch, after)
|
||||
}
|
||||
return i.merges(ctx, `where repository = lower($1) and branch = $2 and merged_at > $3 and plan_id is not null
|
||||
order by merged_at desc`, repository, branch, after)
|
||||
}
|
||||
@@ -67,6 +87,11 @@ func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string
|
||||
// KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a
|
||||
// module's last look says (novox/hq ADR 0276).
|
||||
func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return false, time.Time{}, err
|
||||
} else if on {
|
||||
return i.chainKeptOnBranch(ctx, c, repository, branch)
|
||||
}
|
||||
var first *time.Time
|
||||
var kept bool
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
@@ -80,12 +105,22 @@ func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string)
|
||||
|
||||
// AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3).
|
||||
func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainAloneMerges(ctx, c)
|
||||
}
|
||||
return i.merges(ctx, `where plan_id is null and alone order by merged_at desc nulls last, heard_at desc`)
|
||||
}
|
||||
|
||||
// AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to
|
||||
// be walked on its own commit.
|
||||
func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return err
|
||||
} else if on {
|
||||
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`,
|
||||
repository, commit, plan, alone)
|
||||
@@ -95,6 +130,11 @@ func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan st
|
||||
// Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more
|
||||
// are read so a fault is seen.
|
||||
func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated)
|
||||
}
|
||||
return i.plans(ctx, `where state in ('assembling', 'queued') order by created`)
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// What has been built.
|
||||
@@ -62,6 +64,10 @@ type Build struct {
|
||||
Asked time.Time
|
||||
// At is when the outcome was recorded — when it finished, not when it was asked.
|
||||
At time.Time
|
||||
// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1), and Published what it published: an artifact by its
|
||||
// digest, or a package release by name and version (ADR 0408 §1). Kept on the bus; a build kept in the table has neither.
|
||||
ReuseKey string
|
||||
Published *chain.Published
|
||||
}
|
||||
|
||||
// AskedOrAt is when the build was asked, or when it was recorded when that is not known — the
|
||||
@@ -128,6 +134,11 @@ func (b Build) Worked() bool { return b.Failed == "" }
|
||||
// whoever asked and once on the exchange when nobody was. Recording both would show one build as
|
||||
// two, and which of the two is real is not a question anybody could answer afterwards.
|
||||
func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return err
|
||||
} else if on {
|
||||
return i.chainRecordBuild(ctx, c, b)
|
||||
}
|
||||
made, err := json.Marshal(b.Made)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -207,6 +218,14 @@ func (i *Inventory) RecordMirrored(ctx context.Context, build, why string, refer
|
||||
// now* after something goes wrong, and *what has happened to this* when deciding whether to
|
||||
// trust it.
|
||||
func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Build, error) {
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainBuildList(ctx, c, module, limit)
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
@@ -249,6 +268,11 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui
|
||||
// by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the
|
||||
// repository, path, ref and commit from. False when no outcome with that id was recorded.
|
||||
func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return Build{}, false, err
|
||||
} else if on {
|
||||
return i.chainBuildByID(ctx, c, id)
|
||||
}
|
||||
var b Build
|
||||
var made []byte
|
||||
var asked *time.Time
|
||||
@@ -351,6 +375,11 @@ func registeredOnly(held, registered map[string]string) map[string]string {
|
||||
// edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
|
||||
// module standing on nothing looks like: an edge the mesh has not derived is not an edge.
|
||||
func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainBuiltAgainst(ctx, c)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select distinct on (module) module, built_against
|
||||
from build
|
||||
@@ -394,6 +423,11 @@ func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, erro
|
||||
// none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read
|
||||
// as before: every file of a context, and its own directory or root.
|
||||
func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainReadRepositories(ctx, c)
|
||||
}
|
||||
// The newest build of each module whatever its outcome: a failed one newer than the newest that
|
||||
// worked leaves that one's build source stale — the merge it was asked for may have changed the closure
|
||||
// (novox/hq ADR 0267), so its module is read whole until a build works again.
|
||||
@@ -580,6 +614,11 @@ func manifestOrNil(raw []byte) any {
|
||||
// One row per module and commit: a module built twice at the same commit is one fact, and the
|
||||
// row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219).
|
||||
func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainAnnounceable(ctx, c)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select distinct on (module, commit_hash)
|
||||
id, repository, ref, module, commit_hash, built_on, failed, made,
|
||||
|
||||
@@ -4,6 +4,7 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -215,11 +216,15 @@ func TestABuildsBuildSourceComesBackOverWhatItRead(t *testing.T) {
|
||||
if got := read["build-agent"]; len(got) != 1 || got[0].Paths != nil || got[0].Own {
|
||||
t.Fatalf("a build that said no build source reads as %+v", got)
|
||||
}
|
||||
var stored string
|
||||
if err := inv.store.Pool().QueryRow(ctx, `select built_contexts::text from build where id = 'said'`).Scan(&stored); err != nil {
|
||||
kept, found, err := inv.chain.Builds.Get(ctx, "said")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the build is not on the bus: %v %v", found, err)
|
||||
}
|
||||
storedRead, err := json.Marshal(kept.Value.Read)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(stored, "paths") || strings.Contains(stored, "own") {
|
||||
if stored := string(storedRead); strings.Contains(stored, "paths") || strings.Contains(stored, "own") {
|
||||
t.Fatalf("the build source was stored among what the build read: %s", stored)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -258,15 +258,31 @@ func (i *Inventory) SourceOf(ctx context.Context, module string) (Source, error)
|
||||
// machines are running last week's version" is the question somebody actually has, and it is the
|
||||
// one novox/hq ADR 0010 names as the thing that must not be lost.
|
||||
func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select m.name, coalesce(n.name, '')
|
||||
query := `select m.name, coalesce(n.name, '')
|
||||
from module m
|
||||
left join assignment a on a.module = m.name
|
||||
left join node n on n.id = a.node
|
||||
where m.source is not null
|
||||
and m.source_head is not null
|
||||
and coalesce(m.built_from, '') is distinct from m.source_head
|
||||
order by m.name, n.name`)
|
||||
order by m.name, n.name`
|
||||
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var assigned []assignRow
|
||||
if onTheBus {
|
||||
query = `select m.name, ''
|
||||
from module m
|
||||
where m.source is not null
|
||||
and m.source_head is not null
|
||||
and coalesce(m.built_from, '') is distinct from m.source_head
|
||||
order by m.name`
|
||||
if assigned, err = i.assignments(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -285,7 +301,18 @@ func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) {
|
||||
out[module] = append(out[module], node)
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, a := range assigned {
|
||||
if _, behind := out[a.Module]; behind {
|
||||
out[a.Module] = append(out[a.Module], a.Node)
|
||||
}
|
||||
}
|
||||
for module := range out {
|
||||
sort.Strings(out[module])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Catalogue is every module the mesh knows about, which is what resolution needs: the question
|
||||
@@ -401,22 +428,7 @@ func (i *Inventory) RetireUnshipped(ctx context.Context, shipped []string) (reti
|
||||
|
||||
// assignedOn is the machines a module is assigned to, sorted.
|
||||
func (i *Inventory) assignedOn(ctx context.Context, name string) ([]string, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name from assignment a join node n on n.id = a.node where a.module = $1
|
||||
order by n.name`, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var on []string
|
||||
for rows.Next() {
|
||||
var node string
|
||||
if err := rows.Scan(&node); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
on = append(on, node)
|
||||
}
|
||||
return on, rows.Err()
|
||||
return i.assignedNodes(ctx, name)
|
||||
}
|
||||
|
||||
// Provided reports whether a module came with the control plane rather than from a repository.
|
||||
@@ -600,25 +612,10 @@ func (i *Inventory) mayForget(ctx context.Context, name string) error {
|
||||
"%s comes with the control plane and cannot be forgotten; unassign it instead", name)
|
||||
}
|
||||
|
||||
var on []string
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name from assignment a join node n on n.id = a.node where a.module = $1
|
||||
order by n.name`, name)
|
||||
on, err := i.assignedNodes(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var node string
|
||||
if err := rows.Scan(&node); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
on = append(on, node)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(on) > 0 {
|
||||
return fmt.Errorf("%w: %s. Unassign it first", ErrStillAssigned, strings.Join(on, ", "))
|
||||
}
|
||||
@@ -657,6 +654,11 @@ func (i *Inventory) Assign(ctx context.Context, nodeName, module string) (bool,
|
||||
if err := i.runsSomewhere(ctx, module); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
|
||||
return false, err
|
||||
} else if on {
|
||||
return i.chainAssign(ctx, c, node, module)
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`insert into assignment (node, module) values ($1, $2) on conflict do nothing`,
|
||||
node.ID, module)
|
||||
@@ -675,13 +677,25 @@ func (i *Inventory) Unassign(ctx context.Context, nodeName, module string) error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`delete from assignment where node = $1 and module = $2`, node.ID, module)
|
||||
if err != nil {
|
||||
if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
|
||||
return err
|
||||
} else if on {
|
||||
if err := i.chainUnassign(ctx, c, node, module); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`delete from assignment where node = $1 and module = $2`, node.ID, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%s is not assigned to %s", module, nodeName)
|
||||
}
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%s is not assigned to %s", module, nodeName)
|
||||
// The seat it held falls back to derivation rather than pointing at nothing: what the store's foreign key did.
|
||||
if _, err := i.store.Pool().Exec(ctx, `delete from seat_holding where node = $1 and module = $2`, node.ID, module); err != nil {
|
||||
return err
|
||||
}
|
||||
// And its ports go back. Kept-once-chosen is a promise about a module that is still here —
|
||||
// held past unassignment, a fixed port stays claimed in the name of something that is gone,
|
||||
@@ -697,22 +711,7 @@ func (i *Inventory) Assigned(ctx context.Context, nodeName string) ([]string, er
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select module from assignment where node = $1 order by module`, node.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var m string
|
||||
if err := rows.Scan(&m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
return i.assignedModules(ctx, node.Name)
|
||||
}
|
||||
|
||||
// ProfileOf is what a node last said it can do, as resolution needs it: the capabilities that are
|
||||
@@ -1363,8 +1362,7 @@ type Entry struct {
|
||||
// catalogue nobody lists, and the questions here — what is this, where did it come from, who is
|
||||
// running it — are asked together every time.
|
||||
func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select m.name, m.manifest,
|
||||
query := `select m.name, m.manifest,
|
||||
coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''),
|
||||
coalesce(m.built_from, ''), coalesce(m.source_head, ''),
|
||||
coalesce(m.source_seen, to_timestamp(0)),
|
||||
@@ -1374,7 +1372,30 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
|
||||
left join node n on n.id = a.node
|
||||
group by m.name, m.manifest, m.source, m.source_path, m.source_seat, m.ref, m.built_from,
|
||||
m.source_head, m.source_seen
|
||||
order by m.name`)
|
||||
order by m.name`
|
||||
// On the bus the assignments are read from the bucket, and the machines of a module put beside it here.
|
||||
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var assignedBy map[string][]string
|
||||
if onTheBus {
|
||||
query = `select m.name, m.manifest,
|
||||
coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''),
|
||||
coalesce(m.built_from, ''), coalesce(m.source_head, ''),
|
||||
coalesce(m.source_seen, to_timestamp(0)), '{}'::text[]
|
||||
from module m
|
||||
order by m.name`
|
||||
rows, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
assignedBy = map[string][]string{}
|
||||
for _, a := range rows {
|
||||
assignedBy[a.Module] = append(assignedBy[a.Module], a.Node)
|
||||
}
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1393,6 +1414,9 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
|
||||
if source.Seen.Unix() == 0 {
|
||||
source.Seen = time.Time{}
|
||||
}
|
||||
if onTheBus {
|
||||
on = assignedBy[name]
|
||||
}
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, err
|
||||
@@ -1612,23 +1636,7 @@ func (i *Inventory) CurrentBuilds(ctx context.Context) (map[string]CurrentBuild,
|
||||
// exactly the machine an upgrade most needs to reach; waiting for it to report the old version
|
||||
// first would mean the machines furthest behind are the last to be caught up.
|
||||
func (i *Inventory) Running(ctx context.Context, module string) ([]string, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name from assignment a join node n on n.id = a.node
|
||||
where a.module = $1 order by n.name`, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, name)
|
||||
}
|
||||
return out, rows.Err()
|
||||
return i.assignedNodes(ctx, module)
|
||||
}
|
||||
|
||||
// runsSomewhere refuses to put a module on a machine when it would put nothing there.
|
||||
|
||||
@@ -113,13 +113,15 @@ func TestRemovingANodeTakesItsAssignments(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var left int
|
||||
if err := inv.store.Pool().QueryRow(t.Context(),
|
||||
`select count(*) from assignment`).Scan(&left); err != nil {
|
||||
// On the bus the key outlives the node it names, and is read as no assignment: the machine is not one the store knows.
|
||||
entries, err := inv.Catalogued(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if left != 0 {
|
||||
t.Errorf("%d assignment(s) outlived the node they were on", left)
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == "thing" && len(e.On) != 0 {
|
||||
t.Errorf("thing is assigned to %v, a machine that is gone", e.On)
|
||||
}
|
||||
}
|
||||
// And the module itself survives, because other nodes may be running it.
|
||||
shelf, err := inv.Catalogue(t.Context())
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **The chain moves to the bus kind by kind** (novox/hq ADR 0408 §8, issue 981). The tables of builds, rollouts and
|
||||
// assignments are copied once into the chain's buckets by the serving controller at its start (MoveChain), which then
|
||||
// records each kind it moved in chain_moved (migration 0104). **Until a kind is recorded there nothing reads its bucket;
|
||||
// once it is, nothing reads its tables**: ChainReads answers which, and every method of a kind asks it first.
|
||||
|
||||
// The kinds of the chain, as chain_moved names them.
|
||||
const (
|
||||
KindBuilds = "builds"
|
||||
KindRollouts = "rollouts"
|
||||
)
|
||||
|
||||
// testChains are the chains in memory a test made for its store, by the store's address: every inventory opened on that
|
||||
// store is given the same chain, as every process of a mesh is on the same bus. Empty outside a test.
|
||||
var testChains sync.Map
|
||||
|
||||
// ErrNoChain is a kind moved to the bus read by an inventory that has no chain attached: a process that did not
|
||||
// open the bus. Refused loudly, never answered from the tables the move left behind.
|
||||
var ErrNoChain = errors.New("this kind of record lives on the bus, and this process has not opened the bus")
|
||||
|
||||
// movedKinds is which kinds this process has read as moved: once moved, always moved, so it is read from the store
|
||||
// until it is true and not after.
|
||||
type movedKinds struct {
|
||||
mu sync.Mutex
|
||||
kind map[string]bool
|
||||
open ChainOpener
|
||||
// view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that
|
||||
// is not saved yet.
|
||||
view *rolloutView
|
||||
pending map[string]pendingMerge
|
||||
// switching is set when the assignments begin to be switched to the bus and kept once they are: the store's mirror of
|
||||
// them writes nothing from then on (AssignmentsMirrorMayWrite). Cleared only if the switch fails.
|
||||
switching atomic.Bool
|
||||
}
|
||||
|
||||
// AssignmentsMirrorMayWrite says the store's mirror of the assignments may still write the bucket: the assignments are
|
||||
// neither being switched to the bus nor on it. Asked before every write of the mirror's writer, not once a pass.
|
||||
func (i *Inventory) AssignmentsMirrorMayWrite() bool {
|
||||
i.moved.mu.Lock()
|
||||
moved := i.moved.kind[KindAssignments]
|
||||
i.moved.mu.Unlock()
|
||||
return !moved && !i.moved.switching.Load()
|
||||
}
|
||||
|
||||
// ChainOpener opens the chain on the bus when a process first needs it: a command that reads and never moved anything has
|
||||
// no bus connection until a kind it reads turns out to be moved.
|
||||
type ChainOpener func(ctx context.Context) (*chain.Chain, error)
|
||||
|
||||
// UseChainOpener gives the inventory the means to open the chain when it needs it and none is attached.
|
||||
func (i *Inventory) UseChainOpener(open ChainOpener) {
|
||||
i.moved.mu.Lock()
|
||||
i.moved.open = open
|
||||
i.moved.mu.Unlock()
|
||||
}
|
||||
|
||||
// UseChain attaches the chain the moved kinds are read and written through. The serving controller attaches the bus's,
|
||||
// holding the lease for its writes; a test attaches one in memory.
|
||||
func (i *Inventory) UseChain(c *chain.Chain) {
|
||||
i.moved.mu.Lock()
|
||||
defer i.moved.mu.Unlock()
|
||||
i.chain = c
|
||||
if i.moved.view != nil {
|
||||
i.moved.view.stop()
|
||||
i.moved.view = nil
|
||||
}
|
||||
}
|
||||
|
||||
// attached is the chain this inventory has, nil when none.
|
||||
func (i *Inventory) attached() *chain.Chain {
|
||||
i.moved.mu.Lock()
|
||||
defer i.moved.mu.Unlock()
|
||||
return i.chain
|
||||
}
|
||||
|
||||
// ChainReads says whether a kind of the chain is read from its bucket: chain_moved names it. It answers the chain to read
|
||||
// through, and ErrNoChain when the kind is moved and none is attached.
|
||||
func (i *Inventory) ChainReads(ctx context.Context, kind string) (*chain.Chain, bool, error) {
|
||||
i.moved.mu.Lock()
|
||||
known := i.moved.kind[kind]
|
||||
i.moved.mu.Unlock()
|
||||
if !known {
|
||||
var moved bool
|
||||
err := i.store.Pool().QueryRow(ctx, `select counts ? $1 from chain_moved`, kind).Scan(&moved)
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.Is(err, pgx.ErrNoRows) || (errors.As(err, &pgErr) && pgErr.Code == "42P01") {
|
||||
// No row, or a store older than the migration that makes the table: nothing moved.
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("whether %s moved to the bus cannot be read: %w", kind, err)
|
||||
}
|
||||
if !moved {
|
||||
return nil, false, nil
|
||||
}
|
||||
i.moved.mu.Lock()
|
||||
if i.moved.kind == nil {
|
||||
i.moved.kind = map[string]bool{}
|
||||
}
|
||||
i.moved.kind[kind] = true
|
||||
i.moved.mu.Unlock()
|
||||
}
|
||||
i.moved.mu.Lock()
|
||||
defer i.moved.mu.Unlock()
|
||||
if i.chain == nil && i.moved.open != nil {
|
||||
c, err := i.moved.open(ctx)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%w: %s: %v", ErrNoChain, kind, err)
|
||||
}
|
||||
i.chain = c
|
||||
}
|
||||
if i.chain == nil {
|
||||
return nil, false, fmt.Errorf("%w: %s", ErrNoChain, kind)
|
||||
}
|
||||
return i.chain, true, nil
|
||||
}
|
||||
|
||||
// ChainMoved is, per kind moved, how many records the move wrote; empty before any is.
|
||||
func (i *Inventory) ChainMoved(ctx context.Context) (map[string]int, error) {
|
||||
var counts map[string]int
|
||||
err := i.store.Pool().QueryRow(ctx, `select counts from chain_moved`).Scan(&counts)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return map[string]int{}, nil
|
||||
}
|
||||
return counts, err
|
||||
}
|
||||
|
||||
// NewForTest is an inventory with no store, for a test of what needs none.
|
||||
func NewForTest() *Inventory { return &Inventory{moved: &movedKinds{}} }
|
||||
|
||||
// BeginSwitchForTest sets the flag a kind's switch sets, as a switch under way does.
|
||||
func (i *Inventory) BeginSwitchForTest(kind string) {
|
||||
if kind == KindAssignments {
|
||||
i.moved.switching.Store(true)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// Assignments on the bus (novox/hq ADR 0408, issue 1093): once the kind `assignments` is moved (chain.go) an assignment is
|
||||
// a key `<node>.<module>` of the assignments bucket, written by compare-and-set by the controller that holds the lease,
|
||||
// and every question the joins of the store answered is answered from the bucket. **No key is removed** (ADR 0408 §4): an
|
||||
// unassignment writes the value again with `ended` set, and a value with `ended` or a `state` is no assignment to any reader
|
||||
// here (chain.Assignment.Stands). No `pending` state is written yet (issue 1094).
|
||||
|
||||
// KindAssignments is the kind of the chain that is each module assigned to a node, with the build it runs there.
|
||||
const KindAssignments = "assignments"
|
||||
|
||||
func assignmentKey(node, module string) string { return node + "." + module }
|
||||
|
||||
// assignRow is one assignment as the joins of the store read it.
|
||||
type assignRow struct {
|
||||
Node, Module string
|
||||
Assigned time.Time
|
||||
}
|
||||
|
||||
// assignments is every assignment, by node then module: from the bucket when the kind is moved, else from the store.
|
||||
// A key whose value names no time assigned is a module only pulled in by another (issue 765), and one that has ended or
|
||||
// waits in a state is gone: no assignment. One on a machine the store does not know is none either, as the store took a
|
||||
// removed machine's.
|
||||
func (i *Inventory) assignments(ctx context.Context) ([]assignRow, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !on {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node order by n.name, a.module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []assignRow
|
||||
for rows.Next() {
|
||||
var a assignRow
|
||||
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the assignments cannot be read from the bus: %w", err)
|
||||
}
|
||||
nodes, err := i.nodeNames(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []assignRow
|
||||
for _, k := range all {
|
||||
if k.Value.Assigned.IsZero() || !k.Value.Stands() || !nodes[k.Value.Node] {
|
||||
continue
|
||||
}
|
||||
out = append(out, assignRow{k.Value.Node, k.Value.Module, k.Value.Assigned})
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool {
|
||||
if out[a].Node != out[b].Node {
|
||||
return out[a].Node < out[b].Node
|
||||
}
|
||||
return out[a].Module < out[b].Module
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) nodeNames(ctx context.Context) (map[string]bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select name from node`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// assignedNodes is the machines a module is assigned to, sorted.
|
||||
func (i *Inventory) assignedNodes(ctx context.Context, module string) ([]string, error) {
|
||||
rows, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, a := range rows {
|
||||
if a.Module == module {
|
||||
out = append(out, a.Node)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// assignedModules is the modules assigned to a machine, sorted.
|
||||
func (i *Inventory) assignedModules(ctx context.Context, node string) ([]string, error) {
|
||||
rows, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, a := range rows {
|
||||
if a.Node == node {
|
||||
out = append(out, a.Module)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// isAssigned says a module is assigned to a machine.
|
||||
func (i *Inventory) isAssigned(ctx context.Context, node, module string) (bool, error) {
|
||||
rows, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, a := range rows {
|
||||
if a.Node == node && a.Module == module {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// raiseGeneration raises the assignment generation, as the store's triggers do for a change of its table (migration
|
||||
// 0094): in the chain nothing in the store changes with an assignment, so the writer says it.
|
||||
func (i *Inventory) raiseGeneration(ctx context.Context) error {
|
||||
_, err := i.store.Pool().Exec(ctx, `update assignment_generation set generation = generation + 1`)
|
||||
return err
|
||||
}
|
||||
|
||||
// chainAssign puts a module on a machine, and says whether that is new.
|
||||
func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, module string) (bool, error) {
|
||||
var known bool
|
||||
if err := i.store.Pool().QueryRow(ctx, `select exists (select 1 from module where name = $1)`, module).Scan(&known); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !known {
|
||||
return false, fmt.Errorf("%w: %s", ErrNoSuchModule, module)
|
||||
}
|
||||
key := assignmentKey(node.Name, module)
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Assignments.Get(ctx, key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if found && !cur.Value.Assigned.IsZero() && cur.Value.Stands() {
|
||||
return false, nil // one assignment of a module per machine: a repeat is absorbed (novox/hq ADR 0115)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
fresh := chain.Assignment{Node: node.Name, Module: module, Assigned: now}
|
||||
switch {
|
||||
case found && cur.Value.Stands():
|
||||
// A module only pulled in by another becomes an assignment, keeping what it holds.
|
||||
cur.Value.Assigned = now
|
||||
_, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision)
|
||||
case found:
|
||||
// One that ended is assigned again: a fresh value over it, the ended one staying in history.
|
||||
_, err = c.Assignments.Update(ctx, key, fresh, cur.Revision)
|
||||
default:
|
||||
_, err = c.Assignments.Create(ctx, key, fresh)
|
||||
}
|
||||
if errors.Is(err, chain.ErrMoved) || errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s is not assigned to %s on the bus: %w", module, node.Name, err)
|
||||
}
|
||||
if err := i.raiseGeneration(ctx); err != nil {
|
||||
return true, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name)
|
||||
}
|
||||
|
||||
// chainUnassign takes a module off a machine: the value is written again with `ended`, the key stays.
|
||||
func (i *Inventory) chainUnassign(ctx context.Context, c *chain.Chain, node Node, module string) error {
|
||||
key := assignmentKey(node.Name, module)
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Assignments.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found || cur.Value.Assigned.IsZero() || !cur.Value.Stands() {
|
||||
return fmt.Errorf("%s is not assigned to %s", module, node.Name)
|
||||
}
|
||||
cur.Value.Ended = time.Now().UTC()
|
||||
if _, err := c.Assignments.Update(ctx, key, cur.Value, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("%s is not taken off %s on the bus: %w", module, node.Name, err)
|
||||
}
|
||||
return i.raiseGeneration(ctx)
|
||||
}
|
||||
return fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name)
|
||||
}
|
||||
|
||||
// chainAwaitingSince is AwaitingSince over the bucket: the oldest assignment on the machine made after it was last sent.
|
||||
func (i *Inventory) chainAwaitingSince(ctx context.Context, name string) (time.Time, error) {
|
||||
var sentAt *time.Time
|
||||
var created time.Time
|
||||
err := i.store.Pool().QueryRow(ctx, `select sent_at, created from node where name = $1`, name).Scan(&sentAt, &created)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
rows, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
var oldest time.Time
|
||||
for _, a := range rows {
|
||||
if a.Node == name && (sentAt == nil || a.Assigned.After(*sentAt)) && (oldest.IsZero() || a.Assigned.Before(oldest)) {
|
||||
oldest = a.Assigned
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !oldest.IsZero():
|
||||
return oldest, nil
|
||||
case sentAt != nil:
|
||||
return *sentAt, nil
|
||||
}
|
||||
return created, nil
|
||||
}
|
||||
|
||||
// KeepAssignmentPlans writes the provisioner's plan of each assignment into its value on the bus, and a value of its own for
|
||||
// each module only pulled in by another on a machine (issue 765), and ends the value of one that is no longer pulled in. The
|
||||
// plans are derived from what each machine was last sent, not part of the record of what is assigned, so they are read
|
||||
// again on a tick and written where they differ: **by compare-and-set on the one field**, never laying over the build, commit
|
||||
// and rollout another write put in the same value, and never ending an assignment. A plan the provisioner's rules refuse is
|
||||
// refused by name and nothing is written (AssignmentValues).
|
||||
func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed int, err error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on {
|
||||
return 0, 0, err
|
||||
}
|
||||
var plans map[string]Provisioning
|
||||
if i.provisioningPlans != nil {
|
||||
if plans, err = i.provisioningPlans(ctx); err != nil {
|
||||
return 0, 0, fmt.Errorf("composing the provisioner's plans: %w", err)
|
||||
}
|
||||
}
|
||||
assigned, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
rows := make([]AssignmentMirrored, 0, len(assigned))
|
||||
for _, a := range assigned {
|
||||
rows = append(rows, AssignmentMirrored{Node: a.Node, Module: a.Module, Assigned: a.Assigned.UTC()})
|
||||
}
|
||||
want, err := AssignmentValues(rows, plans)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
have, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("the assignments cannot be read from the bus: %w", err)
|
||||
}
|
||||
for key, body := range want {
|
||||
var next chain.Assignment
|
||||
if err := json.Unmarshal(body, &next); err != nil {
|
||||
return written, removed, err
|
||||
}
|
||||
cur, has := have[key]
|
||||
switch keepStep(has, cur.Value, next) {
|
||||
case keepSkip:
|
||||
continue
|
||||
case keepCreate:
|
||||
next.Assigned = time.Time{}
|
||||
_, err = c.Assignments.Create(ctx, key, next)
|
||||
case keepFresh:
|
||||
// A value that ended, pulled in again: a fresh one over it, the ended one staying in history.
|
||||
next.Assigned = time.Time{}
|
||||
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
|
||||
default:
|
||||
cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning
|
||||
_, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision)
|
||||
}
|
||||
if errors.Is(err, chain.ErrMoved) || errors.Is(err, chain.ErrExists) {
|
||||
continue // written meanwhile; the next pass reads it
|
||||
}
|
||||
if err != nil {
|
||||
return written, removed, fmt.Errorf("the plan of %s is not written: %w", key, err)
|
||||
}
|
||||
written++
|
||||
}
|
||||
for key, cur := range have {
|
||||
if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() {
|
||||
continue
|
||||
}
|
||||
// A value of a module no longer pulled in, which no assignment names: it ends, and stays.
|
||||
cur.Value.Ended = time.Now().UTC()
|
||||
if _, err := c.Assignments.Update(ctx, key, cur.Value, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return written, removed, fmt.Errorf("the plan of %s is not ended: %w", key, err)
|
||||
}
|
||||
removed++
|
||||
}
|
||||
return written, removed, nil
|
||||
}
|
||||
|
||||
// sameJSON says two encoded values are the same bytes, an absent value being a null one.
|
||||
func sameJSON(a, b json.RawMessage) bool {
|
||||
trim := func(x json.RawMessage) string {
|
||||
if s := strings.TrimSpace(string(x)); s != "null" {
|
||||
return s
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return trim(a) == trim(b)
|
||||
}
|
||||
|
||||
// KeptValue is an assignment's value as the bucket holds it, encoded, at its revision.
|
||||
type KeptValue struct {
|
||||
Value []byte
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// KeptAssignments is every key of the assignments bucket that holds a value that stands, and false when the assignments are
|
||||
// not on the bus.
|
||||
// What the serving controller compares from one look to the next to say a key changed (`assignment-changed`).
|
||||
func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, bool, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on {
|
||||
return nil, false, err
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
out := make(map[string]KeptValue, len(all))
|
||||
for key, k := range all {
|
||||
if !k.Value.Stands() {
|
||||
continue // an ended value is no assignment: for what is said of it, the key has gone
|
||||
}
|
||||
body, err := json.Marshal(k.Value)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
out[key] = KeptValue{Value: body, Revision: k.Revision}
|
||||
}
|
||||
return out, true, nil
|
||||
}
|
||||
|
||||
// What a pass that keeps the plans does with one key.
|
||||
const (
|
||||
keepSkip = "skip"
|
||||
keepCreate = "create"
|
||||
keepFresh = "fresh"
|
||||
keepField = "field"
|
||||
)
|
||||
|
||||
// keepStep decides what the pass does with a key from what it wants there (read from the assignments) and what the bucket
|
||||
// holds (read a moment later). **It never makes or revives an assignment**: that is Assign's, and an unassignment between the
|
||||
// two reads leaves a key that ended while the pass still wants it as an assignment, which would come back as a value
|
||||
// pulled in by another. Only a key with no time assigned, on both sides, is made or made again here.
|
||||
func keepStep(has bool, cur, next chain.Assignment) string {
|
||||
switch {
|
||||
case !has && !next.Assigned.IsZero():
|
||||
return keepSkip // assigned since the bucket was read: its own write is not here yet
|
||||
case !has:
|
||||
return keepCreate
|
||||
case !cur.Stands():
|
||||
if !cur.Assigned.IsZero() || !next.Assigned.IsZero() {
|
||||
return keepSkip // an assignment that ended is not pulled in again by a plan
|
||||
}
|
||||
return keepFresh
|
||||
case sameJSON(cur.Provisioning, next.Provisioning) && cur.RequiredBy == next.RequiredBy:
|
||||
return keepSkip
|
||||
}
|
||||
return keepField
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **An assignment's key is `<node>.<module>`, as the bucket has always had it** (ADR 0408 §3): its readers parse nothing
|
||||
// but it.
|
||||
func TestAnAssignmentsKeyIsTheNodeAndTheModule(t *testing.T) {
|
||||
if got := assignmentKey("anchor", "notes"); got != "anchor.notes" {
|
||||
t.Fatalf("the key is %q", got)
|
||||
}
|
||||
if got := assignmentKey("g14.example", "dunst"); got != "g14.example.dunst" {
|
||||
t.Fatalf("the key of a machine whose name holds a dot is %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A plan that is absent and a null one are one** (the provisioner's plan in an assignment's value): the pass that keeps
|
||||
// the plans writes only where they differ.
|
||||
func TestAnAbsentPlanAndANullOneAreTheSame(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
a, b json.RawMessage
|
||||
same bool
|
||||
}{
|
||||
"both absent": {nil, nil, true},
|
||||
"null and absent": {json.RawMessage(`null`), nil, true},
|
||||
"the same plan": {json.RawMessage(`{"files":[]}`), json.RawMessage(` {"files":[]} `), true},
|
||||
"another plan": {json.RawMessage(`{"files":[]}`), json.RawMessage(`{"files":[1]}`), false},
|
||||
"a plan and none": {json.RawMessage(`{"files":[]}`), nil, false},
|
||||
"none and a plan": {nil, json.RawMessage(`{"files":[]}`), false},
|
||||
} {
|
||||
if got := sameJSON(tc.a, tc.b); got != tc.same {
|
||||
t.Errorf("%s: %v", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A pass that keeps the plans never brings back an assignment that ended** (issue 1093): it reads what it wants and what
|
||||
// the bucket holds in two reads, and an unassignment between them leaves a key that ended while the pass still wants it
|
||||
// assigned.
|
||||
func TestAPassThatKeepsThePlansNeverRevivesAnEndedAssignment(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
plan := json.RawMessage(`{"files":[]}`)
|
||||
assigned := chain.Assignment{Node: "a", Module: "m", Assigned: at, Provisioning: plan}
|
||||
ended := chain.Assignment{Node: "a", Module: "m", Assigned: at, Ended: at.Add(time.Minute)}
|
||||
pulledIn := chain.Assignment{Node: "a", Module: "p", RequiredBy: "m", Provisioning: plan}
|
||||
pulledInEnded := chain.Assignment{Node: "a", Module: "p", Ended: at}
|
||||
for name, tc := range map[string]struct {
|
||||
has bool
|
||||
cur, next chain.Assignment
|
||||
want string
|
||||
}{
|
||||
"the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip},
|
||||
"wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip},
|
||||
"a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate},
|
||||
"a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh},
|
||||
"an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip},
|
||||
"the plan is the same": {true, assigned, assigned, keepSkip},
|
||||
"the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField},
|
||||
} {
|
||||
if got := keepStep(tc.has, tc.cur, tc.next); got != tc.want {
|
||||
t.Errorf("%s: %s, want %s", name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The store's mirror of the assignments is refused to its writer once the assignments are moved** (ADR 0408 §4, issue
|
||||
// 1093): asked at the moment of the write, so a tick that decided before the switch lays nothing after it.
|
||||
func TestTheStoresMirrorOfTheAssignmentsIsRefusedToItsWriterOnceTheyAreMoved(t *testing.T) {
|
||||
inv := &Inventory{moved: &movedKinds{kind: map[string]bool{KindAssignments: true}}}
|
||||
inv.UseChain(chain.InMemory(nil))
|
||||
if _, err := inv.AssignmentsMirrorToWrite(context.Background()); !errors.Is(err, ErrAssignmentsMoved) {
|
||||
t.Fatalf("the writer was given the store's rows: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,570 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// Builds on the bus (novox/hq ADR 0408, issue 1090): once the kind `builds` is moved (chain.go) every build is a key of
|
||||
// the builds bucket, under its id, and every read below is made from the bucket's records. The tables are read when the
|
||||
// kind is not moved, by the methods these stand beside.
|
||||
|
||||
// toChainBuild is a build as the bucket keeps it.
|
||||
func toChainBuild(b Build) chain.Build {
|
||||
c := chain.Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On,
|
||||
Path: b.Path, Against: b.Against, Mirrored: b.Mirrored, SourceFingerprint: b.SourceFingerprint,
|
||||
Failed: b.Failed, Asked: b.Asked.UTC(), At: b.At.UTC(), ReuseKey: b.ReuseKey, Published: b.Published}
|
||||
if len(b.Manifest) > 0 {
|
||||
c.Manifest = json.RawMessage(b.Manifest)
|
||||
}
|
||||
for _, r := range b.Read {
|
||||
c.Read = append(c.Read, chain.ReadRepository{Repository: r.Repository, Ref: r.Ref, Paths: r.Paths, Own: r.Own})
|
||||
}
|
||||
for _, s := range b.Sources {
|
||||
c.Sources = append(c.Sources, chain.Source{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
for _, a := range b.Made {
|
||||
c.Made = append(c.Made, chain.Artifact{Name: a.Name, Kind: a.Kind, Reference: a.Reference})
|
||||
}
|
||||
if c.Published == nil {
|
||||
c.Published = publishedOf(b.Made)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// fromChainBuild is a build as the mesh reads it.
|
||||
func fromChainBuild(c chain.Build) Build {
|
||||
b := Build{ID: c.ID, Repository: c.Repository, Ref: c.Ref, Module: c.Module, Commit: c.Commit, On: c.On, Path: c.Path,
|
||||
Manifest: []byte(c.Manifest), Against: c.Against, Mirrored: c.Mirrored, SourceFingerprint: c.SourceFingerprint,
|
||||
Failed: c.Failed, Asked: c.Asked, At: c.At, ReuseKey: c.ReuseKey, Published: c.Published}
|
||||
if len(b.Manifest) == 0 {
|
||||
b.Manifest = nil
|
||||
}
|
||||
for _, r := range c.Read {
|
||||
b.Read = append(b.Read, ReadRepository{Repository: r.Repository, Ref: r.Ref, Paths: r.Paths, Own: r.Own})
|
||||
}
|
||||
for _, s := range c.Sources {
|
||||
b.Sources = append(b.Sources, BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
for _, a := range c.Made {
|
||||
b.Made = append(b.Made, Artifact{Name: a.Name, Kind: a.Kind, Reference: a.Reference})
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// publishedOf is what a build published, read from what it made: a package by name and version, and an artifact by the
|
||||
// digest of the first thing it made that has one (`<reference>@sha256:…`). Nil for a build that published nothing.
|
||||
func publishedOf(made []Artifact) *chain.Published {
|
||||
var out chain.Published
|
||||
for _, a := range made {
|
||||
switch {
|
||||
case a.Kind == "package":
|
||||
if name, version, ok := strings.Cut(a.Reference, "@"); ok && name != "" && version != "" && out.Package == nil {
|
||||
out.Package = &chain.Package{Name: name, Version: version}
|
||||
}
|
||||
case out.Artifact == "":
|
||||
if _, digest, ok := strings.Cut(a.Reference, "@"); ok && strings.HasPrefix(digest, "sha256:") {
|
||||
out.Artifact = digest
|
||||
}
|
||||
}
|
||||
}
|
||||
if out.Artifact == "" && out.Package == nil {
|
||||
return nil
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
// chainBuildKey is a build's key. An id the bus takes no spelling of is refused by the write, loudly.
|
||||
func chainBuildKey(id string) string { return id }
|
||||
|
||||
// chainRecordBuild keeps a build under its id. **Idempotent on the id**, as the table's insert was: a result arrives twice,
|
||||
// and the first record stands. A request nobody has answered is the record in state asked (issue 1091); the outcome
|
||||
// takes its place under the same key, and keeps the request it answers.
|
||||
func (i *Inventory) chainRecordBuild(ctx context.Context, c *chain.Chain, b Build) error {
|
||||
if b.At.IsZero() {
|
||||
b.At = time.Now().UTC()
|
||||
}
|
||||
key := chainBuildKey(b.ID)
|
||||
for try := 0; try < casTries; try++ {
|
||||
next := toChainBuild(b)
|
||||
cur, found, err := c.Builds.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build %s is not kept on the bus: %w", b.ID, err)
|
||||
}
|
||||
switch {
|
||||
case !found:
|
||||
_, err = c.Builds.Create(ctx, key, next)
|
||||
if errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
}
|
||||
case cur.Value.State == chain.BuildAsked:
|
||||
next.Request = cur.Value.Request
|
||||
_, err = c.Builds.Update(ctx, key, next, cur.Revision)
|
||||
if errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
}
|
||||
default:
|
||||
// An outcome stands: the first record is the record.
|
||||
return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("build %s is not kept on the bus: %w", b.ID, err)
|
||||
}
|
||||
return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
|
||||
}
|
||||
return fmt.Errorf("build %s is not kept on the bus: it kept moving under the write", b.ID)
|
||||
}
|
||||
|
||||
// casTries is how often a write that lost a compare-and-set reads the key again and tries once more.
|
||||
const casTries = 8
|
||||
|
||||
// chainBuilds is every build of the bucket, whole, in no order.
|
||||
func chainBuilds(ctx context.Context, c *chain.Chain) ([]Build, error) {
|
||||
all, err := c.Builds.All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the builds cannot be read from the bus: %w", err)
|
||||
}
|
||||
out := make([]Build, 0, len(all))
|
||||
for _, k := range all {
|
||||
// A request nobody has answered is no build: it has no outcome to read.
|
||||
if k.Value.State == chain.BuildAsked {
|
||||
continue
|
||||
}
|
||||
out = append(out, fromChainBuild(k.Value))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// newestRequestOrder sorts builds as newestRequestFirst does: the newest request first, then the newest record.
|
||||
func newestRequestOrder(builds []Build) {
|
||||
sort.SliceStable(builds, func(a, b int) bool {
|
||||
x, y := builds[a].AskedOrAt(), builds[b].AskedOrAt()
|
||||
if !x.Equal(y) {
|
||||
return x.After(y)
|
||||
}
|
||||
if !builds[a].At.Equal(builds[b].At) {
|
||||
return builds[a].At.After(builds[b].At)
|
||||
}
|
||||
return builds[a].ID > builds[b].ID
|
||||
})
|
||||
}
|
||||
|
||||
// byModule sorts builds by module, then newest request first.
|
||||
func byModule(builds []Build) {
|
||||
newestRequestOrder(builds)
|
||||
sort.SliceStable(builds, func(a, b int) bool { return builds[a].Module < builds[b].Module })
|
||||
}
|
||||
|
||||
// worked is the builds that made something for a module: succeeded, and knew what they built.
|
||||
func worked(all []Build) []Build {
|
||||
var out []Build
|
||||
for _, b := range all {
|
||||
if b.Failed == "" && b.Module != "" {
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (i *Inventory) chainBuildList(ctx context.Context, c *chain.Chain, module string, limit int) ([]Build, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []Build
|
||||
for _, b := range all {
|
||||
if module == "" || b.Module == module {
|
||||
out = append(out, Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit,
|
||||
On: b.On, Failed: b.Failed, Made: b.Made, Asked: b.Asked, At: b.At, ReuseKey: b.ReuseKey, Published: b.Published})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(a, b int) bool {
|
||||
if !out[a].At.Equal(out[b].At) {
|
||||
return out[a].At.After(out[b].At)
|
||||
}
|
||||
return out[a].ID > out[b].ID
|
||||
})
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainBuildByID(ctx context.Context, c *chain.Chain, id string) (Build, bool, error) {
|
||||
k, ok, err := c.Builds.Get(ctx, chainBuildKey(id))
|
||||
if err != nil || !ok || k.Value.State == chain.BuildAsked {
|
||||
// A request nobody has answered has no outcome to read.
|
||||
return Build{}, false, err
|
||||
}
|
||||
b := fromChainBuild(k.Value)
|
||||
return Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On,
|
||||
Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At}, true, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainBuiltAgainst(ctx context.Context, c *chain.Chain) (map[string][]string, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ok := worked(all)
|
||||
byModule(ok)
|
||||
out := map[string][]string{}
|
||||
seen := map[string]bool{}
|
||||
for _, b := range ok {
|
||||
if seen[b.Module] {
|
||||
continue
|
||||
}
|
||||
seen[b.Module] = true
|
||||
if len(b.Against) > 0 {
|
||||
out[b.Module] = b.Against
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainReadRepositories(ctx context.Context, c *chain.Chain) (map[string][]ReadRepository, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The newest build of each module whatever its outcome: a failed one newer than the newest that worked leaves that
|
||||
// one's build source stale.
|
||||
type newestOf struct {
|
||||
at time.Time
|
||||
failed bool
|
||||
}
|
||||
newest := map[string]newestOf{}
|
||||
tried := make([]Build, 0, len(all))
|
||||
for _, b := range all {
|
||||
if b.Module != "" {
|
||||
tried = append(tried, b)
|
||||
}
|
||||
}
|
||||
byModule(tried)
|
||||
for _, b := range tried {
|
||||
if _, seen := newest[b.Module]; !seen {
|
||||
newest[b.Module] = newestOf{b.AskedOrAt(), b.Failed != ""}
|
||||
}
|
||||
}
|
||||
owns, err := i.moduleSources(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ok := worked(all)
|
||||
byModule(ok)
|
||||
read := map[string][]ReadRepository{}
|
||||
seen := map[string]bool{}
|
||||
for _, b := range ok {
|
||||
if seen[b.Module] {
|
||||
continue
|
||||
}
|
||||
seen[b.Module] = true
|
||||
built := b.AskedOrAt()
|
||||
of := append([]ReadRepository(nil), b.Read...)
|
||||
sources := b.Sources
|
||||
whole := ""
|
||||
if n, known := newest[b.Module]; known && n.failed && n.at.After(built) && len(sources) > 0 {
|
||||
sources, whole = nil, "a newer build of it failed"
|
||||
}
|
||||
of = WithBuildSources(of, sources)
|
||||
if own, has := owns[b.Module]; has {
|
||||
of = OwnRepositoryAsOwn(of, own)
|
||||
}
|
||||
if whole != "" {
|
||||
of = append(of, ReadRepository{Own: true, Whole: whole})
|
||||
}
|
||||
if len(of) > 0 {
|
||||
for k := range of {
|
||||
of[k].Built, of[k].Looked = built, newest[b.Module].at
|
||||
}
|
||||
read[b.Module] = of
|
||||
}
|
||||
}
|
||||
return read, nil
|
||||
}
|
||||
|
||||
// moduleSources is the repository each registered module is registered from.
|
||||
func (i *Inventory) moduleSources(ctx context.Context) (map[string]string, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select name, source from module where source is not null`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]string{}
|
||||
for rows.Next() {
|
||||
var name, source string
|
||||
if err := rows.Scan(&name, &source); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = source
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (i *Inventory) chainAnnounceable(ctx context.Context, c *chain.Chain) ([]Build, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ok []Build
|
||||
for _, b := range worked(all) {
|
||||
if b.Commit != "" {
|
||||
ok = append(ok, b)
|
||||
}
|
||||
}
|
||||
newestRequestOrder(ok)
|
||||
type key struct{ module, commit string }
|
||||
seen := map[key]bool{}
|
||||
var out []Build
|
||||
for _, b := range ok {
|
||||
k := key{b.Module, b.Commit}
|
||||
if seen[k] {
|
||||
continue
|
||||
}
|
||||
seen[k] = true
|
||||
out = append(out, Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit,
|
||||
On: b.On, Failed: b.Failed, Made: b.Made, Path: b.Path, Manifest: b.Manifest, Against: b.Against, At: b.At})
|
||||
}
|
||||
sort.SliceStable(out, func(a, b int) bool { return out[a].At.Before(out[b].At) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// failedAtAGate is the builds that failed their gate. The gate's verdicts stay in their table (issue 1097).
|
||||
func (i *Inventory) failedAtAGate(ctx context.Context) (map[string]bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select build from build_gate where verdict = 'failed'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[id] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module string) (map[string][]sourceRow, []string, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
failed, err := i.failedAtAGate(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ok := worked(all)
|
||||
byModule(ok)
|
||||
out := map[string][]sourceRow{}
|
||||
var order []string
|
||||
for _, b := range ok {
|
||||
if module != "" && b.Module != module {
|
||||
continue
|
||||
}
|
||||
made, err := json.Marshal(b.Made)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if _, seen := out[b.Module]; !seen {
|
||||
order = append(order, b.Module)
|
||||
}
|
||||
out[b.Module] = append(out[b.Module], sourceRow{id: b.ID, commit: b.Commit, fingerprint: b.SourceFingerprint,
|
||||
made: made, manifest: b.Manifest, failedGate: failed[b.ID]})
|
||||
}
|
||||
return out, order, nil
|
||||
}
|
||||
|
||||
// canonical is a JSON document encoded the one way (objects' keys sorted, compact), so the same content hashes alike
|
||||
// however the bucket's record spells it.
|
||||
func canonicalJSON(raw []byte) []byte {
|
||||
if len(bytes.TrimSpace(raw)) == 0 {
|
||||
return nil
|
||||
}
|
||||
out, err := canonical(raw)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// chainFingerprint is what a build would put on a machine, hashed: its artifacts and its manifest.
|
||||
func chainFingerprint(b Build) string {
|
||||
made, _ := json.Marshal(b.Made)
|
||||
sum := sha256.Sum256(append(append(canonicalJSON(made), 0), canonicalJSON(b.Manifest)...))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (i *Inventory) chainBuildFingerprints(ctx context.Context, c *chain.Chain, module string) (map[string]string, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ok []Build
|
||||
for _, b := range all {
|
||||
if b.Module == module && b.Failed == "" && b.Commit != "" {
|
||||
ok = append(ok, b)
|
||||
}
|
||||
}
|
||||
sortByRecorded(ok)
|
||||
if len(ok) > 50 {
|
||||
ok = ok[:50]
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, b := range ok {
|
||||
if _, seen := out[b.Commit]; !seen {
|
||||
out[b.Commit] = chainFingerprint(b)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainFingerprints(ctx context.Context, c *chain.Chain) (map[string]map[string]string, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ok []Build
|
||||
for _, b := range all {
|
||||
if b.Module != "" && b.Failed == "" && b.Commit != "" {
|
||||
ok = append(ok, b)
|
||||
}
|
||||
}
|
||||
sortByRecorded(ok)
|
||||
out := map[string]map[string]string{}
|
||||
for _, b := range ok {
|
||||
if out[b.Module] == nil {
|
||||
out[b.Module] = map[string]string{}
|
||||
}
|
||||
if _, seen := out[b.Module][b.Commit]; !seen {
|
||||
out[b.Module][b.Commit] = chainFingerprint(b)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sortByRecorded is newest record first, as `order by at desc`.
|
||||
func sortByRecorded(builds []Build) {
|
||||
sort.SliceStable(builds, func(a, b int) bool {
|
||||
if !builds[a].At.Equal(builds[b].At) {
|
||||
return builds[a].At.After(builds[b].At)
|
||||
}
|
||||
return builds[a].ID > builds[b].ID
|
||||
})
|
||||
}
|
||||
|
||||
func (i *Inventory) chainBuildOf(ctx context.Context, c *chain.Chain, module, commit string) (string, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var ok []Build
|
||||
for _, b := range all {
|
||||
if b.Module == module && b.Commit == commit && b.Failed == "" {
|
||||
ok = append(ok, b)
|
||||
}
|
||||
}
|
||||
sortByRecorded(ok)
|
||||
if len(ok) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return ok[0].ID, nil
|
||||
}
|
||||
|
||||
// chainManifestBuild is the newest successful build of a module from a commit (or one it abbreviates) that kept a manifest.
|
||||
func (i *Inventory) chainManifestBuild(ctx context.Context, c *chain.Chain, module, commit string) (string, []byte, bool, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return "", nil, false, err
|
||||
}
|
||||
var ok []Build
|
||||
for _, b := range all {
|
||||
if b.Module == module && b.Failed == "" && len(b.Manifest) > 0 && string(bytes.TrimSpace(b.Manifest)) != "null" &&
|
||||
(b.Commit == commit || strings.HasPrefix(b.Commit, commit)) {
|
||||
ok = append(ok, b)
|
||||
}
|
||||
}
|
||||
sortByRecorded(ok)
|
||||
if len(ok) == 0 {
|
||||
return "", nil, false, nil
|
||||
}
|
||||
return ok[0].ID, ok[0].Manifest, true, nil
|
||||
}
|
||||
|
||||
// chainManifestOf is the manifest a build was recorded with.
|
||||
func (i *Inventory) chainManifestOf(ctx context.Context, c *chain.Chain, id string) ([]byte, error) {
|
||||
k, ok, err := c.Builds.Get(ctx, chainBuildKey(id))
|
||||
if err != nil || !ok {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(k.Value.Manifest), nil
|
||||
}
|
||||
|
||||
// chainRecentMade is the artifacts and bases of the KeptBuilds newest successful builds of each registered module.
|
||||
func (i *Inventory) chainRecentMade(ctx context.Context, c *chain.Chain) ([]Build, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx, `select name from module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
registered := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
registered[name] = true
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ok []Build
|
||||
for _, b := range worked(all) {
|
||||
if registered[b.Module] {
|
||||
ok = append(ok, b)
|
||||
}
|
||||
}
|
||||
sortByRecorded(ok)
|
||||
back := map[string]int{}
|
||||
var out []Build
|
||||
for _, b := range ok {
|
||||
back[b.Module]++
|
||||
if back[b.Module] <= KeptBuilds {
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// chainEveryReferenceMade is every successful build's artifacts, oldest build first.
|
||||
func (i *Inventory) chainEveryReferenceMade(ctx context.Context, c *chain.Chain) ([]Build, error) {
|
||||
all, err := chainBuilds(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ok := worked(all)
|
||||
sort.SliceStable(ok, func(a, b int) bool {
|
||||
if !ok[a].At.Equal(ok[b].At) {
|
||||
return ok[a].At.Before(ok[b].At)
|
||||
}
|
||||
return ok[a].ID < ok[b].ID
|
||||
})
|
||||
return ok, nil
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **A build keeps its reuse key and what it published, and comes back whole from the bucket** (novox/hq ADR 0408 §1,
|
||||
// issue 1090): an artifact by the digest of what it made, a package release by name and version.
|
||||
func TestABuildKeepsItsReuseKeyAndWhatItPublishedWhenKeptOnTheBus(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
b := Build{ID: "build-1", Repository: "novox/notes", Ref: "main", Module: "notes", Commit: "c1", On: "ace", Path: "modules/notes",
|
||||
Manifest: []byte(`{"module":"notes"}`), Against: []string{"artifact-store://base@sha256:aa"},
|
||||
Read: []ReadRepository{{Repository: "novox/other", Ref: "main"}},
|
||||
Sources: []BuildSource{{Paths: []string{"modules/notes/module.json"}}},
|
||||
SourceFingerprint: "fp", Asked: at, At: at.Add(time.Minute), ReuseKey: "rk-1",
|
||||
Made: []Artifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes@sha256:bb"},
|
||||
{Name: "sdk", Kind: "package", Reference: "mesh-sdk@1.4.0"}}}
|
||||
kept := toChainBuild(b)
|
||||
if kept.ReuseKey != "rk-1" {
|
||||
t.Errorf("the reuse key is %q", kept.ReuseKey)
|
||||
}
|
||||
if kept.Published == nil || kept.Published.Artifact != "sha256:bb" || kept.Published.Package == nil ||
|
||||
kept.Published.Package.Name != "mesh-sdk" || kept.Published.Package.Version != "1.4.0" {
|
||||
t.Errorf("it published %+v", kept.Published)
|
||||
}
|
||||
// Through the bucket's bytes and back: nothing is lost.
|
||||
raw, err := json.Marshal(kept)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var back chain.Build
|
||||
if err := json.Unmarshal(raw, &back); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := fromChainBuild(back)
|
||||
b.Published = kept.Published
|
||||
if !reflect.DeepEqual(got, b) {
|
||||
t.Fatalf("the build came back as\n %+v\nnot\n %+v", got, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatABuildPublishedIsReadFromWhatItMade(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
made []Artifact
|
||||
want *chain.Published
|
||||
}{
|
||||
"nothing": {nil, nil},
|
||||
"an archive with no digest": {[]Artifact{{Name: "config", Kind: "archive", Reference: "…/blobs/sha256:…"}}, nil},
|
||||
"an image": {[]Artifact{{Name: "a", Kind: "image", Reference: "artifact-store://a@sha256:11"}},
|
||||
&chain.Published{Artifact: "sha256:11"}},
|
||||
"the first digest stands": {[]Artifact{{Kind: "image", Reference: "x@sha256:11"}, {Kind: "archive", Reference: "y@sha256:22"}},
|
||||
&chain.Published{Artifact: "sha256:11"}},
|
||||
"a package": {[]Artifact{{Name: "sdk", Kind: "package", Reference: "mesh-sdk@2.0.1"}},
|
||||
&chain.Published{Package: &chain.Package{Name: "mesh-sdk", Version: "2.0.1"}}},
|
||||
"a package with no version": {[]Artifact{{Name: "sdk", Kind: "package", Reference: "mesh-sdk"}}, nil},
|
||||
} {
|
||||
if got := publishedOf(tc.made); !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("%s: published %+v, not %+v", name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The same content hashes alike, however the record spells it** (issue 280's fingerprints, now over the bucket's bytes
|
||||
// and not the store's jsonb text): key order and white space of a manifest change nothing.
|
||||
func TestTheSameBuildHashesAlikeWhateverTheSpellingOfItsRecord(t *testing.T) {
|
||||
a := Build{Manifest: []byte(`{"module": "notes", "version": "1"}`), Made: []Artifact{{Name: "n", Kind: "image", Reference: "r"}}}
|
||||
b := Build{Manifest: []byte("{\"version\":\"1\",\n \"module\":\"notes\"}"), Made: []Artifact{{Name: "n", Kind: "image", Reference: "r"}}}
|
||||
if chainFingerprint(a) != chainFingerprint(b) {
|
||||
t.Fatal("two spellings of one manifest hash apart")
|
||||
}
|
||||
b.Made[0].Reference = "other"
|
||||
if chainFingerprint(a) == chainFingerprint(b) {
|
||||
t.Fatal("two artifacts hash alike")
|
||||
}
|
||||
}
|
||||
|
||||
// **Newest request first, as the store's ordering was**: the newest ask, then the newest record, then the newest id.
|
||||
func TestBuildsAreOrderedNewestRequestFirst(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
builds := []Build{
|
||||
{ID: "old", Asked: t0, At: t0.Add(time.Hour)},
|
||||
{ID: "unasked", At: t0.Add(2 * time.Hour)},
|
||||
{ID: "new", Asked: t0.Add(3 * time.Hour), At: t0.Add(3*time.Hour + time.Minute)},
|
||||
{ID: "same-a", Asked: t0.Add(4 * time.Hour), At: t0.Add(5 * time.Hour)},
|
||||
{ID: "same-b", Asked: t0.Add(4 * time.Hour), At: t0.Add(5 * time.Hour)},
|
||||
}
|
||||
newestRequestOrder(builds)
|
||||
var got []string
|
||||
for _, b := range builds {
|
||||
got = append(got, b.ID)
|
||||
}
|
||||
if want := []string{"same-b", "same-a", "new", "unasked", "old"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ordered %v, not %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// **A bus that does not answer, a lease lost and a key written first are "not now"** (issue 1092): a merge or a report whose
|
||||
// write failed so is held and handed over again, as it is for a store that is away.
|
||||
func TestTheBusBeingAwayIsNotNowLikeTheStoreBeingAway(t *testing.T) {
|
||||
for name, err := range map[string]error{
|
||||
"a timeout": fmt.Errorf("the plan is not written: %w", nats.ErrTimeout),
|
||||
"no responders": fmt.Errorf("build 1: %w", nats.ErrNoResponders),
|
||||
"a closed bus": nats.ErrConnectionClosed,
|
||||
"the lease lost": fmt.Errorf("%w: another controller holds it", chain.ErrNotHeld),
|
||||
"a key moved": chain.ErrMoved,
|
||||
"a deadline": context.DeadlineExceeded,
|
||||
} {
|
||||
if !Unreachable(err) {
|
||||
t.Errorf("%s is read as an answer", name)
|
||||
}
|
||||
}
|
||||
for name, err := range map[string]error{
|
||||
"a value too large": chain.ErrTooLarge,
|
||||
"a key not taken": chain.ErrBadKey,
|
||||
"a plan moved": ErrPlanMoved,
|
||||
"a key that exists": chain.ErrExists,
|
||||
} {
|
||||
if Unreachable(err) {
|
||||
t.Errorf("%s is read as a bus being away: asking again gets the same answer", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// The links of the chain (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): a send that puts a build on a machine
|
||||
// writes, in the assignment of the module there, the build, what it published, its commit and the rollout that sent it, and
|
||||
// since when; the rollout lists the assignments it changed with the build before and after (the controller's send, in
|
||||
// release_plan.go). One read of an assignment then names the commit, the build and the rollout that put it there, and one read
|
||||
// of the rollout lists its assignments.
|
||||
|
||||
type rolloutKey struct{}
|
||||
|
||||
// RolloutModule is what a rollout sends of one module: the commit and the build of it.
|
||||
type RolloutModule struct{ Commit, Build string }
|
||||
|
||||
type rolloutOf struct {
|
||||
id string
|
||||
modules map[string]RolloutModule
|
||||
}
|
||||
|
||||
// WithRollout is a context for a send a rollout makes: what RecordSent writes in the assignments of the modules it carries
|
||||
// names this rollout and the build it sent. A module the send carries beside these, that another walk is moving, is named
|
||||
// by the build the mesh holds of its commit and by no rollout.
|
||||
func WithRollout(ctx context.Context, id string, modules map[string]RolloutModule) context.Context {
|
||||
return context.WithValue(ctx, rolloutKey{}, rolloutOf{id: id, modules: modules})
|
||||
}
|
||||
|
||||
// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the
|
||||
// commit. Empty when the assignments are not on the bus, and for a module not sent yet.
|
||||
func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]chain.Ref{}
|
||||
for _, k := range all {
|
||||
a := k.Value
|
||||
if a.Node == node && !a.Assigned.IsZero() && a.Stands() && (a.Build != "" || a.Commit != "") {
|
||||
out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links.
|
||||
func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []chain.Assignment
|
||||
for _, k := range all {
|
||||
if a := k.Value; a.Node == node && !a.Assigned.IsZero() && a.Stands() {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].Module < out[b].Module })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// recordRuns writes what a send put on a machine into the assignments, once the send is recorded: for each module it carried
|
||||
// a build of — by the commit — the build, what it published, the commit, the rollout and since when. **A module that runs the
|
||||
// commit already keeps what it names**, filled where it names nothing and never blanked, so a send that carries it again, or a
|
||||
// push by hand, changes nothing it knows. A module the machine is not assigned is no assignment to write.
|
||||
func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on || len(builds) == 0 {
|
||||
return err
|
||||
}
|
||||
var node string
|
||||
if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf)
|
||||
modules := make([]string, 0, len(builds))
|
||||
for m := range builds {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
var failed []error
|
||||
for _, m := range modules {
|
||||
commit := builds[m]
|
||||
if commit == "" {
|
||||
continue
|
||||
}
|
||||
if err := i.recordRun(ctx, c, node, m, commit, rollout, at); err != nil {
|
||||
failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(failed...)
|
||||
}
|
||||
|
||||
func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error {
|
||||
key := assignmentKey(node, module)
|
||||
var build, rolloutID string
|
||||
if sent, ours := rollout.modules[module]; ours && sent.Commit == commit {
|
||||
build, rolloutID = sent.Build, rollout.id
|
||||
}
|
||||
var published *chain.Published
|
||||
resolved := false
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Assignments.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found || cur.Value.Assigned.IsZero() || !cur.Value.Stands() {
|
||||
return nil // nothing assigned there to say what it runs
|
||||
}
|
||||
same := cur.Value.Commit == commit
|
||||
if same && cur.Value.Build != "" && (rolloutID == "" || cur.Value.Rollout != "") {
|
||||
return nil // a send that carries what the machine runs already says nothing new
|
||||
}
|
||||
if !resolved {
|
||||
// The build of the commit is looked up once, and only for a module that moved or names no build.
|
||||
if build == "" {
|
||||
if build, err = i.BuildOf(ctx, module, commit); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if build != "" {
|
||||
if b, found, err := i.BuildByID(ctx, build); err != nil {
|
||||
return err
|
||||
} else if found {
|
||||
published = publishedOf(b.Made)
|
||||
}
|
||||
}
|
||||
resolved = true
|
||||
}
|
||||
next := cur.Value
|
||||
if same {
|
||||
// Already running it: fill what is missing, blank nothing.
|
||||
if next.Build == "" {
|
||||
next.Build = build
|
||||
}
|
||||
if next.Rollout == "" {
|
||||
next.Rollout = rolloutID
|
||||
}
|
||||
if next.Since.IsZero() {
|
||||
next.Since = at.UTC()
|
||||
}
|
||||
} else {
|
||||
next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC()
|
||||
next.Artifact, next.Package = "", nil
|
||||
}
|
||||
if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build {
|
||||
next.Artifact, next.Package = published.Artifact, published.Package
|
||||
}
|
||||
if next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout &&
|
||||
next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) &&
|
||||
next.Since.Equal(cur.Value.Since) {
|
||||
return nil
|
||||
}
|
||||
if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the assignment kept moving under the write")
|
||||
}
|
||||
|
||||
// saidRunsNot says a send's links could not be written, to the log of whoever sent: the send is away and is recorded, and
|
||||
// a record of what it put where that could not be written is said, never a reason to call the send failed.
|
||||
func saidRunsNot(node string, err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s was sent its declaration, and the build, commit and rollout of each module "+
|
||||
"could not be written in its assignments: %v\n", node, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,663 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **The move** (novox/hq ADR 0408 §8, issue 981 note 7 Q2): the serving controller writes the rows of a kind into its
|
||||
// bucket at its start, holding the lease and after the buckets exist, and records the kind in chain_moved. Idempotent: a
|
||||
// key that exists is already moved and is never overwritten, so a bucket value written since is not laid over by the row
|
||||
// it came from. It counts the rows against the keys, logs the largest value, and **refuses** (a Refusal, which the
|
||||
// controller raises as an urgent condition) on a count that differs or a value over the bus's largest, with nothing
|
||||
// switched: a kind is recorded moved only once all of it is in the bucket.
|
||||
|
||||
// Refusal is a move that stopped without switching a kind, and why. It is permanent until the cause is put right; any
|
||||
// other error of a move (the bus away, a write not granted) is tried again.
|
||||
type Refusal struct {
|
||||
Kind, Why string
|
||||
}
|
||||
|
||||
func (r *Refusal) Error() string {
|
||||
return fmt.Sprintf("the move of %s to the bus is refused: %s", r.Kind, r.Why)
|
||||
}
|
||||
|
||||
// MoveReport is what a move of one kind did.
|
||||
type MoveReport struct {
|
||||
Kind string
|
||||
// Rows is how many records the tables held, Written how many keys this move made, and Present how many already
|
||||
// held a value (moved by an earlier start).
|
||||
Rows, Written, Present int
|
||||
// TableRows is how many rows of the tables those records came from: a request and its outcome are two rows and one
|
||||
// record.
|
||||
TableRows int
|
||||
// Largest is the biggest value, in bytes.
|
||||
Largest int
|
||||
// Already is a kind an earlier start moved: nothing was done.
|
||||
Already bool
|
||||
}
|
||||
|
||||
// mover is one kind of the chain: its rows, read from its tables, as the keys and values they become.
|
||||
type mover struct {
|
||||
kind string
|
||||
// rows reads the tables: each record's key and its value, whole, and how many rows of the tables they came from.
|
||||
rows func(ctx context.Context) ([]moveRow, int, error)
|
||||
// create writes one record under its key; chain.ErrExists when the key holds a value.
|
||||
create func(ctx context.Context, c *chain.Chain, row moveRow) error
|
||||
// keys are the keys the bucket holds.
|
||||
keys func(ctx context.Context, c *chain.Chain) ([]string, error)
|
||||
// count is how many rows the tables hold now, counted as rows reports them.
|
||||
count func(ctx context.Context, tx pgx.Tx) (int, error)
|
||||
// lock keeps the tables from taking a row while the switch is recorded.
|
||||
lock string
|
||||
// settle runs under that lock, with the rows read again: it makes the bucket hold what the rows say now. A row the old
|
||||
// controller changed after the first pass is in the bucket before the kind is recorded as moved, for before the switch
|
||||
// the table is the only truth.
|
||||
settle func(ctx context.Context, c *chain.Chain, rows []moveRow) error
|
||||
}
|
||||
|
||||
type moveRow struct {
|
||||
key string
|
||||
value any
|
||||
size int
|
||||
}
|
||||
|
||||
func (i *Inventory) movers() []mover {
|
||||
return []mover{{
|
||||
kind: KindBuilds,
|
||||
rows: func(ctx context.Context) ([]moveRow, int, error) {
|
||||
builds, err := i.allBuildRows(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
requests, err := i.allRequestRows(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
// A build request is the build's record in state asked, and the request stays beside the outcome that
|
||||
// answered it: one record under one id (issue 1091).
|
||||
byID := make(map[string]*chain.Build, len(builds))
|
||||
var order []string
|
||||
for _, b := range builds {
|
||||
v := toChainBuild(b)
|
||||
byID[b.ID] = &v
|
||||
order = append(order, b.ID)
|
||||
}
|
||||
for _, r := range requests {
|
||||
if v, answered := byID[r.ID]; answered {
|
||||
v.Request = toRequest(r)
|
||||
continue
|
||||
}
|
||||
v := askedRecord(r)
|
||||
if r.NotAsked != "" {
|
||||
// Never handed over: asked again once, after the move (askAgain).
|
||||
v.Request.Reask = chain.ReaskPending
|
||||
}
|
||||
byID[r.ID] = &v
|
||||
order = append(order, r.ID)
|
||||
}
|
||||
out := make([]moveRow, 0, len(order))
|
||||
for _, id := range order {
|
||||
body, err := json.Marshal(byID[id])
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, moveRow{key: chainBuildKey(id), value: *byID[id], size: len(body)})
|
||||
}
|
||||
return out, len(builds) + len(requests), nil
|
||||
},
|
||||
create: func(ctx context.Context, c *chain.Chain, row moveRow) error {
|
||||
_, err := c.Builds.Create(ctx, row.key, row.value.(chain.Build))
|
||||
return err
|
||||
},
|
||||
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Builds.Keys(ctx) },
|
||||
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
|
||||
var n int
|
||||
err := tx.QueryRow(ctx, `select (select count(*) from build) + (select count(*) from build_request)`).Scan(&n)
|
||||
return n, err
|
||||
},
|
||||
lock: `lock table build, build_request in share mode`,
|
||||
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
|
||||
return settleBucket(ctx, c.Builds, rows)
|
||||
},
|
||||
}, {
|
||||
kind: KindRollouts,
|
||||
rows: func(ctx context.Context) ([]moveRow, int, error) {
|
||||
plans, err := i.plans(ctx, `order by created`)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
merges, err := i.merges(ctx, ``)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
tablePlans := len(plans)
|
||||
if plans, err = holdMerges(plans, merges); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out := make([]moveRow, 0, len(plans))
|
||||
for _, p := range plans {
|
||||
body, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, moveRow{key: p.ID, value: p, size: len(body)})
|
||||
}
|
||||
return out, tablePlans + len(merges), nil
|
||||
},
|
||||
create: func(ctx context.Context, c *chain.Chain, row moveRow) error {
|
||||
_, err := chain.RolloutsOf[Plan](c).Create(ctx, row.key, row.value.(Plan))
|
||||
return err
|
||||
},
|
||||
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) {
|
||||
return chain.RolloutsOf[Plan](c).Keys(ctx)
|
||||
},
|
||||
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
|
||||
var n int
|
||||
err := tx.QueryRow(ctx, `select (select count(*) from release_plan) + (select count(*) from batched_merge)`).Scan(&n)
|
||||
return n, err
|
||||
},
|
||||
lock: `lock table release_plan, batched_merge in share mode`,
|
||||
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
|
||||
return settleBucket(ctx, chain.RolloutsOf[Plan](c), rows)
|
||||
},
|
||||
}, {
|
||||
kind: KindAssignments,
|
||||
rows: func(ctx context.Context) ([]moveRow, int, error) {
|
||||
assigned, err := i.assignedWithSends(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out := make([]moveRow, 0, len(assigned))
|
||||
for _, a := range assigned {
|
||||
body, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, moveRow{key: assignmentKey(a.Node, a.Module), value: a, size: len(body)})
|
||||
}
|
||||
return out, len(out), nil
|
||||
},
|
||||
create: createAssignment,
|
||||
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Assignments.Keys(ctx) },
|
||||
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
|
||||
var n int
|
||||
err := tx.QueryRow(ctx, `select count(*) from assignment`).Scan(&n)
|
||||
return n, err
|
||||
},
|
||||
lock: `lock table assignment in share mode`,
|
||||
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
|
||||
have := make(map[string]bool, len(rows))
|
||||
for _, r := range rows {
|
||||
have[r.key] = true
|
||||
if err := settleAssignment(ctx, c, r); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// An assignment the store no longer holds ended since the first pass: the bucket says so.
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for key, k := range all {
|
||||
if have[key] || k.Value.Assigned.IsZero() || !k.Value.Stands() {
|
||||
continue
|
||||
}
|
||||
k.Value.Ended = time.Now().UTC()
|
||||
if _, err := c.Assignments.Update(ctx, key, k.Value, k.Revision); err != nil && !errors.Is(err, chain.ErrMoved) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
// assignedWithSends is every assignment of the store as the chain keeps it: with the commit its machine was last sent for
|
||||
// the module (node.sent_builds), the build that is of it and what the build published, and since when (when the machine was
|
||||
// last sent). An assignment of a module the machine was never sent has none of those.
|
||||
func (i *Inventory) assignedWithSends(ctx context.Context) ([]chain.Assignment, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name, a.module, a.assigned, n.sent_at, n.sent_builds
|
||||
from assignment a join node n on n.id = a.node order by n.name, a.module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []chain.Assignment
|
||||
for rows.Next() {
|
||||
var a chain.Assignment
|
||||
var sentAt *time.Time
|
||||
var raw []byte
|
||||
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned, &sentAt, &raw); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
a.Assigned = a.Assigned.UTC()
|
||||
var sent map[string]string
|
||||
if len(raw) > 0 {
|
||||
_ = json.Unmarshal(raw, &sent)
|
||||
}
|
||||
if commit := sent[a.Module]; commit != "" && sentAt != nil {
|
||||
a.Commit, a.Since = commit, sentAt.UTC()
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The build of each commit, read once: the newest successful build of the module at it.
|
||||
type at struct{ module, commit string }
|
||||
var all []Build
|
||||
if _, moved, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if moved {
|
||||
if all, err = chainBuilds(ctx, i.attached()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if all, err = i.allBuildRows(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newest := map[at]Build{}
|
||||
for _, b := range all {
|
||||
k := at{b.Module, b.Commit}
|
||||
if b.Failed != "" || b.Module == "" {
|
||||
continue
|
||||
}
|
||||
if had, has := newest[k]; !has || b.At.After(had.At) || (b.At.Equal(had.At) && b.ID > had.ID) {
|
||||
newest[k] = b
|
||||
}
|
||||
}
|
||||
for k := range out {
|
||||
if out[k].Commit == "" {
|
||||
continue
|
||||
}
|
||||
b, found := newest[at{out[k].Module, out[k].Commit}]
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
out[k].Build = b.ID
|
||||
if published := publishedOf(b.Made); published != nil {
|
||||
out[k].Artifact, out[k].Package = published.Artifact, published.Package
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// holdMerges puts each batched merge into the record of the plan that answers it. A merge no plan answers — the plan_id
|
||||
// of a merge a failed walk carried is null, or names a plan the store never kept — is held, flagged waiting, by the
|
||||
// newest plan that lists it among its merges, else the record kept for the merges no plan answers: the record that held it
|
||||
// last.
|
||||
func holdMerges(plans []Plan, merges []BatchedMerge) ([]Plan, error) {
|
||||
waitingAt := -1
|
||||
at := make(map[string]int, len(plans))
|
||||
for k, p := range plans {
|
||||
at[p.ID] = k
|
||||
}
|
||||
newest := func(match func(Plan) bool) int {
|
||||
best := -1
|
||||
for k, p := range plans {
|
||||
if match(p) && (best < 0 || p.Created.After(plans[best].Created)) {
|
||||
best = k
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
for _, m := range merges {
|
||||
held := heldOf(m, m.Heard)
|
||||
holder, answered := at[m.Plan]
|
||||
if m.Plan == "" || !answered {
|
||||
held.Waiting = true
|
||||
holder = newest(func(p Plan) bool {
|
||||
if p.Delivery == nil {
|
||||
return false
|
||||
}
|
||||
for _, listed := range p.Delivery.Merges {
|
||||
if strings.EqualFold(listed.Repository, m.Repository) && listed.Commit == m.Commit {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
if holder < 0 {
|
||||
// Held by the record kept for the merges no plan answers, made with the others.
|
||||
if waitingAt < 0 {
|
||||
plans = append(plans, Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
|
||||
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
|
||||
waitingAt = len(plans) - 1
|
||||
}
|
||||
holder = waitingAt
|
||||
}
|
||||
}
|
||||
plans[holder].Held = append(plans[holder].Held, held)
|
||||
}
|
||||
return plans, nil
|
||||
}
|
||||
|
||||
// MoveChain moves every kind not moved yet, in order, and says what it did. It needs the chain attached and the lease
|
||||
// held (the chain's writes ask). It stops at the first kind that fails.
|
||||
func (i *Inventory) MoveChain(ctx context.Context, say func(format string, args ...any)) ([]MoveReport, error) {
|
||||
if say == nil {
|
||||
say = func(string, ...any) {}
|
||||
}
|
||||
if i.attached() == nil {
|
||||
return nil, ErrNoChain
|
||||
}
|
||||
var out []MoveReport
|
||||
for _, m := range i.movers() {
|
||||
r, err := i.moveKind(ctx, m, say)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// movePasses is how often a move reads the tables again when they took a row while it wrote: the controller before this
|
||||
// one may still have been writing as the lease changed hands.
|
||||
const movePasses = 5
|
||||
|
||||
func (i *Inventory) moveKind(ctx context.Context, m mover, say func(string, ...any)) (MoveReport, error) {
|
||||
report := MoveReport{Kind: m.kind}
|
||||
moved, err := i.ChainMoved(ctx)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if _, done := moved[m.kind]; done {
|
||||
report.Already = true
|
||||
return report, nil
|
||||
}
|
||||
for pass := 1; pass <= movePasses; pass++ {
|
||||
rows, tableRows, err := m.rows(ctx)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("the %s cannot be read from the tables: %w", m.kind, err)
|
||||
}
|
||||
report.Rows, report.TableRows, report.Largest = len(rows), tableRows, 0
|
||||
// Every value measured before any is written: a value over the bus's largest stops the move with nothing
|
||||
// written (ADR 0408 §6), never cut.
|
||||
for _, r := range rows {
|
||||
report.Largest = max(report.Largest, r.size)
|
||||
if r.size > broker.ChainValueBytes {
|
||||
return report, &Refusal{m.kind, fmt.Sprintf("the value of %s is %d bytes and the bus holds at most %d; "+
|
||||
"nothing is cut, so the record must be made smaller first", r.key, r.size, broker.ChainValueBytes)}
|
||||
}
|
||||
}
|
||||
report.Written, report.Present = 0, 0
|
||||
for _, r := range rows {
|
||||
switch err := m.create(ctx, i.attached(), r); {
|
||||
case err == nil:
|
||||
report.Written++
|
||||
case errors.Is(err, chain.ErrExists):
|
||||
report.Present++
|
||||
case errors.Is(err, chain.ErrBadKey):
|
||||
return report, &Refusal{m.kind, fmt.Sprintf("%s cannot be a key of the bus: %v", r.key, err)}
|
||||
default:
|
||||
return report, fmt.Errorf("%s %s is not written to the bus: %w", m.kind, r.key, err)
|
||||
}
|
||||
}
|
||||
// Rows against keys: every row's key is in the bucket.
|
||||
keys, err := m.keys(ctx, i.attached())
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("the keys of the %s cannot be read back: %w", m.kind, err)
|
||||
}
|
||||
have := make(map[string]bool, len(keys))
|
||||
for _, k := range keys {
|
||||
have[k] = true
|
||||
}
|
||||
for _, r := range rows {
|
||||
if !have[r.key] {
|
||||
return report, &Refusal{m.kind, fmt.Sprintf("the tables hold %d records and the bucket does not hold %s", len(rows), r.key)}
|
||||
}
|
||||
}
|
||||
settled, err := i.switchKind(ctx, m, report)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if settled {
|
||||
say("moved the %s to the bus: %d records, %d written now and %d already there, the largest %d bytes",
|
||||
m.kind, report.Rows, report.Written, report.Present, report.Largest)
|
||||
return report, nil
|
||||
}
|
||||
say("the %s took a row while they were moved, so they are read again (pass %d of %d)", m.kind, pass, movePasses)
|
||||
}
|
||||
return report, &Refusal{m.kind, fmt.Sprintf("the tables kept taking rows through %d passes; another controller is still writing them", movePasses)}
|
||||
}
|
||||
|
||||
// switchKind records the kind as moved, in one transaction that holds the tables against writes and finds their count
|
||||
// as the move read it: false when they took a row since.
|
||||
func (i *Inventory) switchKind(ctx context.Context, m mover, r MoveReport) (bool, error) {
|
||||
committed := false
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if _, err := tx.Exec(ctx, m.lock); err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := m.count(ctx, tx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if n != r.TableRows {
|
||||
return false, nil
|
||||
}
|
||||
// The store's mirror of the assignments writes nothing from here: it would lay values without the links over the keys
|
||||
// the settle writes and end the ones it ended.
|
||||
if m.kind == KindAssignments {
|
||||
i.moved.switching.Store(true)
|
||||
defer func() {
|
||||
if !committed {
|
||||
i.moved.switching.Store(false)
|
||||
}
|
||||
}()
|
||||
}
|
||||
// The rows read again under the lock: nothing writes them now, so what the bucket holds is made what they say.
|
||||
rows, tableRows, err := m.rows(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if tableRows != r.TableRows {
|
||||
return false, nil
|
||||
}
|
||||
if err := m.settle(ctx, i.attached(), rows); err != nil {
|
||||
return false, fmt.Errorf("the %s changed since they were first written, and the bucket cannot be brought to them: %w", m.kind, err)
|
||||
}
|
||||
counts, _ := json.Marshal(map[string]int{m.kind: r.Rows})
|
||||
largest, _ := json.Marshal(map[string]int{m.kind: r.Largest})
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into chain_moved (counts, largest) values ($1, $2)
|
||||
on conflict (only_one) do update
|
||||
set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest,
|
||||
moved_at = now()`, counts, largest); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
committed = true
|
||||
i.moved.mu.Lock()
|
||||
if i.moved.kind == nil {
|
||||
i.moved.kind = map[string]bool{}
|
||||
}
|
||||
i.moved.kind[m.kind] = true
|
||||
i.moved.mu.Unlock()
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// allBuildRows is every build of the table, whole.
|
||||
func (i *Inventory) allBuildRows(ctx context.Context) ([]Build, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select id, repository, ref, coalesce(module, ''), commit_hash, built_on, failed, made, source_path, manifest,
|
||||
built_against, built_contexts, build_sources, source_fingerprint, asked, at
|
||||
from build order by at, id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Build
|
||||
for rows.Next() {
|
||||
var b Build
|
||||
var made, manifest, against, read, sources []byte
|
||||
var asked *time.Time
|
||||
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path,
|
||||
&manifest, &against, &read, &sources, &b.SourceFingerprint, &asked, &b.At); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if asked != nil {
|
||||
b.Asked = *asked
|
||||
}
|
||||
b.Manifest = manifest
|
||||
for _, part := range []struct {
|
||||
raw []byte
|
||||
to any
|
||||
who string
|
||||
}{{made, &b.Made, "made"}, {against, &b.Against, "built_against"}, {read, &b.Read, "built_contexts"},
|
||||
{sources, &b.Sources, "build_sources"}} {
|
||||
if len(part.raw) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(part.raw, part.to); err != nil {
|
||||
return nil, fmt.Errorf("build %s: %s is not readable: %w", b.ID, part.who, err)
|
||||
}
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// allRequestRows is every build request of the table.
|
||||
func (i *Inventory) allRequestRows(ctx context.Context) ([]BuildRequest, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select id, repository, seat, source_path, ref, commit_hash, asked_for, coalesce(not_asked, ''),
|
||||
coalesce(outcome_unknown, ''), asked_at, at_terminal
|
||||
from build_request order by asked_at, id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []BuildRequest
|
||||
for rows.Next() {
|
||||
var a BuildRequest
|
||||
if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked,
|
||||
&a.OutcomeUnknown, &a.At, &a.AtTerminal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// createAssignment writes an assignment's row under its key; chain.ErrExists when the key holds a value that stays.
|
||||
func createAssignment(ctx context.Context, c *chain.Chain, row moveRow) error {
|
||||
want := row.value.(chain.Assignment)
|
||||
// The mirror of the store kept a value under this key until now, with the provisioner's plan beside it: the
|
||||
// plan stays, the record's links are laid in. A value that names its build already is a later word, and stays.
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Assignments.Get(ctx, row.key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
if _, err := c.Assignments.Create(ctx, row.key, want); !errors.Is(err, chain.ErrExists) {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if cur.Value.Build != "" || cur.Value.Commit != "" || !cur.Value.Since.IsZero() || (want.Build == "" && want.Commit == "") {
|
||||
return chain.ErrExists
|
||||
}
|
||||
merged := want
|
||||
merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning
|
||||
if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the assignment %s kept moving under the write", row.key)
|
||||
}
|
||||
|
||||
// settleBucket makes a bucket of whole values hold each row's value, by compare-and-set: a key that is absent is made, one
|
||||
// whose bytes differ from the row's is written over, and one that is the same is left.
|
||||
func settleBucket[T any](ctx context.Context, b chain.Bucket[T], rows []moveRow) error {
|
||||
for _, r := range rows {
|
||||
want, err := json.Marshal(r.value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
settled := false
|
||||
for try := 0; try < casTries && !settled; try++ {
|
||||
e, found, err := b.GetRaw(ctx, r.key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case !found:
|
||||
_, err = b.CreateRaw(ctx, r.key, want)
|
||||
if errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
}
|
||||
case bytes.Equal(e.Value, want):
|
||||
settled = true
|
||||
continue
|
||||
default:
|
||||
_, err = b.UpdateRaw(ctx, r.key, want, e.Revision)
|
||||
if errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
settled = true
|
||||
}
|
||||
if !settled {
|
||||
return fmt.Errorf("%s kept moving under the write", r.key)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// settleAssignment makes the value under an assignment's key what its row says, keeping the plan the mirror put beside it:
|
||||
// before the switch the store is the only truth of the links, so a commit it changed since the first pass is written over.
|
||||
func settleAssignment(ctx context.Context, c *chain.Chain, row moveRow) error {
|
||||
want := row.value.(chain.Assignment)
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Assignments.Get(ctx, row.key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
if _, err := c.Assignments.Create(ctx, row.key, want); errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
merged := want
|
||||
merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning
|
||||
a, _ := json.Marshal(merged)
|
||||
b, _ := json.Marshal(cur.Value)
|
||||
if bytes.Equal(a, b) {
|
||||
return nil
|
||||
}
|
||||
if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the assignment %s kept moving under the write", row.key)
|
||||
}
|
||||
@@ -0,0 +1,949 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// A build row as the store held it before the move: written by SQL, the only hand-written state these tests have, for
|
||||
// it stands for what the mesh held before this release.
|
||||
func aBuildRow(t *testing.T, inv *Inventory, id, module, commit, failed string) {
|
||||
t.Helper()
|
||||
inv.ExecForTest(t,
|
||||
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest,
|
||||
built_against, built_contexts, asked, source_fingerprint, build_sources)
|
||||
values ($1::text, 'novox/' || $2::text, 'main', $2::text, $3::text, 'ace', $4::text,
|
||||
case when $4::text = ''
|
||||
then ('[{"name":"app","kind":"image","reference":"artifact-store://' || $2::text || '@sha256:' || $1::text || '"}]')::jsonb
|
||||
else '[]'::jsonb end,
|
||||
'modules/' || $2::text, ('{"module":"' || $2::text || '"}')::jsonb,
|
||||
'["artifact-store://base@sha256:aa"]'::jsonb, '[{"repository":"novox/other","ref":"main"}]'::jsonb,
|
||||
now() - interval '1 hour', 'fp-' || $1::text,
|
||||
('[{"paths":["modules/' || $2::text || '/module.json"]}]')::jsonb)`,
|
||||
id, module, commit, failed)
|
||||
}
|
||||
|
||||
// inTheTables is an inventory before the move: its builds and assignments are rows of the store, which a test of a data
|
||||
// migration of the store (that reads those rows) needs.
|
||||
func inTheTables(t *testing.T) *Inventory {
|
||||
t.Helper()
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
return inv
|
||||
}
|
||||
|
||||
// reportOf is the report of one kind among those a move made, which are one for each kind in the order they are moved.
|
||||
func reportOf(t *testing.T, reports []MoveReport, kind string) MoveReport {
|
||||
t.Helper()
|
||||
for _, r := range reports {
|
||||
if r.Kind == kind {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("no report of %s among %+v", kind, reports)
|
||||
return MoveReport{}
|
||||
}
|
||||
|
||||
// allAlready says every kind was moved by an earlier start: the second run writes nothing, and reports one for each kind.
|
||||
func allAlready(t *testing.T, reports []MoveReport) {
|
||||
t.Helper()
|
||||
if len(reports) != 3 {
|
||||
t.Fatalf("the move reported %d kinds, want 3: %+v", len(reports), reports)
|
||||
}
|
||||
for _, r := range reports {
|
||||
if !r.Already || r.Written != 0 {
|
||||
t.Fatalf("the second move of %s %+v", r.Kind, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func rowsInTheTable(t *testing.T, inv *Inventory, table string) string {
|
||||
t.Helper()
|
||||
rows := inv.QueryForTest(t, `select count(*) from `+table)
|
||||
return rows[0][0]
|
||||
}
|
||||
|
||||
// **The move copies every build row into the builds bucket, and a second run writes nothing** (ADR 0408 §8, issue
|
||||
// 1090): each row under its id, whole; the kind recorded in chain_moved; and from then on a build is read from the
|
||||
// bucket and written to it, the table taking no more.
|
||||
func TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
aBuildRow(t, inv, "build-2", "notes", "c2", "")
|
||||
aBuildRow(t, inv, "build-3", "wiki", "", "compile error")
|
||||
|
||||
// Before the move the tables are read, and the bucket is empty.
|
||||
if before, err := inv.Builds(ctx, "", 10); err != nil || len(before) != 3 {
|
||||
t.Fatalf("before the move the tables hold %d builds: %v", len(before), err)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("moved before the move: %v", moved)
|
||||
}
|
||||
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.Written != 3 || b.Largest == 0 {
|
||||
t.Fatalf("the move of the builds reported %+v", reports)
|
||||
}
|
||||
moved, err := inv.ChainMoved(ctx)
|
||||
if err != nil || moved[KindBuilds] != 3 {
|
||||
t.Fatalf("chain_moved says %v: %v", moved, err)
|
||||
}
|
||||
kept, found, err := inv.chain.Builds.Get(ctx, "build-2")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("build-2 is not in the bucket: %v %v", found, err)
|
||||
}
|
||||
if v := kept.Value; v.Module != "notes" || v.Commit != "c2" || v.Repository != "novox/notes" || v.Path != "modules/notes" ||
|
||||
v.SourceFingerprint != "fp-build-2" || len(v.Made) != 1 || len(v.Against) != 1 || len(v.Read) != 1 || len(v.Sources) != 1 ||
|
||||
v.Published == nil || v.Published.Artifact != "sha256:build-2" || v.Asked.IsZero() || v.At.IsZero() || len(v.Manifest) == 0 {
|
||||
t.Fatalf("build-2 was moved as %+v", v)
|
||||
}
|
||||
|
||||
// From now on the bucket is what is read and written.
|
||||
if err := inv.RecordBuild(ctx, aBuild("build-4", "notes", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "build"); n != "3" {
|
||||
t.Fatalf("the table took a build after the move: %s rows", n)
|
||||
}
|
||||
after, err := inv.Builds(ctx, "notes", 10)
|
||||
if err != nil || len(after) != 3 {
|
||||
t.Fatalf("notes has %d builds on the bus: %v", len(after), err)
|
||||
}
|
||||
|
||||
// A second run finds the kind moved and writes nothing: every key still has the one value it was made with.
|
||||
again, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allAlready(t, again)
|
||||
for _, id := range []string{"build-1", "build-2", "build-3", "build-4"} {
|
||||
if history, err := inv.chain.Builds.History(ctx, chainBuildKey(id)); err != nil || len(history) != 1 {
|
||||
t.Fatalf("%s has %d values: %v", id, len(history), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// failingToCreate is a bucket that fails to make one key while told to: the bus going away half way through a move.
|
||||
type failingToCreate struct {
|
||||
chain.Backend
|
||||
key string
|
||||
fail *atomic.Bool
|
||||
}
|
||||
|
||||
func (f failingToCreate) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if key == f.key && f.fail.Load() {
|
||||
return 0, errors.New("the bus went away")
|
||||
}
|
||||
return f.Backend.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
// **A controller restarted half way through the move finishes it: the bucket equals the table, with no key made twice and
|
||||
// no refusal** (ADR 0408 §8). Before a kind is recorded as moved nothing but the move writes its keys, and every write of
|
||||
// the kind goes to its table (each method of a kind asks ChainReads first and writes the store until it says moved), so the
|
||||
// table is the only truth: a key the first start made is the row it was, and the second start settles it to the row it is.
|
||||
func TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
aBuildRow(t, inv, "build-2", "notes", "c2", "")
|
||||
aBuildRow(t, inv, "build-3", "wiki", "c3", "")
|
||||
var away atomic.Bool
|
||||
away.Store(true)
|
||||
builds := chain.NewMemory()
|
||||
inv.UseChain(chain.New(failingToCreate{builds, "build-3", &away}, chain.NewMemory(), chain.NewMemory(),
|
||||
func(context.Context) error { return nil }))
|
||||
// The first start made two of the three keys and the bus went away.
|
||||
_, err := inv.MoveChain(ctx, nil)
|
||||
var refusal *Refusal
|
||||
if err == nil || errors.As(err, &refusal) {
|
||||
t.Fatalf("the first start answered %v: it should fail to be tried again, not be refused", err)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("a move that did not finish recorded %v", moved)
|
||||
}
|
||||
if keys, _ := builds.Keys(ctx); len(keys) != 2 {
|
||||
t.Fatalf("the first start made %v", keys)
|
||||
}
|
||||
// The controller is started again, the bus is back, and the move finishes.
|
||||
away.Store(false)
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the second start was refused or failed: %v", err)
|
||||
}
|
||||
if b := reportOf(t, reports, KindBuilds); b.Written != 1 || b.Present != 2 || b.Rows != 3 {
|
||||
t.Fatalf("the move reported %+v", b)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); moved[KindBuilds] != 3 {
|
||||
t.Fatalf("chain_moved says %v", moved)
|
||||
}
|
||||
// The bucket is the table: three keys, each made once and not again, each the row's own.
|
||||
for _, id := range []string{"build-1", "build-2", "build-3"} {
|
||||
history, err := builds.History(ctx, id)
|
||||
if err != nil || len(history) != 1 {
|
||||
t.Fatalf("%s has %d values: %v", id, len(history), err)
|
||||
}
|
||||
}
|
||||
rows, err := inv.allBuildRows(ctx)
|
||||
if err != nil || len(rows) != 3 {
|
||||
t.Fatalf("the table holds %d builds: %v", len(rows), err)
|
||||
}
|
||||
for _, row := range rows {
|
||||
got, found, err := inv.chain.Builds.Get(ctx, row.ID)
|
||||
want, _ := json.Marshal(toChainBuild(row))
|
||||
have, _ := json.Marshal(got.Value)
|
||||
if err != nil || !found || string(want) != string(have) {
|
||||
t.Fatalf("%s in the bucket is %s, the row says %s (%v)", row.ID, have, want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A value over the bus's largest message stops the move with nothing written and nothing switched** (ADR 0408 §6):
|
||||
// a Refusal, no key in the bucket, no kind recorded, and the tables still what is read.
|
||||
func TestTheMoveRefusesAValueOverTheLargestMessage(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
inv.ExecForTest(t, `insert into build (id, repository, module, commit_hash, manifest)
|
||||
values ('build-big', 'novox/wiki', 'wiki', 'c9', ('{"x":"' || repeat('a', 1100000) || '"}')::jsonb)`)
|
||||
_, err := inv.MoveChain(ctx, nil)
|
||||
var refusal *Refusal
|
||||
if !errors.As(err, &refusal) || refusal.Kind != KindBuilds {
|
||||
t.Fatalf("the move answered %v", err)
|
||||
}
|
||||
if keys, _ := inv.chain.Builds.Keys(ctx); len(keys) != 0 {
|
||||
t.Fatalf("the bucket holds %v after a refused move", keys)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("a refused move recorded %v", moved)
|
||||
}
|
||||
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 2 {
|
||||
t.Fatalf("the tables are no longer read: %d builds, %v", len(builds), err)
|
||||
}
|
||||
}
|
||||
|
||||
// losing is a bucket that says it wrote a key and kept nothing: the move's count of keys against rows must see it.
|
||||
type losing struct {
|
||||
chain.Backend
|
||||
lose string
|
||||
}
|
||||
|
||||
func (l losing) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if key == l.lose {
|
||||
return 1, nil
|
||||
}
|
||||
return l.Backend.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
// **The move refuses on a count that differs** (ADR 0408 §8): a row whose key is not in the bucket afterwards stops it,
|
||||
// nothing is recorded, and the tables are still read.
|
||||
func TestTheMoveRefusesOnACountThatDiffers(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
aBuildRow(t, inv, "build-2", "notes", "c2", "")
|
||||
inv.UseChain(chain.New(losing{chain.NewMemory(), "build-2"}, chain.NewMemory(), chain.NewMemory(),
|
||||
func(context.Context) error { return nil }))
|
||||
_, err := inv.MoveChain(ctx, nil)
|
||||
var refusal *Refusal
|
||||
if !errors.As(err, &refusal) {
|
||||
t.Fatalf("the move answered %v", err)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("a refused move recorded %v", moved)
|
||||
}
|
||||
}
|
||||
|
||||
// **Without the lease the move writes nothing**, and fails the way the bus being away does — tried again, not refused:
|
||||
// the tables are still read.
|
||||
func TestTheMoveWithoutTheLeaseWritesNothing(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
mem := chain.NewMemory()
|
||||
inv.UseChain(chain.New(mem, chain.NewMemory(), chain.NewMemory(),
|
||||
func(context.Context) error { return errors.New("another controller holds the lease") }))
|
||||
_, err := inv.MoveChain(ctx, nil)
|
||||
var refusal *Refusal
|
||||
if err == nil || errors.As(err, &refusal) || !errors.Is(err, chain.ErrNotHeld) {
|
||||
t.Fatalf("the move answered %v", err)
|
||||
}
|
||||
if mem.Writes() != 0 {
|
||||
t.Fatalf("the bucket took %d writes without the lease", mem.Writes())
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("recorded %v", moved)
|
||||
}
|
||||
}
|
||||
|
||||
// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its
|
||||
// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none.
|
||||
func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
first := aBuild("build-1", "notes", "")
|
||||
first.Commit = "c1"
|
||||
first.Against = []string{"artifact-store://base@sha256:aa"}
|
||||
first.ReuseKey = "rk-1"
|
||||
second := aBuild("build-2", "notes", "")
|
||||
second.Commit = "c2"
|
||||
failed := aBuild("build-3", "notes", "compile error")
|
||||
other := aBuild("build-4", "wiki", "")
|
||||
for _, b := range []Build{first, second, failed, other, first} {
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "build"); n != "0" {
|
||||
t.Fatalf("the table took %s builds", n)
|
||||
}
|
||||
kept, found, err := inv.chain.Builds.Get(ctx, "build-1")
|
||||
if err != nil || !found || kept.Value.ReuseKey != "rk-1" {
|
||||
t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err)
|
||||
}
|
||||
if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 1 {
|
||||
t.Fatalf("a build recorded twice has %d values", len(history))
|
||||
}
|
||||
list, err := inv.Builds(ctx, "notes", 10)
|
||||
if err != nil || len(list) != 3 {
|
||||
t.Fatalf("notes has %d builds: %v", len(list), err)
|
||||
}
|
||||
one, found, err := inv.BuildByID(ctx, "build-2")
|
||||
if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
|
||||
t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
|
||||
}
|
||||
if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
|
||||
t.Fatal("a build nobody recorded was found")
|
||||
}
|
||||
if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" {
|
||||
t.Fatalf("the build of notes at c1 is %q: %v", id, err)
|
||||
}
|
||||
held, err := inv.Held(ctx)
|
||||
if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" {
|
||||
t.Fatalf("held %v: %v", held, err)
|
||||
}
|
||||
against, err := inv.BuiltAgainst(ctx)
|
||||
if err != nil || len(against["notes"]) != 0 {
|
||||
// The newest successful build of notes, build-2, stood on nothing.
|
||||
t.Fatalf("built against %v: %v", against, err)
|
||||
}
|
||||
announced, err := inv.Announceable(ctx)
|
||||
if err != nil || len(announced) != 3 {
|
||||
t.Fatalf("announceable %d: %v", len(announced), err)
|
||||
}
|
||||
prints, err := inv.Fingerprints(ctx)
|
||||
if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
|
||||
t.Fatalf("fingerprints %v: %v", prints, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build request row as the store held it before the move.
|
||||
func aRequestRow(t *testing.T, inv *Inventory, id, notAsked string) {
|
||||
t.Helper()
|
||||
inv.ExecForTest(t,
|
||||
`insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at)
|
||||
values ($1::text, 'novox/notes', '', 'modules/notes', 'main', 'c1', 'merge', nullif($2::text, ''), now() - interval '1 hour')`,
|
||||
id, notAsked)
|
||||
}
|
||||
|
||||
// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue
|
||||
// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none
|
||||
// after the outcome is heard. The table of requests takes none.
|
||||
func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true}
|
||||
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
|
||||
t.Fatalf("a request kept twice: %v", err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "build_request"); n != "0" {
|
||||
t.Fatalf("the table took %s requests", n)
|
||||
}
|
||||
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 {
|
||||
t.Fatalf("a request nobody answered is a build: %+v %v", builds, err)
|
||||
}
|
||||
if _, found, _ := inv.BuildByID(ctx, "build-1"); found {
|
||||
t.Fatal("a request nobody answered has an outcome")
|
||||
}
|
||||
got, found, err := inv.BuildRequestByID(ctx, "build-1")
|
||||
if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() {
|
||||
t.Fatalf("the request reads back as %+v %v %v", got, found, err)
|
||||
}
|
||||
if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, err := inv.RequestsNamed(ctx, "notes")
|
||||
if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" {
|
||||
t.Fatalf("the request is %+v: %v", asked, err)
|
||||
}
|
||||
if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) {
|
||||
t.Fatalf("requested names %v: %v", names, err)
|
||||
}
|
||||
|
||||
// The outcome comes: the same key, now a build, with the request beside it.
|
||||
built := aBuild("build-1", "notes", "")
|
||||
if err := inv.RecordBuild(ctx, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" {
|
||||
t.Fatalf("the outcome reads as %+v %v %v", one, found, err)
|
||||
}
|
||||
answered, err := inv.RequestsNamed(ctx, "notes")
|
||||
if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" {
|
||||
t.Fatalf("the request answered is %+v: %v", answered, err)
|
||||
}
|
||||
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" {
|
||||
t.Fatalf("the request was lost with the outcome: %+v", again)
|
||||
}
|
||||
// Heard: the marks of an unanswered request change nothing.
|
||||
if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" {
|
||||
t.Fatalf("an answered request was marked not asked: %+v", again)
|
||||
}
|
||||
if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 3 {
|
||||
// asked, marked unknown, answered.
|
||||
t.Fatalf("the record has %d values", len(history))
|
||||
}
|
||||
// An outcome heard before its request is kept, and the request lands beside it.
|
||||
if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" {
|
||||
t.Fatalf("the request kept after its outcome reads %+v", late)
|
||||
}
|
||||
// A request older than the time they are kept is no longer listed.
|
||||
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old",
|
||||
At: time.Now().Add(-2 * KeptFor)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") {
|
||||
t.Fatalf("a request past its time is listed: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a
|
||||
// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it
|
||||
// is asked and so asked at most once.
|
||||
func TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
aRequestRow(t, inv, "build-1", "")
|
||||
aRequestRow(t, inv, "build-2", "")
|
||||
aRequestRow(t, inv, "not-asked-c1-modules-notes", "no seat holder")
|
||||
|
||||
if requests, err := inv.RequestsNamed(ctx, "notes"); err != nil || len(requests) != 3 {
|
||||
t.Fatalf("before the move the table holds %d requests: %v", len(requests), err)
|
||||
}
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.TableRows != 4 {
|
||||
t.Fatalf("the move reported %+v", b)
|
||||
}
|
||||
one, _, _ := inv.chain.Builds.Get(ctx, "build-1")
|
||||
if one.Value.State != "" || one.Value.Request == nil || one.Value.Request.For != "merge" || one.Value.Module != "notes" {
|
||||
t.Fatalf("an answered request moved as %+v", one.Value)
|
||||
}
|
||||
two, _, _ := inv.chain.Builds.Get(ctx, "build-2")
|
||||
if two.Value.State != chain.BuildAsked || two.Value.Request.Reask != "" {
|
||||
t.Fatalf("an unanswered request moved as %+v", two.Value)
|
||||
}
|
||||
never, _, _ := inv.chain.Builds.Get(ctx, "not-asked-c1-modules-notes")
|
||||
if never.Value.State != chain.BuildAsked || never.Value.Request.NotAsked != "no seat holder" || never.Value.Request.Reask != chain.ReaskPending {
|
||||
t.Fatalf("a request never handed over moved as %+v", never.Value)
|
||||
}
|
||||
requests, err := inv.RequestsNamed(ctx, "notes")
|
||||
if err != nil || len(requests) != 3 {
|
||||
t.Fatalf("after the move the bucket holds %d requests: %v", len(requests), err)
|
||||
}
|
||||
toAsk, err := inv.RequestsToAskAgain(ctx)
|
||||
if err != nil || len(toAsk) != 1 || toAsk[0].ID != "not-asked-c1-modules-notes" {
|
||||
t.Fatalf("to ask again: %+v %v", toAsk, err)
|
||||
}
|
||||
first, err := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
|
||||
second, err2 := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
|
||||
if err != nil || err2 != nil || !first || second {
|
||||
t.Fatalf("claimed %v then %v: %v %v", first, second, err, err2)
|
||||
}
|
||||
if left, _ := inv.RequestsToAskAgain(ctx); len(left) != 0 {
|
||||
t.Fatalf("a claimed request is still to be asked: %+v", left)
|
||||
}
|
||||
if err := inv.AskedAgainAs(ctx, toAsk[0].ID, "build-9"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if done, _, _ := inv.BuildRequestByID(ctx, toAsk[0].ID); done.NotAsked != "no seat holder" {
|
||||
t.Fatalf("the request lost its words: %+v", done)
|
||||
}
|
||||
}
|
||||
|
||||
// **The move carries every plan with the merges it answers, and the plans read the same from the bucket as from the
|
||||
// table** (ADR 0408 §8, ADR 0276, issue 1092): a batch with a merge, a walk that was built and rolled, a merge waiting
|
||||
// for a walk of its own, and a merge whose plan the store never kept — each in the record of the plan that held it last.
|
||||
func TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
batch := aBatch("plan-1", time.Now().Add(-3*time.Hour).UTC())
|
||||
walk := aBatch("plan-2", time.Now().Add(-2*time.Hour).UTC())
|
||||
walk.State, walk.Tier, walk.Tiers = PlanRolling, 1, [][]string{{"a"}, {"b"}}
|
||||
walk.Modules = map[string]*PlanModule{"a": {State: "built"}, "b": {}}
|
||||
walk.Delivery = &PlanDelivery{Awaits: "mesh-delivery", Merges: []PlanMerge{{Repository: "novox/mesh-catalog", Commit: "m-waiting"}}}
|
||||
for _, p := range []*Plan{batch, walk} {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
walk.Note = "second save"
|
||||
if err := inv.SavePlan(ctx, walk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []BatchedMerge{aMerge("m-batch", "plan-1"), aMerge("m-walk", "plan-2"), aMerge("m-dangling", "plan-never-kept")} {
|
||||
if added, err := inv.AddMerge(ctx, m); err != nil || !added {
|
||||
t.Fatalf("added %v: %v", added, err)
|
||||
}
|
||||
}
|
||||
waiting := aMerge("m-waiting", "plan-2")
|
||||
if _, err := inv.AddMerge(ctx, waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.AnswerMerge(ctx, waiting.Repository, waiting.Commit, "", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasPlans, err := inv.RecentPlans(ctx, 10)
|
||||
if err != nil || len(wasPlans) != 2 {
|
||||
t.Fatalf("the table holds %d plans: %v", len(wasPlans), err)
|
||||
}
|
||||
wasMerge, _, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")
|
||||
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var moved MoveReport
|
||||
for _, r := range reports {
|
||||
if r.Kind == KindRollouts {
|
||||
moved = r
|
||||
}
|
||||
}
|
||||
// Two plans, and the record of the merge whose plan the store never kept.
|
||||
if moved.Rows != 3 || moved.TableRows != 6 || moved.Written != 3 {
|
||||
t.Fatalf("the move of the rollouts reported %+v", moved)
|
||||
}
|
||||
nowPlans, err := inv.RecentPlans(ctx, 10)
|
||||
if err != nil || len(nowPlans) != 2 {
|
||||
t.Fatalf("the bucket holds %d plans: %v", len(nowPlans), err)
|
||||
}
|
||||
for k := range wasPlans {
|
||||
was, now := wasPlans[k], nowPlans[k]
|
||||
if was.ID != now.ID || was.State != now.State || was.Tier != now.Tier || was.Revision != now.Revision || was.Note != now.Note ||
|
||||
!was.Created.Equal(now.Created) || !reflect.DeepEqual(was.Tiers, now.Tiers) || len(was.Modules) != len(now.Modules) {
|
||||
t.Errorf("plan %s was %+v and is %+v", was.ID, was, now)
|
||||
}
|
||||
}
|
||||
if open, _ := inv.OpenPlans(ctx); len(open) != 1 || open[0].ID != "plan-2" {
|
||||
t.Fatalf("open plans %+v", open)
|
||||
}
|
||||
if batches, _ := inv.Batches(ctx); len(batches) != 1 || batches[0].ID != "plan-1" {
|
||||
t.Fatalf("batches %+v", batches)
|
||||
}
|
||||
if got, _ := inv.MergesOf(ctx, "plan-1"); len(got) != 1 || got[0].Commit != "m-batch" {
|
||||
t.Fatalf("the batch answers %+v", got)
|
||||
}
|
||||
nowMerge, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")
|
||||
if !known || nowMerge.Plan != "plan-2" || nowMerge.Branch != wasMerge.Branch || !nowMerge.Merged.Equal(wasMerge.Merged) ||
|
||||
!nowMerge.Heard.Equal(wasMerge.Heard) || !sameEvent(nowMerge.Event, wasMerge.Event) {
|
||||
t.Fatalf("a merge was %+v and is %+v", wasMerge, nowMerge)
|
||||
}
|
||||
alone, _ := inv.AloneMerges(ctx)
|
||||
if len(alone) != 1 || alone[0].Commit != "m-waiting" || alone[0].Plan != "" {
|
||||
t.Fatalf("waiting: %+v", alone)
|
||||
}
|
||||
if dangling, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-dangling"); !known || dangling.Plan != "" || dangling.Alone {
|
||||
t.Fatalf("a merge whose plan was never kept reads as %+v %v", dangling, known)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
|
||||
t.Fatalf("the table is %s plans", n)
|
||||
}
|
||||
// From now on a plan is saved to the bucket and the table takes none.
|
||||
later := aBatch("plan-3", time.Now().UTC())
|
||||
if err := inv.SavePlan(ctx, later); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
|
||||
t.Fatalf("the table took a plan after the move: %s", n)
|
||||
}
|
||||
again, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allAlready(t, again)
|
||||
}
|
||||
|
||||
// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
|
||||
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
|
||||
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
|
||||
// assignment goes with it.
|
||||
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres", "web")
|
||||
ctx := t.Context()
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := generationNow(t, inv)
|
||||
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
|
||||
t.Fatalf("assigned %v: %v", added, err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added {
|
||||
t.Fatalf("assigned again %v: %v", added, err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "assignment"); n != "0" {
|
||||
t.Fatalf("the table took %s assignments", n)
|
||||
}
|
||||
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
|
||||
if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
|
||||
t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
|
||||
}
|
||||
if generationNow(t, inv) <= start {
|
||||
t.Fatal("an assignment left the generation where it was")
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
|
||||
t.Fatalf("a module the mesh does not know: %v", err)
|
||||
}
|
||||
if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) {
|
||||
t.Fatalf("assigned on %s: %v %v", node, on, err)
|
||||
}
|
||||
if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) {
|
||||
t.Fatalf("running: %v %v", running, err)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
switch e.Manifest.Module {
|
||||
case "postgres":
|
||||
if !reflect.DeepEqual(e.On, []string{node}) {
|
||||
t.Errorf("postgres is on %v", e.On)
|
||||
}
|
||||
case "web":
|
||||
if len(e.On) != 0 {
|
||||
t.Errorf("web is on %v", e.On)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) {
|
||||
t.Fatalf("a module still assigned was forgotten: %v", err)
|
||||
}
|
||||
// A seat is held by an assignment, or by nothing.
|
||||
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil {
|
||||
t.Fatal("a seat was handed to a module that is not assigned")
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil {
|
||||
t.Fatalf("a seat handed to an assignment: %v", err)
|
||||
}
|
||||
|
||||
before := generationNow(t, inv)
|
||||
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if generationNow(t, inv) <= before {
|
||||
t.Fatal("an unassignment left the generation where it was")
|
||||
}
|
||||
// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
|
||||
gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
|
||||
if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
|
||||
t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err)
|
||||
}
|
||||
if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() ||
|
||||
history[1].Value.Stands() {
|
||||
t.Fatalf("the earlier values of the key: %+v", history)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
|
||||
t.Fatalf("an ended assignment is still assigned: %v", on)
|
||||
}
|
||||
if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 {
|
||||
t.Fatalf("an ended assignment is running: %v", running)
|
||||
}
|
||||
if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 {
|
||||
t.Fatalf("the seat the assignment held is held by %+v: %v", held, err)
|
||||
}
|
||||
if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" {
|
||||
t.Fatalf("an unassignment of nothing: %v", err)
|
||||
}
|
||||
if err := inv.ForgetModule(ctx, "postgres"); err != nil {
|
||||
t.Fatalf("a module nothing is assigned could not be forgotten: %v", err)
|
||||
}
|
||||
// Made again after it ended: a fresh value over the ended one, which stays in history.
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
|
||||
t.Fatal(added, err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || added {
|
||||
t.Fatal(added, err)
|
||||
}
|
||||
if err := inv.Unassign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
|
||||
t.Fatalf("assigned again after it ended: %v %v", added, err)
|
||||
}
|
||||
again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
|
||||
if !again.Value.Stands() || !again.Value.Ended.IsZero() {
|
||||
t.Fatalf("the assignment made again is %+v", again.Value)
|
||||
}
|
||||
if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() {
|
||||
t.Fatalf("the values of the key: %+v", history)
|
||||
}
|
||||
}
|
||||
|
||||
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
|
||||
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
|
||||
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
|
||||
// ends an assignment.
|
||||
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "dunst", "web")
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"dunst", "web"} {
|
||||
if _, err := inv.Assign(ctx, node, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A build and a commit put in the value by another write.
|
||||
cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
|
||||
cur.Value.Build, cur.Value.Commit = "build-1", "c1"
|
||||
if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _ := inv.NodeByName(ctx, node)
|
||||
digest := "sha256:" + strings.Repeat("d", 64)
|
||||
plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
|
||||
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
|
||||
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
|
||||
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
written, removed, err := inv.KeepAssignmentPlans(ctx)
|
||||
if err != nil || written != 2 || removed != 0 {
|
||||
t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
|
||||
}
|
||||
dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
|
||||
if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
|
||||
t.Fatalf("dunst's value is %+v", dunst.Value)
|
||||
}
|
||||
if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
|
||||
t.Fatalf("web has a plan: %+v", web.Value)
|
||||
}
|
||||
notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify")
|
||||
if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil {
|
||||
t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
|
||||
t.Fatalf("a pulled-in module is assigned: %v", on)
|
||||
}
|
||||
if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 {
|
||||
t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err)
|
||||
}
|
||||
// Sent again without the pulled-in module: its value ends, the assignments stay.
|
||||
only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
|
||||
t.Fatalf("ended %d: %v", removed, err)
|
||||
}
|
||||
if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() {
|
||||
t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
|
||||
t.Fatalf("assigned: %v", on)
|
||||
}
|
||||
}
|
||||
|
||||
// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
|
||||
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
|
||||
// already is left as it is.
|
||||
func TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
for _, name := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.AddNode(ctx, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"notes", "wiki"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "notes"}, {"anchor", "wiki"}, {"laptop", "notes"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
aBuildRow(t, inv, "build-notes", "notes", "c-notes", "")
|
||||
anchor, _ := inv.NodeByName(ctx, "anchor")
|
||||
if err := inv.RecordSent(ctx, anchor.ID, "sent-1", map[string]string{"notes": "c-notes"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The mirror of the store put a value under one key, with a plan.
|
||||
if _, err := inv.chain.Assignments.Create(ctx, "anchor.wiki", chain.Assignment{Node: "anchor", Module: "wiki",
|
||||
Assigned: time.Now().UTC(), Provisioning: json.RawMessage(`{"plan":1}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var moved MoveReport
|
||||
for _, r := range reports {
|
||||
if r.Kind == KindAssignments {
|
||||
moved = r
|
||||
}
|
||||
}
|
||||
if moved.Rows != 3 || moved.Written+moved.Present != 3 {
|
||||
t.Fatalf("the move of the assignments reported %+v", moved)
|
||||
}
|
||||
notes, _, _ := inv.chain.Assignments.Get(ctx, "anchor.notes")
|
||||
if notes.Value.Commit != "c-notes" || notes.Value.Build != "build-notes" || notes.Value.Since.IsZero() || notes.Value.Assigned.IsZero() {
|
||||
t.Fatalf("anchor.notes moved as %+v", notes.Value)
|
||||
}
|
||||
if notes.Value.Artifact != "sha256:build-notes" {
|
||||
t.Fatalf("what the build published: %+v", notes.Value)
|
||||
}
|
||||
laptop, _, _ := inv.chain.Assignments.Get(ctx, "laptop.notes")
|
||||
if laptop.Value.Commit != "" || laptop.Value.Build != "" {
|
||||
t.Fatalf("a machine never sent has a build: %+v", laptop.Value)
|
||||
}
|
||||
wiki, _, _ := inv.chain.Assignments.Get(ctx, "anchor.wiki")
|
||||
if string(wiki.Value.Provisioning) != `{"plan":1}` {
|
||||
t.Fatalf("the plan the mirror put in the value was lost: %+v", wiki.Value)
|
||||
}
|
||||
// From now on the bucket is what is written.
|
||||
if _, err := inv.Assign(ctx, "laptop", "wiki"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "assignment"); n != "3" {
|
||||
t.Fatalf("the table took an assignment after the move: %s", n)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, "laptop"); !reflect.DeepEqual(on, []string{"notes", "wiki"}) {
|
||||
t.Fatalf("assigned on laptop: %v", on)
|
||||
}
|
||||
}
|
||||
|
||||
// hooked runs a function the first time a key is made in a bucket: the old controller writing the store between the move's
|
||||
// first pass and its switch.
|
||||
type hooked struct {
|
||||
chain.Backend
|
||||
once *sync.Once
|
||||
then func()
|
||||
}
|
||||
|
||||
func (h hooked) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
rev, err := h.Backend.Create(ctx, key, value)
|
||||
h.once.Do(h.then)
|
||||
return rev, err
|
||||
}
|
||||
|
||||
// **A row the old controller changes after the move's first pass is in the bucket when the kind is recorded as moved**
|
||||
// (ADR 0408 §8, issue 981 note 7 Q2): before the switch the store is the only truth, so under the lock the move reads the
|
||||
// rows again and writes over every key that differs — a plan's state and a build's failure — and ends an assignment the
|
||||
// store dropped. The count alone would have kept the older value and refused the next save of the plan.
|
||||
func TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
for _, n := range []string{"anchor"} {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"notes", "wiki"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"notes", "wiki"} {
|
||||
if _, err := inv.Assign(ctx, "anchor", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p := aBatch("plan-1", time.Now().Add(-time.Hour).UTC())
|
||||
p.State, p.Tiers = PlanBuilding, [][]string{{"notes"}}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Between a kind's first pass and its switch the old controller saves the plan again, fails the build and unassigns wiki:
|
||||
// each is done when the first key of its kind is made, so the first pass has read the rows as they were.
|
||||
after := func(sql string) hooked {
|
||||
return hooked{chain.NewMemory(), &sync.Once{}, func() { inv.ExecForTest(t, sql) }}
|
||||
}
|
||||
inv.UseChain(chain.New(
|
||||
after(`update build set failed = 'it broke later' where id = 'build-1'`),
|
||||
after(`update release_plan set state = 'rolling', revision = revision + 1, note = 'the old controller was here' where id = 'plan-1'`),
|
||||
after(`delete from assignment where module = 'wiki'`),
|
||||
func(context.Context) error { return nil }))
|
||||
if _, err := inv.MoveChain(ctx, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now, err := inv.PlanByID(ctx, "plan-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if now.State != PlanRolling || now.Note != "the old controller was here" || now.Revision != 2 {
|
||||
t.Fatalf("the bucket holds a plan from before the change: %s %q revision %d", now.State, now.Note, now.Revision)
|
||||
}
|
||||
if b, found, _ := inv.BuildByID(ctx, "build-1"); !found || b.Failed != "it broke later" {
|
||||
t.Fatalf("the bucket holds the build as it was: %+v", b)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, "anchor"); !slices.Equal(on, []string{"notes"}) {
|
||||
t.Fatalf("anchor is assigned %v, though the store dropped wiki", on)
|
||||
}
|
||||
// The plan can be saved at the revision the store had.
|
||||
now.Note = "saved after the move"
|
||||
if err := inv.SavePlan(ctx, &now); err != nil {
|
||||
t.Fatalf("the next save of the plan is refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// sameEvent says two encodings of the forge's announcement are one: the store keeps it as jsonb, which spells it with its own
|
||||
// white space, and whoever reads it (the cut, mesh-delivery) decodes it, so the white space is no part of what it says.
|
||||
func sameEvent(a, b json.RawMessage) bool {
|
||||
var x, y bytes.Buffer
|
||||
return json.Compact(&x, a) == nil && json.Compact(&y, b) == nil && x.String() == y.String()
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The move of the chain has a switch, made by the last migration and written by no migration** (ADR 0408 §8, issue
|
||||
// 1089): migration 0104 makes the one-row table chain_moved, and does not drop a table of the chain, which a later
|
||||
// delivery does once the move is read back.
|
||||
func TestMigration0104MakesTheSwitchOfTheChainsMoveAndNothingElse(t *testing.T) {
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, m := range migrations {
|
||||
if m.Number != 104 {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
sql := strings.ToLower(m.SQL)
|
||||
if !strings.Contains(sql, "create table chain_moved") || !strings.Contains(sql, "check (only_one)") {
|
||||
t.Errorf("migration 0104 does not make the one-row chain_moved:\n%s", m.SQL)
|
||||
}
|
||||
for _, line := range strings.Split(sql, "\n") {
|
||||
if strings.HasPrefix(strings.TrimSpace(line), "--") {
|
||||
continue
|
||||
}
|
||||
for _, verb := range []string{"drop table", "insert ", "delete ", "update "} {
|
||||
if strings.Contains(line, verb) {
|
||||
t.Errorf("migration 0104 does more than make the switch: %q", line)
|
||||
}
|
||||
}
|
||||
// The one other change: a seat's holder is no longer a foreign key to the assignment (issue 1093); the tables of
|
||||
// the chain are dropped by a later migration (issue 1096).
|
||||
if strings.Contains(line, "alter ") && !strings.Contains(line, "alter table seat_holding drop constraint") {
|
||||
t.Errorf("migration 0104 alters more than the seat holdings' foreign key: %q", line)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("no migration 0104")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// Build requests on the bus (novox/hq ADR 0408, issue 1091): a request is the build record in state asked, under the id its
|
||||
// outcome will carry; the outcome takes its place and keeps the request. The bucket removes no key, so a request is not
|
||||
// deleted after KeptFor: it is no longer listed.
|
||||
|
||||
func toRequest(a BuildRequest) *chain.Request {
|
||||
return &chain.Request{Repository: a.Repository, Seat: a.Seat, Path: strings.Trim(a.Path, "/"), Ref: a.Ref, Commit: a.Commit,
|
||||
For: a.For, NotAsked: a.NotAsked, OutcomeUnknown: a.OutcomeUnknown, At: a.At.UTC(), AtTerminal: a.AtTerminal}
|
||||
}
|
||||
|
||||
func fromRequest(id string, r chain.Request) BuildRequest {
|
||||
return BuildRequest{ID: id, Repository: r.Repository, Seat: r.Seat, Path: r.Path, Ref: r.Ref, Commit: r.Commit, For: r.For,
|
||||
NotAsked: r.NotAsked, OutcomeUnknown: r.OutcomeUnknown, At: r.At, AtTerminal: r.AtTerminal}
|
||||
}
|
||||
|
||||
// askedRecord is a request as the build record in state asked.
|
||||
func askedRecord(a BuildRequest) chain.Build {
|
||||
r := toRequest(a)
|
||||
return chain.Build{ID: a.ID, State: chain.BuildAsked, Repository: a.Repository, Ref: a.Ref, Path: r.Path, Asked: r.At, Request: r}
|
||||
}
|
||||
|
||||
func (i *Inventory) chainRecordBuildRequest(ctx context.Context, c *chain.Chain, a BuildRequest) error {
|
||||
if a.At.IsZero() {
|
||||
a.At = time.Now().UTC()
|
||||
}
|
||||
key := chainBuildKey(a.ID)
|
||||
for try := 0; try < casTries; try++ {
|
||||
_, err := c.Builds.Create(ctx, key, askedRecord(a))
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, chain.ErrExists) {
|
||||
return fmt.Errorf("build request %s is not kept on the bus: %w", a.ID, err)
|
||||
}
|
||||
// The key holds a value: the request is kept already, or its outcome came first and has no request beside it.
|
||||
cur, found, err := c.Builds.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found || cur.Value.Request != nil {
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
cur.Value.Request = toRequest(a)
|
||||
if _, err := c.Builds.Update(ctx, key, cur.Value, cur.Revision); err == nil {
|
||||
return nil
|
||||
} else if !errors.Is(err, chain.ErrMoved) {
|
||||
return fmt.Errorf("build request %s is not kept on the bus: %w", a.ID, err)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("build request %s is not kept on the bus: it kept moving under the write", a.ID)
|
||||
}
|
||||
|
||||
func (i *Inventory) chainBuildRequestByID(ctx context.Context, c *chain.Chain, id string) (BuildRequest, bool, error) {
|
||||
k, found, err := c.Builds.Get(ctx, chainBuildKey(id))
|
||||
if err != nil || !found || k.Value.Request == nil {
|
||||
return BuildRequest{}, false, err
|
||||
}
|
||||
return fromRequest(id, *k.Value.Request), true, nil
|
||||
}
|
||||
|
||||
// chainMarkRequest changes a request that has no outcome yet, by compare-and-set; one answered, or none kept, is left.
|
||||
func (i *Inventory) chainMarkRequest(ctx context.Context, c *chain.Chain, id string, mark func(*chain.Request)) error {
|
||||
key := chainBuildKey(id)
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Builds.Get(ctx, key)
|
||||
if err != nil || !found || cur.Value.State != chain.BuildAsked || cur.Value.Request == nil {
|
||||
return err
|
||||
}
|
||||
mark(cur.Value.Request)
|
||||
if _, err := c.Builds.Update(ctx, key, cur.Value, cur.Revision); err == nil {
|
||||
return nil
|
||||
} else if !errors.Is(err, chain.ErrMoved) {
|
||||
return fmt.Errorf("build request %s is not marked on the bus: %w", id, err)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("build request %s is not marked: it kept moving under the write", id)
|
||||
}
|
||||
|
||||
// chainRequests is every request kept, newest first, each with its outcome where its build was heard. Requests older than
|
||||
// KeptFor are not listed: the bucket keeps them, and no reader needs them.
|
||||
func (i *Inventory) chainRequests(ctx context.Context, c *chain.Chain) ([]RequestOutcome, error) {
|
||||
all, err := c.Builds.All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the builds cannot be read from the bus: %w", err)
|
||||
}
|
||||
oldest := time.Now().Add(-KeptFor)
|
||||
var out []RequestOutcome
|
||||
for id, k := range all {
|
||||
r := k.Value.Request
|
||||
if r == nil || !r.At.After(oldest) {
|
||||
continue
|
||||
}
|
||||
a := RequestOutcome{BuildRequest: fromRequest(id, *r), HeardAt: r.At}
|
||||
if k.Value.State != chain.BuildAsked {
|
||||
a.Heard, a.Failed, a.Module, a.HeardAt = true, k.Value.Failed, k.Value.Module, k.Value.At
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool {
|
||||
if !out[a].At.Equal(out[b].At) {
|
||||
return out[a].At.After(out[b].At)
|
||||
}
|
||||
return out[a].ID > out[b].ID
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RequestsToAskAgain are the requests the move found never handed over (issue 981 note 7 Q5), not yet asked again: each is
|
||||
// asked once, and never twice.
|
||||
func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindBuilds)
|
||||
if err != nil || !on {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.Builds.All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BuildRequest
|
||||
for id, k := range all {
|
||||
if k.Value.State == chain.BuildAsked && k.Value.Request != nil && k.Value.Request.Reask == chain.ReaskPending {
|
||||
out = append(out, fromRequest(id, *k.Value.Request))
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ClaimToAskAgain takes a request to ask again, by compare-and-set: true for the one process that took it. **Claimed
|
||||
// before it is asked and settled after**, so a crash between loses the ask rather than making it twice.
|
||||
func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindBuilds)
|
||||
if err != nil || !on {
|
||||
return false, err
|
||||
}
|
||||
cur, found, err := c.Builds.Get(ctx, chainBuildKey(id))
|
||||
if err != nil || !found || cur.Value.Request == nil || cur.Value.Request.Reask != chain.ReaskPending {
|
||||
return false, err
|
||||
}
|
||||
cur.Value.Request.Reask = chain.ReaskClaimed
|
||||
if _, err := c.Builds.Update(ctx, chainBuildKey(id), cur.Value, cur.Revision); err != nil {
|
||||
if errors.Is(err, chain.ErrMoved) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// AskedAgainAs records the build id a claimed request was asked as.
|
||||
func (i *Inventory) AskedAgainAs(ctx context.Context, id, asked string) error {
|
||||
c, on, err := i.ChainReads(ctx, KindBuilds)
|
||||
if err != nil || !on {
|
||||
return err
|
||||
}
|
||||
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.Reask = asked })
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **A request is the build record in state asked, under the id its outcome will carry** (ADR 0408 §1, issue 1091): the
|
||||
// whole ask is in it, with a path trimmed as the table kept it.
|
||||
func TestABuildRequestIsTheBuildRecordInStateAsked(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
ask := BuildRequest{ID: "build-1", Repository: "https://forge.invalid/notes.git", Seat: "", Path: "/modules/notes/", Ref: "main",
|
||||
Commit: "c1", For: "merge", NotAsked: "", OutcomeUnknown: "", At: at, AtTerminal: true}
|
||||
record := askedRecord(ask)
|
||||
if record.State != chain.BuildAsked || record.ID != "build-1" || record.Path != "modules/notes" || !record.Asked.Equal(at) {
|
||||
t.Fatalf("the request is %+v", record)
|
||||
}
|
||||
if got := fromRequest(record.ID, *record.Request); !reflect.DeepEqual(got, BuildRequest{ID: "build-1",
|
||||
Repository: ask.Repository, Path: "modules/notes", Ref: "main", Commit: "c1", For: "merge", At: at, AtTerminal: true}) {
|
||||
t.Fatalf("the request reads back as %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,939 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// Rollouts on the bus (novox/hq ADR 0408, ADR 0276, issue 1092): once the kind `rollouts` is moved (chain.go) every plan
|
||||
// and batch is a key of the rollouts bucket under its id, written by compare-and-set, and the merges a batch or walk
|
||||
// answers are held in its record. The tables are read when the kind is not moved, by the methods these stand beside.
|
||||
//
|
||||
// **Two counters.** The bucket's revision is the compare-and-set of every write. The plan's own Revision, kept in its value,
|
||||
// counts the saves of the plan: a save refuses (ErrPlanMoved) a plan another save moved since it was read, and takes no
|
||||
// notice of a merge moving in or out of the record, which a copy read before could not know of. Held is written by the
|
||||
// merge operations alone; a save keeps what the record holds.
|
||||
|
||||
// rolloutView is the rollouts bucket in memory, kept current by one watch (issue 1092): the survey of the plans, the
|
||||
// batches and the merges is read from here, never from the bucket one key at a time. A write is still made by
|
||||
// compare-and-set on the bucket, so a stale view costs a refused write and a read again, never a wrong one.
|
||||
type rolloutView struct {
|
||||
chain *chain.Chain
|
||||
mu sync.RWMutex
|
||||
entries map[string]viewEntry
|
||||
ready chan struct{}
|
||||
once sync.Once
|
||||
err error
|
||||
dead atomic.Bool
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// viewEntry is a record as the bucket holds it, with what the surveys ask of it read out.
|
||||
type viewEntry struct {
|
||||
raw []byte
|
||||
rev uint64
|
||||
sum planSummary
|
||||
}
|
||||
|
||||
// planSummary is the part of a plan the surveys filter and sort by.
|
||||
type planSummary struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
Created time.Time `json:"created"`
|
||||
Repository string `json:"repository"`
|
||||
Branch string `json:"branch"`
|
||||
Merged time.Time `json:"merged"`
|
||||
Release json.RawMessage `json:"release"`
|
||||
Held []chain.HeldMerge `json:"held"`
|
||||
}
|
||||
|
||||
func (s planSummary) open() bool { return s.State == PlanBuilding || s.State == PlanRolling }
|
||||
|
||||
func (s planSummary) isRelease() bool {
|
||||
r := strings.TrimSpace(string(s.Release))
|
||||
return r != "" && r != "null"
|
||||
}
|
||||
|
||||
func newRolloutView(c *chain.Chain) *rolloutView {
|
||||
return &rolloutView{chain: c, entries: map[string]viewEntry{}, ready: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (v *rolloutView) start() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
v.cancel = cancel
|
||||
go func() {
|
||||
w, err := chain.RolloutsOf[Plan](v.chain).Watch(ctx)
|
||||
if err != nil {
|
||||
v.fail(fmt.Errorf("the rollouts cannot be watched: %w", err))
|
||||
return
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
v.fail(ctx.Err())
|
||||
return
|
||||
case e, open := <-w:
|
||||
if !open {
|
||||
v.fail(errors.New("the rollouts bucket stopped answering"))
|
||||
return
|
||||
}
|
||||
if e.Caught {
|
||||
v.once.Do(func() { close(v.ready) })
|
||||
continue
|
||||
}
|
||||
v.apply(e.Key, e.Value, e.Revision)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// fail ends the view: it answers an error to whoever waits for it to fill, and is made again by the next read.
|
||||
func (v *rolloutView) fail(err error) {
|
||||
v.mu.Lock()
|
||||
if v.err == nil {
|
||||
v.err = err
|
||||
}
|
||||
v.mu.Unlock()
|
||||
v.dead.Store(true)
|
||||
v.once.Do(func() { close(v.ready) })
|
||||
}
|
||||
|
||||
func (v *rolloutView) stop() {
|
||||
if v.cancel != nil {
|
||||
v.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// apply keeps a record the watch handed over or a write of this process made, unless the view holds a newer one.
|
||||
func (v *rolloutView) apply(key string, raw []byte, rev uint64) {
|
||||
var sum planSummary
|
||||
if err := json.Unmarshal(raw, &sum); err != nil || sum.ID == "" {
|
||||
sum = planSummary{ID: key}
|
||||
}
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
if old, has := v.entries[key]; has && old.rev >= rev {
|
||||
return
|
||||
}
|
||||
v.entries[key] = viewEntry{raw: append([]byte(nil), raw...), rev: rev, sum: sum}
|
||||
}
|
||||
|
||||
func (v *rolloutView) list() []viewEntry {
|
||||
v.mu.RLock()
|
||||
defer v.mu.RUnlock()
|
||||
out := make([]viewEntry, 0, len(v.entries))
|
||||
for _, e := range v.entries {
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole.
|
||||
func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) {
|
||||
i.moved.mu.Lock()
|
||||
v := i.moved.view
|
||||
if v == nil || v.chain != c || v.dead.Load() {
|
||||
if v != nil {
|
||||
v.stop()
|
||||
}
|
||||
v = newRolloutView(c)
|
||||
i.moved.view = v
|
||||
v.start()
|
||||
}
|
||||
i.moved.mu.Unlock()
|
||||
select {
|
||||
case <-v.ready:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
v.mu.RLock()
|
||||
defer v.mu.RUnlock()
|
||||
if v.err != nil && len(v.entries) == 0 {
|
||||
return nil, fmt.Errorf("the rollouts cannot be read from the bus: %w", v.err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// stopViews ends the view's watch: the inventory is closing.
|
||||
func (i *Inventory) stopViews() {
|
||||
i.moved.mu.Lock()
|
||||
defer i.moved.mu.Unlock()
|
||||
if i.moved.view != nil {
|
||||
i.moved.view.stop()
|
||||
i.moved.view = nil
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePlan(p *Plan) {
|
||||
if p.Modules == nil {
|
||||
p.Modules = map[string]*PlanModule{}
|
||||
}
|
||||
if p.TierEntered.IsZero() {
|
||||
p.TierEntered = p.Created
|
||||
}
|
||||
if !p.Merged.IsZero() {
|
||||
p.Merged = p.Merged.UTC()
|
||||
}
|
||||
}
|
||||
|
||||
func planOf(e viewEntry) (Plan, error) {
|
||||
var p Plan
|
||||
if err := json.Unmarshal(e.raw, &p); err != nil {
|
||||
return Plan{}, fmt.Errorf("the plan %s on the bus cannot be read: %w", e.sum.ID, err)
|
||||
}
|
||||
normalizePlan(&p)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func sortCreated(entries []viewEntry) {
|
||||
sort.SliceStable(entries, func(a, b int) bool {
|
||||
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
|
||||
return entries[a].sum.Created.Before(entries[b].sum.Created)
|
||||
}
|
||||
return entries[a].sum.ID < entries[b].sum.ID
|
||||
})
|
||||
}
|
||||
|
||||
func sortCreatedNewest(entries []viewEntry) {
|
||||
sort.SliceStable(entries, func(a, b int) bool {
|
||||
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
|
||||
return entries[a].sum.Created.After(entries[b].sum.Created)
|
||||
}
|
||||
return entries[a].sum.ID > entries[b].sum.ID
|
||||
})
|
||||
}
|
||||
|
||||
// chainPlans is the plans that pass keep, in an order, read from the view.
|
||||
func (i *Inventory) chainPlans(ctx context.Context, c *chain.Chain, keep func(viewEntry) bool, order func([]viewEntry)) ([]Plan, error) {
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var chosen []viewEntry
|
||||
for _, e := range v.list() {
|
||||
if e.sum.ID != WaitingRecord && keep(e) {
|
||||
chosen = append(chosen, e)
|
||||
}
|
||||
}
|
||||
order(chosen)
|
||||
out := make([]Plan, 0, len(chosen))
|
||||
for _, e := range chosen {
|
||||
p, err := planOf(e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainNewestMergeOf(ctx context.Context, c *chain.Chain, repository, branch string) (Plan, bool, error) {
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return Plan{}, false, err
|
||||
}
|
||||
var chosen []viewEntry
|
||||
for _, e := range v.list() {
|
||||
if strings.EqualFold(e.sum.Repository, repository) && e.sum.Branch == branch && !e.sum.Merged.IsZero() && !e.sum.isRelease() {
|
||||
chosen = append(chosen, e)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(chosen, func(a, b int) bool {
|
||||
if !chosen[a].sum.Merged.Equal(chosen[b].sum.Merged) {
|
||||
return chosen[a].sum.Merged.After(chosen[b].sum.Merged)
|
||||
}
|
||||
return chosen[a].sum.Created.After(chosen[b].sum.Created)
|
||||
})
|
||||
if len(chosen) == 0 {
|
||||
return Plan{}, false, nil
|
||||
}
|
||||
p, err := planOf(chosen[0])
|
||||
return p, err == nil, err
|
||||
}
|
||||
|
||||
// chainPlanByID is one plan, read from the bucket itself: the caller of the verb wants the plan as it is, and the view
|
||||
// learns of it.
|
||||
func (i *Inventory) chainPlanByID(ctx context.Context, c *chain.Chain, id string) (Plan, error) {
|
||||
if !chain.ValidKey(id) {
|
||||
// No plan has an id the bus takes no spelling of; it is a plain id no plan has.
|
||||
return Plan{}, fmt.Errorf("no plan %s", id)
|
||||
}
|
||||
e, found, err := chain.RolloutsOf[Plan](c).GetRaw(ctx, id)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
if !found {
|
||||
return Plan{}, fmt.Errorf("no plan %s", id)
|
||||
}
|
||||
if v, err := i.rolloutView(ctx, c); err == nil {
|
||||
v.apply(id, e.Value, e.Revision)
|
||||
}
|
||||
return planOf(viewEntry{raw: e.Value, rev: e.Revision, sum: planSummary{ID: id}})
|
||||
}
|
||||
|
||||
// ---- the merges
|
||||
|
||||
// mergeHolder is a record's copy of a merge.
|
||||
type mergeHolder struct {
|
||||
plan string
|
||||
m chain.HeldMerge
|
||||
rev uint64
|
||||
}
|
||||
|
||||
func mergeKey(repository, commit string) string { return strings.ToLower(repository) + "|" + commit }
|
||||
|
||||
// mergeHolders is every record's copy of each merge, by the merge's key.
|
||||
func mergeHolders(entries []viewEntry) map[string][]mergeHolder {
|
||||
out := map[string][]mergeHolder{}
|
||||
for _, e := range entries {
|
||||
for _, m := range e.sum.Held {
|
||||
k := mergeKey(m.Repository, m.Commit)
|
||||
out[k] = append(out[k], mergeHolder{plan: e.sum.ID, m: m, rev: e.rev})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// winner is the copy of a merge that counts: the one that entered its record last (ADR 0408, issue 1092 — a merge in two
|
||||
// records is the half-done move of one, and the record it entered last is the one it moved to), a merge a plan answers
|
||||
// before one no plan does, then the record written last.
|
||||
func winner(copies []mergeHolder) mergeHolder {
|
||||
best := copies[0]
|
||||
for _, c := range copies[1:] {
|
||||
switch {
|
||||
case c.m.Placed.After(best.m.Placed):
|
||||
best = c
|
||||
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting && !c.m.Waiting:
|
||||
best = c
|
||||
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting == c.m.Waiting && c.rev > best.rev:
|
||||
best = c
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func winners(entries []viewEntry) map[string]mergeHolder {
|
||||
out := map[string]mergeHolder{}
|
||||
for k, copies := range mergeHolders(entries) {
|
||||
out[k] = winner(copies)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func batchedOf(h mergeHolder) BatchedMerge {
|
||||
b := BatchedMerge{Repository: strings.ToLower(h.m.Repository), Branch: h.m.Branch, Commit: h.m.Commit,
|
||||
Heard: h.m.Heard.UTC(), Event: h.m.Event, Plan: h.plan, Alone: h.m.Alone}
|
||||
if !h.m.Merged.IsZero() {
|
||||
b.Merged = h.m.Merged.UTC()
|
||||
}
|
||||
if h.m.Waiting {
|
||||
b.Plan = ""
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func heldOf(m BatchedMerge, now time.Time) chain.HeldMerge {
|
||||
h := chain.HeldMerge{Repository: strings.ToLower(m.Repository), Branch: m.Branch, Commit: m.Commit, Heard: m.Heard.UTC(),
|
||||
Event: m.Event, Alone: m.Alone, Placed: now.UTC()}
|
||||
if !m.Merged.IsZero() {
|
||||
h.Merged = m.Merged.UTC()
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func (i *Inventory) chainWinners(ctx context.Context, c *chain.Chain) (map[string]mergeHolder, error) {
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return winners(v.list()), nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainMergeOf(ctx context.Context, c *chain.Chain, repository, commit string) (BatchedMerge, bool, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return BatchedMerge{}, false, err
|
||||
}
|
||||
h, ok := w[mergeKey(repository, commit)]
|
||||
if !ok {
|
||||
return BatchedMerge{}, false, nil
|
||||
}
|
||||
return batchedOf(h), true, nil
|
||||
}
|
||||
|
||||
// merged orders merges by the time they were made, those whose time is not known last, then by when they were heard.
|
||||
func sortMerges(ms []BatchedMerge, newest bool) {
|
||||
sort.SliceStable(ms, func(a, b int) bool {
|
||||
x, y := ms[a], ms[b]
|
||||
switch {
|
||||
case x.Merged.IsZero() != y.Merged.IsZero():
|
||||
return !x.Merged.IsZero()
|
||||
case !x.Merged.Equal(y.Merged):
|
||||
if newest {
|
||||
return x.Merged.After(y.Merged)
|
||||
}
|
||||
return x.Merged.Before(y.Merged)
|
||||
}
|
||||
if newest {
|
||||
return x.Heard.After(y.Heard)
|
||||
}
|
||||
return x.Heard.Before(y.Heard)
|
||||
})
|
||||
}
|
||||
|
||||
func (i *Inventory) chainMergesOf(ctx context.Context, c *chain.Chain, plan string) ([]BatchedMerge, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BatchedMerge
|
||||
for _, h := range w {
|
||||
if h.plan == plan && !h.m.Waiting {
|
||||
out = append(out, batchedOf(h))
|
||||
}
|
||||
}
|
||||
sortMerges(out, false)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainLaterMergesOf(ctx context.Context, c *chain.Chain, repository, branch string, after time.Time) ([]BatchedMerge, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BatchedMerge
|
||||
for _, h := range w {
|
||||
if !h.m.Waiting && strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && h.m.Merged.After(after) {
|
||||
out = append(out, batchedOf(h))
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(a, b int) bool { return out[a].Merged.After(out[b].Merged) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainKeptOnBranch(ctx context.Context, c *chain.Chain, repository, branch string) (bool, time.Time, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return false, time.Time{}, err
|
||||
}
|
||||
var first time.Time
|
||||
for _, h := range w {
|
||||
if strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && (first.IsZero() || h.m.Heard.Before(first)) {
|
||||
first = h.m.Heard
|
||||
}
|
||||
}
|
||||
if first.IsZero() {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
return true, first.UTC(), nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainAloneMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BatchedMerge
|
||||
for _, h := range w {
|
||||
if h.m.Waiting && h.m.Alone {
|
||||
out = append(out, batchedOf(h))
|
||||
}
|
||||
}
|
||||
sortMerges(out, true)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// pendingMerge is a merge added to a plan that is not saved yet: carried until that plan's save writes it.
|
||||
type pendingMerge struct {
|
||||
plan string
|
||||
m chain.HeldMerge
|
||||
}
|
||||
|
||||
// chainAddMerge keeps a merge for a plan. **When the plan has a record the merge is written into it at once**; when it has
|
||||
// none — a new batch, saved by the caller next — the merge is carried and written by that plan's save. The forge's event is
|
||||
// acknowledged only after the caller's write succeeded, and handed over again if it did not (ADR 0407 §3).
|
||||
func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m BatchedMerge) (bool, error) {
|
||||
waiting := m.Plan == ""
|
||||
if waiting {
|
||||
// A merge no plan answers is held, flagged, by the record kept for those.
|
||||
m.Plan = WaitingRecord
|
||||
if err := i.ensureWaitingRecord(ctx, c); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
key := mergeKey(m.Repository, m.Commit)
|
||||
if _, known := winners(v.list())[key]; known {
|
||||
return false, nil
|
||||
}
|
||||
held := heldOf(m, time.Now())
|
||||
held.Waiting = waiting
|
||||
found, err := i.editRecord(ctx, c, m.Plan, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
for _, x := range h {
|
||||
if mergeKey(x.Repository, x.Commit) == key {
|
||||
return h, false
|
||||
}
|
||||
}
|
||||
return append(h, held), true
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !found {
|
||||
i.moved.mu.Lock()
|
||||
if i.moved.pending == nil {
|
||||
i.moved.pending = map[string]pendingMerge{}
|
||||
}
|
||||
i.moved.pending[key] = pendingMerge{plan: m.Plan, m: held}
|
||||
i.moved.mu.Unlock()
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// WaitingRecord is the record that holds the merges no plan has ever answered: a merge kept with no plan, which a store
|
||||
// kept as a row with a null plan. It is no plan: the surveys of plans leave it out.
|
||||
const WaitingRecord = "waiting-merges"
|
||||
|
||||
// ensureWaitingRecord makes the record of the merges no plan answers, once.
|
||||
func (i *Inventory) ensureWaitingRecord(ctx context.Context, c *chain.Chain) error {
|
||||
b := chain.RolloutsOf[Plan](c)
|
||||
if _, found, err := b.GetRaw(ctx, WaitingRecord); err != nil || found {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
|
||||
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rev, err := b.CreateRaw(ctx, WaitingRecord, body)
|
||||
if errors.Is(err, chain.ErrExists) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v, verr := i.rolloutView(ctx, c); verr == nil {
|
||||
v.apply(WaitingRecord, body, rev)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrMergeUnsaved is a merge moved to a plan that has no record to hold it.
|
||||
var ErrMergeUnsaved = errors.New("a merge was moved to a plan that has not been saved")
|
||||
|
||||
// editRecord changes the merges a plan's record holds and nothing else of it, by compare-and-set on the bucket, read again
|
||||
// when it moved. It answers false when the plan has no record.
|
||||
func (i *Inventory) editRecord(ctx context.Context, c *chain.Chain, id string, edit func([]chain.HeldMerge) ([]chain.HeldMerge, bool)) (bool, error) {
|
||||
b := chain.RolloutsOf[Plan](c)
|
||||
for try := 0; try < casTries; try++ {
|
||||
e, found, err := b.GetRaw(ctx, id)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
var record map[string]json.RawMessage
|
||||
if err := json.Unmarshal(e.Value, &record); err != nil {
|
||||
return true, fmt.Errorf("the plan %s on the bus cannot be read: %w", id, err)
|
||||
}
|
||||
var held []chain.HeldMerge
|
||||
if raw, has := record["held"]; has {
|
||||
if err := json.Unmarshal(raw, &held); err != nil {
|
||||
return true, fmt.Errorf("the merges of plan %s on the bus cannot be read: %w", id, err)
|
||||
}
|
||||
}
|
||||
next, changed := edit(held)
|
||||
if !changed {
|
||||
return true, nil
|
||||
}
|
||||
if len(next) == 0 {
|
||||
delete(record, "held")
|
||||
} else {
|
||||
raw, err := json.Marshal(next)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
record["held"] = raw
|
||||
}
|
||||
body, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
rev, err := b.UpdateRaw(ctx, id, body, e.Revision)
|
||||
if errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("the merges of plan %s are not written: %w", id, err)
|
||||
}
|
||||
if v, verr := i.rolloutView(ctx, c); verr == nil {
|
||||
v.apply(id, body, rev)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return true, fmt.Errorf("the merges of plan %s kept moving under the write", id)
|
||||
}
|
||||
|
||||
// recordEdit is what the merges of one record gain, lose and have flagged in one act.
|
||||
type recordEdit struct {
|
||||
add []chain.HeldMerge
|
||||
drop map[string]bool
|
||||
flag map[string]chain.HeldMerge
|
||||
}
|
||||
|
||||
func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge {
|
||||
out := make([]chain.HeldMerge, 0, len(held)+len(e.add))
|
||||
for _, h := range held {
|
||||
k := mergeKey(h.Repository, h.Commit)
|
||||
if e.drop[k] {
|
||||
continue
|
||||
}
|
||||
if f, has := e.flag[k]; has {
|
||||
h.Waiting, h.Alone = f.Waiting, f.Alone
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
for _, a := range e.add {
|
||||
k := mergeKey(a.Repository, a.Commit)
|
||||
dup := false
|
||||
for _, h := range out {
|
||||
if mergeKey(h.Repository, h.Commit) == k {
|
||||
dup = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !dup {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (e *recordEdit) empty() bool { return len(e.add) == 0 && len(e.drop) == 0 && len(e.flag) == 0 }
|
||||
|
||||
// ---- the save
|
||||
|
||||
// tierLeftOf is the measurement a plan's save makes when it leaves a tier (planTierLeft, over the record the save
|
||||
// replaces): when the plan entered the tier it is now at, and the duration of the tier it left, if it left one.
|
||||
func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) {
|
||||
if old == nil {
|
||||
return now, nil // a new plan enters its first tier now
|
||||
}
|
||||
if old.State == PlanAssembling || old.State == PlanQueued {
|
||||
return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276)
|
||||
}
|
||||
since := old.TierEntered
|
||||
if since.IsZero() {
|
||||
since = old.Created
|
||||
}
|
||||
wasOpen := old.State == PlanBuilding || old.State == PlanRolling
|
||||
if !wasOpen || (old.Tier == p.Tier && p.Open()) {
|
||||
return since, nil // still in the tier, or already ended
|
||||
}
|
||||
var modules []string
|
||||
if old.Tier >= 0 && old.Tier < len(p.Tiers) {
|
||||
modules = p.Tiers[old.Tier]
|
||||
}
|
||||
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, old.Tier, len(p.Tiers), modules, p.State)
|
||||
if p.Open() {
|
||||
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, old.Tier, len(p.Tiers), modules, p.Tier)
|
||||
}
|
||||
return now, &Duration{Kind: DurationPlanTier, Subject: p.Repository, Ref: fmt.Sprintf("%s/tier-%d", p.ID, old.Tier),
|
||||
Started: since, Took: now.Sub(since), Detail: detail}
|
||||
}
|
||||
|
||||
// chainSavePlans writes plans, as SavePlan does each, and gives merges to the plans that answer them. **Each plan by
|
||||
// compare-and-set on its own Revision**, a copy read before another save refused with ErrPlanMoved before anything is
|
||||
// written; **a merge moved to a plan is written into the receiving record first, then taken from the giving one**, so a
|
||||
// crash between leaves it in both and never in neither, and the one it entered last counts (ADR 0408, issue 1092).
|
||||
func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []*Plan, moves []MergeMove) (err error) {
|
||||
epoch, err := i.actingEpoch(ctx)
|
||||
if err != nil {
|
||||
if len(plans) > 0 {
|
||||
return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b := chain.RolloutsOf[Plan](c)
|
||||
marked := time.Now()
|
||||
undo := make([]func(), 0, len(plans))
|
||||
defer func() {
|
||||
if err != nil {
|
||||
for _, u := range undo {
|
||||
u()
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, p := range plans {
|
||||
undo = append(undo, p.markWalk(marked))
|
||||
}
|
||||
|
||||
// Every plan still at the revision it was read at, before anything is written.
|
||||
cur := make(map[string]chain.Kept[Plan], len(plans))
|
||||
for _, p := range plans {
|
||||
k, found, err := b.Get(ctx, p.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
if k.Value.Revision != p.Revision {
|
||||
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
||||
}
|
||||
cur[p.ID] = k
|
||||
}
|
||||
}
|
||||
|
||||
// What each record gains, loses and has flagged.
|
||||
now := time.Now().UTC()
|
||||
edits := map[string]*recordEdit{}
|
||||
editOf := func(id string) *recordEdit {
|
||||
if edits[id] == nil {
|
||||
edits[id] = &recordEdit{drop: map[string]bool{}, flag: map[string]chain.HeldMerge{}}
|
||||
}
|
||||
return edits[id]
|
||||
}
|
||||
copies := mergeHolders(v.list())
|
||||
for _, m := range moves {
|
||||
key := mergeKey(m.Repository, m.Commit)
|
||||
held, has := copies[key]
|
||||
if !has {
|
||||
// A merge carried for a plan not saved yet goes where it is moved.
|
||||
i.moved.mu.Lock()
|
||||
if pm, pending := i.moved.pending[key]; pending && m.Plan != "" {
|
||||
pm.plan, pm.m.Alone = m.Plan, m.Alone
|
||||
i.moved.pending[key] = pm
|
||||
}
|
||||
i.moved.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
w := winner(held)
|
||||
if m.Plan == "" {
|
||||
f := w.m
|
||||
f.Waiting, f.Alone = true, m.Alone
|
||||
editOf(w.plan).flag[key] = f
|
||||
continue
|
||||
}
|
||||
if w.plan == m.Plan {
|
||||
f := w.m
|
||||
f.Waiting, f.Alone = false, m.Alone
|
||||
editOf(w.plan).flag[key] = f
|
||||
continue
|
||||
}
|
||||
moved := w.m
|
||||
moved.Waiting, moved.Alone, moved.Placed = false, m.Alone, now
|
||||
editOf(m.Plan).add = append(editOf(m.Plan).add, moved)
|
||||
for _, h := range held {
|
||||
editOf(h.plan).drop[key] = true
|
||||
}
|
||||
}
|
||||
|
||||
// The records not among the plans: receivers now, givers after.
|
||||
for id, e := range edits {
|
||||
if _, written := cur[id]; written || planNamed(plans, id) || len(e.add) == 0 {
|
||||
continue
|
||||
}
|
||||
e := e
|
||||
found, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
return (&recordEdit{add: e.add}).apply(h), true
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
// Nothing is taken from the record that holds a merge to give it to a plan that has none.
|
||||
return fmt.Errorf("%w: %s has no record to hold %d merge(s) moved to it", ErrMergeUnsaved, id, len(e.add))
|
||||
}
|
||||
}
|
||||
|
||||
// The plans, those that gain a merge first.
|
||||
order := append([]*Plan(nil), plans...)
|
||||
sort.SliceStable(order, func(a, b int) bool {
|
||||
x, y := edits[order[a].ID], edits[order[b].ID]
|
||||
return x != nil && len(x.add) > 0 && (y == nil || len(y.add) == 0)
|
||||
})
|
||||
type written struct {
|
||||
revision int64
|
||||
entered time.Time
|
||||
held []chain.HeldMerge
|
||||
duration *Duration
|
||||
}
|
||||
done := map[string]written{}
|
||||
for _, p := range order {
|
||||
var w written
|
||||
for try := 0; ; try++ {
|
||||
if try == casTries {
|
||||
return fmt.Errorf("the plan for %s %s (%s) kept moving under the write", p.Repository, short(p.Commit), p.ID)
|
||||
}
|
||||
k, found := cur[p.ID]
|
||||
if try > 0 {
|
||||
fresh, still, gerr := b.Get(ctx, p.ID)
|
||||
if gerr != nil {
|
||||
return gerr
|
||||
}
|
||||
if still && fresh.Value.Revision != p.Revision {
|
||||
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
||||
}
|
||||
k, found = fresh, still
|
||||
cur[p.ID] = k
|
||||
}
|
||||
record := *p
|
||||
var old *Plan
|
||||
var held []chain.HeldMerge
|
||||
if found {
|
||||
o := k.Value
|
||||
old, held = &o, append([]chain.HeldMerge(nil), k.Value.Held...)
|
||||
record.Revision = k.Value.Revision + 1
|
||||
} else {
|
||||
record.Revision = 1
|
||||
}
|
||||
if e := edits[p.ID]; e != nil {
|
||||
held = e.apply(held)
|
||||
}
|
||||
i.moved.mu.Lock()
|
||||
for key, pm := range i.moved.pending {
|
||||
if pm.plan == p.ID {
|
||||
held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held)
|
||||
_ = key
|
||||
}
|
||||
}
|
||||
i.moved.mu.Unlock()
|
||||
record.Held = held
|
||||
entered, duration := tierLeftOf(old, *p, time.Now())
|
||||
record.TierEntered, record.Updated = entered, time.Now().UTC()
|
||||
if epoch != nil {
|
||||
record.Epoch = uint64(*epoch)
|
||||
} else {
|
||||
record.Epoch = 0
|
||||
}
|
||||
body, merr := json.Marshal(record)
|
||||
if merr != nil {
|
||||
return merr
|
||||
}
|
||||
var rev uint64
|
||||
var werr error
|
||||
if found {
|
||||
rev, werr = b.UpdateRaw(ctx, p.ID, body, k.Revision)
|
||||
} else {
|
||||
rev, werr = createRaw(ctx, c, p.ID, body)
|
||||
}
|
||||
if errors.Is(werr, chain.ErrMoved) || (!found && errors.Is(werr, chain.ErrExists)) {
|
||||
if !found {
|
||||
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if werr != nil {
|
||||
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, werr)
|
||||
}
|
||||
v.apply(p.ID, body, rev)
|
||||
w = written{revision: record.Revision, entered: entered, held: held, duration: duration}
|
||||
break
|
||||
}
|
||||
done[p.ID] = w
|
||||
}
|
||||
|
||||
// The records that gave a merge up, now it is held by the one it went to.
|
||||
for id, e := range edits {
|
||||
if planNamed(plans, id) || (len(e.drop) == 0 && len(e.flag) == 0) {
|
||||
continue
|
||||
}
|
||||
e := e
|
||||
if _, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
return (&recordEdit{drop: e.drop, flag: e.flag}).apply(h), true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
i.moved.mu.Lock()
|
||||
for key, pm := range i.moved.pending {
|
||||
if _, saved := done[pm.plan]; saved {
|
||||
delete(i.moved.pending, key)
|
||||
}
|
||||
}
|
||||
i.moved.mu.Unlock()
|
||||
for _, p := range plans {
|
||||
w := done[p.ID]
|
||||
p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held
|
||||
if epoch != nil {
|
||||
p.Epoch = uint64(*epoch)
|
||||
} else {
|
||||
p.Epoch = 0
|
||||
}
|
||||
if w.duration != nil {
|
||||
// After the write: the key of the duration is the plan and its tier, so a measurement made twice is one.
|
||||
record := i.RecordDuration
|
||||
if i.recorded != nil {
|
||||
record = i.recorded
|
||||
}
|
||||
if derr := record(ctx, *w.duration); derr != nil {
|
||||
fmt.Printf("the time plan %s spent in a tier is not recorded: %v\n", p.ID, derr)
|
||||
}
|
||||
}
|
||||
if PlanSaved != nil {
|
||||
PlanSaved(i, *p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func planNamed(plans []*Plan, id string) bool {
|
||||
for _, p := range plans {
|
||||
if p.ID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// createRaw writes a plan's first record.
|
||||
func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uint64, error) {
|
||||
return chain.RolloutsOf[json.RawMessage](c).CreateRaw(ctx, id, body)
|
||||
}
|
||||
|
||||
// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired
|
||||
// at the next cut (ADR 0408, issue 1092). It says how many it repaired.
|
||||
func (i *Inventory) RepairMerges(ctx context.Context) (int, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindRollouts)
|
||||
if err != nil || !on {
|
||||
return 0, err
|
||||
}
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
repaired := 0
|
||||
for key, copies := range mergeHolders(v.list()) {
|
||||
if len(copies) < 2 {
|
||||
continue
|
||||
}
|
||||
w := winner(copies)
|
||||
for _, h := range copies {
|
||||
if h.plan == w.plan {
|
||||
continue
|
||||
}
|
||||
key := key
|
||||
if _, err := i.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
|
||||
}); err != nil {
|
||||
return repaired, err
|
||||
}
|
||||
repaired++
|
||||
}
|
||||
}
|
||||
return repaired, nil
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the
|
||||
// store but for the durations a save measures, taken here by nothing.
|
||||
func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) {
|
||||
t.Helper()
|
||||
c := chain.InMemory(nil)
|
||||
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv.UseChain(c)
|
||||
t.Cleanup(inv.stopViews)
|
||||
return inv, c
|
||||
}
|
||||
|
||||
var plansAt = time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
|
||||
func aBatch(id string, at time.Time) *Plan {
|
||||
return &Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c-" + id, Created: at, State: PlanAssembling,
|
||||
Tiers: [][]string{}, Modules: map[string]*PlanModule{}}
|
||||
}
|
||||
|
||||
func aMerge(commit, plan string) BatchedMerge {
|
||||
return BatchedMerge{Repository: "Novox/Mesh-Catalog", Branch: "main", Commit: commit, Merged: plansAt, Heard: plansAt.Add(time.Second),
|
||||
Event: []byte(`{"commit":"` + commit + `"}`), Plan: plan}
|
||||
}
|
||||
|
||||
// **A plan is kept under its id; a save of a copy another save has moved since is refused and writes nothing** (ADR 0408
|
||||
// §2, issue 1092): the plan's own Revision counts its saves, 1 for the first.
|
||||
func TestAPlanIsKeptUnderItsIdAndASaveOfAStaleCopyIsRefused(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := context.Background()
|
||||
p := aBatch("plan-1", plansAt)
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Revision != 1 {
|
||||
t.Fatalf("a new plan is at revision %d", p.Revision)
|
||||
}
|
||||
stale := *p
|
||||
p.Note = "moved on"
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil || p.Revision != 2 {
|
||||
t.Fatalf("the second save: revision %d, %v", p.Revision, err)
|
||||
}
|
||||
stale.Note = "laid over"
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{&stale}, nil); !errors.Is(err, ErrPlanMoved) {
|
||||
t.Fatalf("a stale copy was saved: %v", err)
|
||||
}
|
||||
got, err := inv.chainPlanByID(ctx, c, "plan-1")
|
||||
if err != nil || got.Note != "moved on" || got.Revision != 2 {
|
||||
t.Fatalf("the plan is %+v: %v", got, err)
|
||||
}
|
||||
// A plan new to the bucket that is already there is refused too.
|
||||
again := aBatch("plan-1", plansAt)
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{again}, nil); !errors.Is(err, ErrPlanMoved) {
|
||||
t.Fatalf("a new plan over a kept one: %v", err)
|
||||
}
|
||||
if _, err := inv.chainPlanByID(ctx, c, "plan-nobody"); err == nil || err.Error() != "no plan plan-nobody" {
|
||||
t.Fatalf("a plan nobody kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A save keeps the merges the record holds, not the ones its copy read** (issue 1092): a merge that moved into the record
|
||||
// after the copy was read is not laid back by the copy's save.
|
||||
func TestASaveKeepsTheMergesTheRecordHoldsAndNotTheCopys(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := context.Background()
|
||||
batch := aBatch("plan-1", plansAt)
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || !added {
|
||||
t.Fatalf("added %v: %v", added, err)
|
||||
}
|
||||
// The copy was read before the merge came in; saving it neither fails nor loses the merge.
|
||||
batch.Note = "window closed"
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil {
|
||||
t.Fatalf("a copy read before a merge came in: %v", err)
|
||||
}
|
||||
merges, err := inv.chainMergesOf(ctx, c, "plan-1")
|
||||
if err != nil || len(merges) != 1 || merges[0].Commit != "m1" || merges[0].Repository != "novox/mesh-catalog" {
|
||||
t.Fatalf("the merges of the plan: %+v %v", merges, err)
|
||||
}
|
||||
if string(merges[0].Event) != `{"commit":"m1"}` || merges[0].Heard.IsZero() || merges[0].Plan != "plan-1" {
|
||||
t.Fatalf("the merge came back as %+v", merges[0])
|
||||
}
|
||||
// A merge heard is not heard twice, whatever the spelling of its repository.
|
||||
if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || added {
|
||||
t.Fatalf("added again %v: %v", added, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A merge added to a plan that is not saved is written by that plan's save, and by nothing else** (issue 1092): until
|
||||
// then it is in no record — so the event that brought it is handed over again if the save fails.
|
||||
func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := context.Background()
|
||||
if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-new")); err != nil || !added {
|
||||
t.Fatalf("added %v: %v", added, err)
|
||||
}
|
||||
if _, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); known {
|
||||
t.Fatal("a merge for a plan not saved is in a record")
|
||||
}
|
||||
// The save is refused: nothing is written, the merge is still carried.
|
||||
held := false
|
||||
away := chain.New(chain.NewMemory(), chain.NewMemory(), chain.NewMemory(), func(context.Context) error {
|
||||
if held {
|
||||
return nil
|
||||
}
|
||||
return errors.New("the lease is not held")
|
||||
})
|
||||
other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
|
||||
other.UseChain(away)
|
||||
t.Cleanup(other.stopViews)
|
||||
other.moved.pending = inv.moved.pending
|
||||
p := aBatch("plan-new", plansAt)
|
||||
if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) {
|
||||
t.Fatalf("the save answered %v", err)
|
||||
}
|
||||
if _, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); known {
|
||||
t.Fatal("a refused save left the merge in a record")
|
||||
}
|
||||
held = true
|
||||
if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); !known || m.Plan != "plan-new" {
|
||||
t.Fatalf("the save did not write the merge: %+v %v", m, known)
|
||||
}
|
||||
}
|
||||
|
||||
// failingOn is a bucket that refuses to write one key: the second half of a move, interrupted.
|
||||
type failingOn struct {
|
||||
chain.Backend
|
||||
key string
|
||||
fail *atomic.Bool
|
||||
}
|
||||
|
||||
func (f failingOn) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if key == f.key && f.fail.Load() {
|
||||
return 0, errors.New("the bus went away")
|
||||
}
|
||||
return f.Backend.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
// **A merge moved to a plan is written to the receiving record first, then taken from the giving one; one found in both is
|
||||
// read from the one it entered last and repaired** (issue 1092, ADR 0299 §7).
|
||||
func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *testing.T) {
|
||||
c := chain.InMemory(nil)
|
||||
var fail atomic.Bool
|
||||
rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail}
|
||||
c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })
|
||||
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv.UseChain(c)
|
||||
t.Cleanup(inv.stopViews)
|
||||
ctx := context.Background()
|
||||
a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute))
|
||||
for _, p := range []*Plan{a, b} {
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The move stops before the giving record is written: the merge is in both.
|
||||
fail.Store(true)
|
||||
err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b"}})
|
||||
if err == nil {
|
||||
t.Fatal("the move was said to be made with the giving record unwritten")
|
||||
}
|
||||
fail.Store(false)
|
||||
if m, _, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); m.Plan != "plan-b" {
|
||||
t.Fatalf("a merge in two records is read from %q, not the one it moved to", m.Plan)
|
||||
}
|
||||
if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 {
|
||||
t.Fatalf("the giving plan still answers %+v", got)
|
||||
}
|
||||
// The next cut repairs it.
|
||||
v, _ := inv.rolloutView(ctx, c)
|
||||
if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 {
|
||||
t.Fatalf("the merge is held by %d records, want the two of a stopped move", n)
|
||||
}
|
||||
inv.moved.mu.Lock()
|
||||
inv.moved.kind = map[string]bool{KindRollouts: true}
|
||||
inv.moved.mu.Unlock()
|
||||
repaired := 0
|
||||
for k, copies := range mergeHolders(v.list()) {
|
||||
if len(copies) < 2 {
|
||||
continue
|
||||
}
|
||||
w := winner(copies)
|
||||
for _, h := range copies {
|
||||
if h.plan != w.plan {
|
||||
key := k
|
||||
if _, err := inv.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repaired++
|
||||
}
|
||||
}
|
||||
}
|
||||
if repaired != 1 {
|
||||
t.Fatalf("repaired %d", repaired)
|
||||
}
|
||||
waitFor(t, func() bool {
|
||||
v, _ := inv.rolloutView(ctx, c)
|
||||
return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1
|
||||
})
|
||||
}
|
||||
|
||||
// waitFor polls a condition that a watch makes true, for a moment.
|
||||
func waitFor(t *testing.T, ok func() bool) {
|
||||
t.Helper()
|
||||
for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("the condition did not come true")
|
||||
}
|
||||
|
||||
// **A merge no plan answers stays, flagged, in the record that held it last, and is walked alone** (issue 1092): not in
|
||||
// the merges of the plan, listed among the merges waiting, and answered again by the plan it is moved to.
|
||||
func TestAWaitingMergeStaysInTheRecordThatHeldItLast(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := context.Background()
|
||||
a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute))
|
||||
for _, p := range []*Plan{a, b} {
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "", Alone: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 {
|
||||
t.Fatalf("a waiting merge is answered by %+v", got)
|
||||
}
|
||||
alone, err := inv.chainAloneMerges(ctx, c)
|
||||
if err != nil || len(alone) != 1 || alone[0].Commit != "m1" || alone[0].Plan != "" || !alone[0].Alone {
|
||||
t.Fatalf("waiting: %+v %v", alone, err)
|
||||
}
|
||||
if m, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); !known || m.Plan != "" {
|
||||
t.Fatalf("a waiting merge reads as %+v %v", m, known)
|
||||
}
|
||||
if later, _ := inv.chainLaterMergesOf(ctx, c, "novox/mesh-catalog", "main", plansAt.Add(-time.Hour)); len(later) != 0 {
|
||||
t.Fatalf("a waiting merge is among those a plan answers: %+v", later)
|
||||
}
|
||||
if kept, first, _ := inv.chainKeptOnBranch(ctx, c, "novox/mesh-catalog", "main"); !kept || first.IsZero() {
|
||||
t.Fatalf("a waiting merge is not kept on its branch: %v %v", kept, first)
|
||||
}
|
||||
// Walked alone: the plan it goes to answers it, and the record that held it lets it go.
|
||||
if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b", Alone: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 || !got[0].Alone {
|
||||
t.Fatalf("plan-b answers %+v", got)
|
||||
}
|
||||
if left, _ := inv.chainAloneMerges(ctx, c); len(left) != 0 {
|
||||
t.Fatalf("still waiting: %+v", left)
|
||||
}
|
||||
waitFor(t, func() bool {
|
||||
v, _ := inv.rolloutView(ctx, c)
|
||||
return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1
|
||||
})
|
||||
// A move to a plan with no record is refused and takes nothing from the one that holds the merge.
|
||||
err = inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-nowhere"}})
|
||||
if !errors.Is(err, ErrMergeUnsaved) {
|
||||
t.Fatalf("a move to a plan with no record: %v", err)
|
||||
}
|
||||
if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 {
|
||||
t.Fatalf("the merge was taken from plan-b: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// countingBackend counts the reads a bucket answers.
|
||||
type countingBackend struct {
|
||||
chain.Backend
|
||||
gets, keys atomic.Int32
|
||||
}
|
||||
|
||||
func (c *countingBackend) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
|
||||
c.gets.Add(1)
|
||||
return c.Backend.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (c *countingBackend) Keys(ctx context.Context) ([]string, error) {
|
||||
c.keys.Add(1)
|
||||
return c.Backend.Keys(ctx)
|
||||
}
|
||||
|
||||
// **The surveys are read from a view kept by one watch, never from the bucket key by key** (issue 1092): the plans and the
|
||||
// merges are asked for many times and the bucket is read for none of them; a write by another process reaches the view
|
||||
// through its watch.
|
||||
func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
|
||||
counting := &countingBackend{Backend: chain.NewMemory()}
|
||||
c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil })
|
||||
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv.UseChain(c)
|
||||
t.Cleanup(inv.stopViews)
|
||||
ctx := context.Background()
|
||||
for k, id := range []string{"plan-1", "plan-2", "plan-3"} {
|
||||
p := aBatch(id, plansAt.Add(time.Duration(k)*time.Minute))
|
||||
if k > 0 {
|
||||
p.State = PlanBuilding
|
||||
}
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before := counting.gets.Load()
|
||||
for n := 0; n < 20; n++ {
|
||||
open, err := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
|
||||
if err != nil || len(open) != 2 || open[0].ID != "plan-2" {
|
||||
t.Fatalf("open plans %+v: %v", open, err)
|
||||
}
|
||||
if _, err := inv.chainAloneMerges(ctx, c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if counting.gets.Load() != before || counting.keys.Load() != 0 {
|
||||
t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load())
|
||||
}
|
||||
// Another process writes; the view follows.
|
||||
other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
|
||||
other.UseChain(c)
|
||||
t.Cleanup(other.stopViews)
|
||||
late := aBatch("plan-4", plansAt.Add(time.Hour))
|
||||
late.State = PlanRolling
|
||||
if err := other.chainSavePlans(ctx, c, []*Plan{late}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, func() bool {
|
||||
open, _ := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
|
||||
return len(open) == 3
|
||||
})
|
||||
}
|
||||
@@ -159,44 +159,23 @@ func (i *Inventory) KeepSet(ctx context.Context) (map[string][]string, error) {
|
||||
// no releases left to return between, and its build rows stay only as history. Without the
|
||||
// join every module ever built kept five builds' artifacts for ever — and once the store's
|
||||
// collector runs for real, what the keep set says is what the disk holds.
|
||||
recent, err := i.store.Pool().Query(ctx,
|
||||
`select made, built_against from (
|
||||
select b.made, b.built_against,
|
||||
row_number() over (partition by b.module order by b.at desc, b.id desc) as back
|
||||
from build b
|
||||
join module m on m.name = b.module
|
||||
where b.failed = '' and b.module is not null and b.module <> ''
|
||||
) ranked where back <= $1`, KeptBuilds)
|
||||
recent, err := i.recentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer recent.Close()
|
||||
var stoodOn []string
|
||||
for recent.Next() {
|
||||
var raw, against []byte
|
||||
if err := recent.Scan(&raw, &against); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var made []Artifact
|
||||
if err := json.Unmarshal(raw, &made); err == nil {
|
||||
for _, a := range made {
|
||||
if reference := asRecorded(a.Reference); reference != "" {
|
||||
because(reference, KeptByRecentBuild)
|
||||
}
|
||||
for _, r := range recent {
|
||||
for _, a := range r.made {
|
||||
if reference := asRecorded(a.Reference); reference != "" {
|
||||
because(reference, KeptByRecentBuild)
|
||||
}
|
||||
}
|
||||
var bases []string
|
||||
if err := json.Unmarshal(against, &bases); err == nil {
|
||||
for _, b := range bases {
|
||||
if reference := asRecorded(b); reference != "" {
|
||||
stoodOn = append(stoodOn, reference)
|
||||
}
|
||||
for _, b := range r.against {
|
||||
if reference := asRecorded(b); reference != "" {
|
||||
stoodOn = append(stoodOn, reference)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := recent.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// **A copied base is kept by the kept builds that stood on it, and by nothing else** (novox/hq
|
||||
// ADR 0257). It has no builds of its own to count five of, and no machine is ever handed a base,
|
||||
@@ -340,27 +319,97 @@ func (i *Inventory) mirroredSet(ctx context.Context) (map[string]bool, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// everyReferenceMade is every artifact reference any successful build recorded, oldest build first.
|
||||
func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
|
||||
// recentBuild is what one of the KeptBuilds newest successful builds of a module made and stood on.
|
||||
type recentBuild struct {
|
||||
made []Artifact
|
||||
against []string
|
||||
}
|
||||
|
||||
// recentBuilds is, of each module the mesh still holds, its KeptBuilds newest successful builds' artifacts and bases.
|
||||
func (i *Inventory) recentBuilds(ctx context.Context) ([]recentBuild, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
builds, err := i.chainRecentMade(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]recentBuild, 0, len(builds))
|
||||
for _, b := range builds {
|
||||
out = append(out, recentBuild{made: b.Made, against: b.Against})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select made from build where failed = '' and module is not null and module <> ''
|
||||
order by at asc, id asc`)
|
||||
`select made, built_against from (
|
||||
select b.made, b.built_against,
|
||||
row_number() over (partition by b.module order by b.at desc, b.id desc) as back
|
||||
from build b
|
||||
join module m on m.name = b.module
|
||||
where b.failed = '' and b.module is not null and b.module <> ''
|
||||
) ranked where back <= $1`, KeptBuilds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
var out []recentBuild
|
||||
for rows.Next() {
|
||||
var raw []byte
|
||||
if err := rows.Scan(&raw); err != nil {
|
||||
var raw, against []byte
|
||||
if err := rows.Scan(&raw, &against); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var made []Artifact
|
||||
if err := json.Unmarshal(raw, &made); err != nil {
|
||||
continue
|
||||
var r recentBuild
|
||||
if err := json.Unmarshal(raw, &r.made); err != nil {
|
||||
r.made = nil
|
||||
}
|
||||
for _, a := range made {
|
||||
if err := json.Unmarshal(against, &r.against); err != nil {
|
||||
r.against = nil
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// everyReferenceMade is every artifact reference any successful build recorded, oldest build first.
|
||||
func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
|
||||
var made [][]Artifact
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
builds, err := i.chainEveryReferenceMade(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, b := range builds {
|
||||
made = append(made, b.Made)
|
||||
}
|
||||
} else {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select made from build where failed = '' and module is not null and module <> ''
|
||||
order by at asc, id asc`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var raw []byte
|
||||
if err := rows.Scan(&raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a []Artifact
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
continue
|
||||
}
|
||||
made = append(made, a)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, row := range made {
|
||||
for _, a := range row {
|
||||
reference := asRecorded(a.Reference)
|
||||
if reference == "" || seen[reference] {
|
||||
continue
|
||||
@@ -369,7 +418,7 @@ func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
|
||||
out = append(out, reference)
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// asRecorded is an artifact reference in the one vocabulary the sweep speaks (novox/hq issue 226).
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/store"
|
||||
)
|
||||
|
||||
@@ -64,6 +65,16 @@ func ForTest(t *testing.T) *Inventory {
|
||||
if _, err := inv.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The chain is on the bus in a mesh; in a test it is in memory, with every kind moved, as a mesh raised after this
|
||||
// release has them (ADR 0408, issue 1089). The tests of the move itself use ForTestBefore.
|
||||
c := chain.InMemory(nil)
|
||||
inv.UseChain(c)
|
||||
testChains.Store(os.Getenv(store.Variable(Name)), c)
|
||||
t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) })
|
||||
if _, err := inv.store.Pool().Exec(t.Context(),
|
||||
`insert into chain_moved (counts, largest) values ($1, $1)`, `{"builds": 0, "rollouts": 0, "assignments": 0}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return inv
|
||||
}
|
||||
|
||||
@@ -121,6 +132,11 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) {
|
||||
if err := inv.Ready(t.Context(), 20*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The chain is in memory and nothing is moved: the tests of a move seed the tables and move them.
|
||||
c := chain.InMemory(nil)
|
||||
inv.UseChain(c)
|
||||
testChains.Store(os.Getenv(store.Variable(Name)), c)
|
||||
t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) })
|
||||
return inv, func() {
|
||||
t.Helper()
|
||||
if _, err := inv.store.Migrate(t.Context(), all); err != nil {
|
||||
|
||||
+59
-12
@@ -187,8 +187,8 @@ func (i *Inventory) PreviousBuild(ctx context.Context, module, commit string, fa
|
||||
} else if bad {
|
||||
continue
|
||||
}
|
||||
var manifest []byte
|
||||
if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, b.ID).Scan(&manifest); err != nil {
|
||||
manifest, err := i.manifestOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return Build{}, false, err
|
||||
}
|
||||
if len(manifest) == 0 || string(manifest) == "null" {
|
||||
@@ -241,6 +241,11 @@ func (i *Inventory) RestoreModule(ctx context.Context, b Build) error {
|
||||
// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told
|
||||
// from one that does (the bus's planned step, ADR 0236).
|
||||
func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainBuildFingerprints(ctx, c, module)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select commit_hash, made, coalesce(manifest::text, '') from build
|
||||
where module = $1 and failed = '' and commit_hash <> '' order by at desc limit 50`, module)
|
||||
@@ -266,6 +271,11 @@ func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[s
|
||||
|
||||
// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint.
|
||||
func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainFingerprints(ctx, c)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select module, commit_hash, made, coalesce(manifest::text, '') from build
|
||||
where module is not null and failed = '' and commit_hash <> '' order by at desc`)
|
||||
@@ -316,6 +326,11 @@ func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bo
|
||||
// BuildOf is the id of the newest successful build of a module from a commit; empty when none is
|
||||
// recorded (a manifest handed over by hand).
|
||||
func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return "", err
|
||||
} else if on {
|
||||
return i.chainBuildOf(ctx, c, module, commit)
|
||||
}
|
||||
var id string
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id from build where module = $1 and commit_hash = $2 and failed = '' order by at desc limit 1`,
|
||||
@@ -335,19 +350,13 @@ func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (cata
|
||||
if commit == "" {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
var id string
|
||||
var raw []byte
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, manifest from build
|
||||
where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null'
|
||||
and (commit_hash = $2 or starts_with(commit_hash, $2))
|
||||
order by at desc limit 1`, module, commit).Scan(&id, &raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
id, raw, found, err := i.newestManifestAt(ctx, module, commit)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false, err
|
||||
}
|
||||
if !found {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
if stands, same, err := i.StandingBuild(ctx, module, id); err != nil {
|
||||
return catalogue.Manifest{}, false, err
|
||||
} else if stands != "" && stands != id && len(same) > 0 {
|
||||
@@ -360,3 +369,41 @@ func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (cata
|
||||
}
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
// manifestOf is the manifest a build was recorded with; nil when it kept none.
|
||||
func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainManifestOf(ctx, c, id)
|
||||
}
|
||||
var manifest []byte
|
||||
if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, id).Scan(&manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
// newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest:
|
||||
// its id and manifest; false when there is none.
|
||||
func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return "", nil, false, err
|
||||
} else if on {
|
||||
return i.chainManifestBuild(ctx, c, module, commit)
|
||||
}
|
||||
var id string
|
||||
var raw []byte
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, manifest from build
|
||||
where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null'
|
||||
and (commit_hash = $2 or starts_with(commit_hash, $2))
|
||||
order by at desc limit 1`, module, commit).Scan(&id, &raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, false, err
|
||||
}
|
||||
return id, raw, true, nil
|
||||
}
|
||||
|
||||
@@ -93,11 +93,28 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s
|
||||
if declared == "" {
|
||||
return nil, nil
|
||||
}
|
||||
// Only a module assigned there: from the bucket when the assignments are on the bus, else the store's join.
|
||||
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
assignedHere := map[string]bool{}
|
||||
join := `join assignment a on a.node = s.node and a.module = s.module`
|
||||
if onTheBus {
|
||||
join = ``
|
||||
modules, err := i.assignedModules(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, m := range modules {
|
||||
assignedHere[m] = true
|
||||
}
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select s.node, s.module, s.name, s.replace_after_start
|
||||
from module_secret s
|
||||
join node n on n.id = s.node
|
||||
join assignment a on a.node = s.node and a.module = s.module
|
||||
`+join+`
|
||||
where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start
|
||||
and s.origin = 'accepted' and s.replace_after_start is not null
|
||||
and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module))
|
||||
@@ -117,6 +134,9 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
if onTheBus && !assignedHere[d.module] {
|
||||
continue
|
||||
}
|
||||
dues = append(dues, d)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
@@ -7,8 +7,10 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// A module with one secret of each kind the rule tells apart (novox/hq ADR 0228), assigned to the
|
||||
@@ -36,8 +38,9 @@ func assign(t *testing.T, inv *Inventory, ctx context.Context, node, module stri
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.store.Pool().Exec(ctx,
|
||||
`insert into assignment (node, module) values ($1, $2) on conflict do nothing`, n.ID, module); err != nil {
|
||||
// Written as the fixture it is, past the checks Assign makes: the assignment on the bus.
|
||||
if _, err := inv.chain.Assignments.Create(ctx, assignmentKey(n.Name, module),
|
||||
chain.Assignment{Node: n.Name, Module: module, Assigned: time.Now().UTC()}); err != nil && !errors.Is(err, chain.ErrExists) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
-- Builds, rollouts and assignments move to the bus (novox/hq ADR 0408 §8, design 47 milestone 5, issue 1089).
|
||||
--
|
||||
-- This migration makes the switch alone. The rows of build, build_request, release_plan, batched_merge, assignment and
|
||||
-- pending_assignment are written into the chain's buckets by the serving controller at its start, holding the lease,
|
||||
-- because a migration runs when the store is opened, before any connection to the bus. This row is the switch: while it
|
||||
-- is absent nothing reads the buckets; once the move has counted every row against the keys it wrote, it is written, and
|
||||
-- nothing reads the tables. One row at most; never written by a migration, so a delivery alone changes nothing. The
|
||||
-- controller keeps no consolidated schema, so this migration is the whole of the change; the tables go by a later one.
|
||||
create table chain_moved (
|
||||
only_one boolean primary key default true check (only_one),
|
||||
moved_at timestamptz not null default now(),
|
||||
-- The counts the move read from the tables and wrote as keys, per kind, and the largest value it wrote: what a
|
||||
-- reader of the move reads back.
|
||||
counts jsonb not null,
|
||||
largest jsonb not null
|
||||
);
|
||||
|
||||
-- A seat's holder was an assignment by a foreign key; the assignments are keys of the bus once moved, so the controller says
|
||||
-- it (HoldSeat refuses a holder that is not assigned) and an unassignment takes the holding with it (Unassign), as the
|
||||
-- key's `on delete cascade` did.
|
||||
alter table seat_holding drop constraint seat_holding_node_module_fkey;
|
||||
@@ -151,7 +151,7 @@ func TestAPersonRecordsACopyAndTheFirstRecordStands(t *testing.T) {
|
||||
// The migration records the copies the build records already name: only a module's own on-<argument>
|
||||
// repository, and only once, by the earliest build naming it.
|
||||
func TestTheCopiesTheRecordsAlreadyNameAreRecorded(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
inv := inTheTables(t)
|
||||
ctx := context.Background()
|
||||
copyRef := mirrorRef("route-proxy/on-go_base", 1)
|
||||
notACopy := ref("go-base", "build", 2)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
@@ -69,6 +70,21 @@ type DeclaredMirrored struct {
|
||||
var DeclaredMirroredFields = []string{"kind", "default", "why", "type", "description", "enum", "minimum", "maximum",
|
||||
"pattern", "trusted", "predefined", "required", "runtime"}
|
||||
|
||||
// ErrAssignmentsMoved is a write of the store's mirror of the assignments, refused because the assignments are keys of the bus.
|
||||
var ErrAssignmentsMoved = errors.New("the assignments are keys of the bus, and the store's rows are no longer what they say")
|
||||
|
||||
// AssignmentsMirrorToWrite is AssignmentsMirror for the mirror's writer, which lays what it returns over the bucket and
|
||||
// ends the keys it does not return: **refused once the assignments are moved** (ADR 0408 §4), asked at the moment of the
|
||||
// write, so a tick that decided before the switch cannot lay mirror-shaped values over the links after it.
|
||||
func (i *Inventory) AssignmentsMirrorToWrite(ctx context.Context) (map[string][]byte, error) {
|
||||
if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return nil, ErrAssignmentsMoved
|
||||
}
|
||||
return i.AssignmentsMirror(ctx)
|
||||
}
|
||||
|
||||
// AssignmentsMirror is every assignment's mirrored value, keyed `<node>.<module>`, with the provisioner's plan of
|
||||
// each assignment of a module handed to it, and a value of its own for each module handed to it on a node that
|
||||
// nothing assigned there, only pulled in by another (issue 765).
|
||||
@@ -88,22 +104,15 @@ func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, e
|
||||
}
|
||||
|
||||
func (i *Inventory) assignmentsMirrored(ctx context.Context) ([]AssignmentMirrored, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node`)
|
||||
rows, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []AssignmentMirrored
|
||||
for rows.Next() {
|
||||
var a AssignmentMirrored
|
||||
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.Assigned = a.Assigned.UTC()
|
||||
out = append(out, a)
|
||||
out := make([]AssignmentMirrored, 0, len(rows))
|
||||
for _, a := range rows {
|
||||
out = append(out, AssignmentMirrored{Node: a.Node, Module: a.Module, Assigned: a.Assigned.UTC()})
|
||||
}
|
||||
return out, rows.Err()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DeclaredConfigMirror is every registered module's mirrored config declarations, keyed by the module.
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/store"
|
||||
)
|
||||
|
||||
@@ -32,6 +34,13 @@ type Inventory struct {
|
||||
// configValues reads a module's config values from mesh-config once the move of config has run (ReadsConfigFrom,
|
||||
// issue 664); nil reads none.
|
||||
configValues ConfigValues
|
||||
// chain is the controller's chain on the bus (novox/hq ADR 0408, issue 1089), once attached (UseChain); moved is which
|
||||
// kinds of it have been moved from the tables, as chain_moved says (ChainReads). A kind not moved is read and written
|
||||
// in the tables as before.
|
||||
chain *chain.Chain
|
||||
moved *movedKinds
|
||||
// recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store.
|
||||
recorded func(context.Context, Duration) error
|
||||
}
|
||||
|
||||
// Open connects to the inventory store.
|
||||
@@ -40,10 +49,18 @@ func Open(ctx context.Context) (*Inventory, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Inventory{store: s}, nil
|
||||
inv := &Inventory{store: s, moved: &movedKinds{}}
|
||||
// A test's store has the chain a test made for it in memory, shared by every inventory opened on that store.
|
||||
if c, ok := testChains.Load(os.Getenv(store.Variable(Name))); ok {
|
||||
inv.chain = c.(*chain.Chain)
|
||||
}
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) Close() { i.store.Close() }
|
||||
func (i *Inventory) Close() {
|
||||
i.stopViews()
|
||||
i.store.Close()
|
||||
}
|
||||
|
||||
// Ready waits for the database to answer.
|
||||
func (i *Inventory) Ready(ctx context.Context, within time.Duration) error {
|
||||
@@ -1259,6 +1276,9 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4 where id = $1`,
|
||||
node, digest, carried, sentEpoch)
|
||||
if err == nil {
|
||||
saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now()))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1268,6 +1288,11 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu
|
||||
// read from: every change a push carries happened after the last push, and an assignment is the one
|
||||
// that says when.
|
||||
func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) {
|
||||
if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
|
||||
return time.Time{}, err
|
||||
} else if on {
|
||||
return i.chainAwaitingSince(ctx, name)
|
||||
}
|
||||
var since time.Time
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select coalesce(
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// What the controller asked to be built, and the assignments waiting for it (novox/hq issue 325, ADR 0261).
|
||||
@@ -67,6 +69,11 @@ type RequestOutcome struct {
|
||||
// RecordBuildRequest keeps one build request, once it is asked. Idempotent on the id. Requests older than
|
||||
// KeptFor are deleted in the same act.
|
||||
func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return err
|
||||
} else if on {
|
||||
return i.chainRecordBuildRequest(ctx, c, a)
|
||||
}
|
||||
at := a.At
|
||||
if at.IsZero() {
|
||||
at = time.Now().UTC()
|
||||
@@ -90,6 +97,11 @@ func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) erro
|
||||
|
||||
// BuildRequestByID is the build request kept under this id, and whether one was kept.
|
||||
func (i *Inventory) BuildRequestByID(ctx context.Context, id string) (BuildRequest, bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return BuildRequest{}, false, err
|
||||
} else if on {
|
||||
return i.chainBuildRequestByID(ctx, c, id)
|
||||
}
|
||||
var a BuildRequest
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, repository, seat, source_path, ref, commit_hash, asked_for, at_terminal, asked_at
|
||||
@@ -104,21 +116,40 @@ func (i *Inventory) BuildRequestByID(ctx context.Context, id string) (BuildReque
|
||||
// MarkNotAsked says a kept build request was never handed over: the words are kept, unless its outcome was
|
||||
// heard first.
|
||||
func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error {
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update build_request set not_asked = $2
|
||||
where id = $1 and not exists (select 1 from build where build.id = $1)`, id, why)
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return err
|
||||
} else if on {
|
||||
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why })
|
||||
}
|
||||
if heard, err := i.outcomeHeard(ctx, id); err != nil || heard {
|
||||
return err
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx, `update build_request set not_asked = $2 where id = $1`, id, why)
|
||||
return err
|
||||
}
|
||||
|
||||
// MarkOutcomeUnknown says the asker of a build it handed over stopped waiting for it: asked, outcome unknown.
|
||||
// The build may still run; its outcome, when heard, is the last word.
|
||||
func (i *Inventory) MarkOutcomeUnknown(ctx context.Context, id, why string) error {
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update build_request set outcome_unknown = $2
|
||||
where id = $1 and not exists (select 1 from build where build.id = $1)`, id, why)
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return err
|
||||
} else if on {
|
||||
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why })
|
||||
}
|
||||
if heard, err := i.outcomeHeard(ctx, id); err != nil || heard {
|
||||
return err
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx, `update build_request set outcome_unknown = $2 where id = $1`, id, why)
|
||||
return err
|
||||
}
|
||||
|
||||
// outcomeHeard says a build of this id was recorded in the table: its outcome was taken in.
|
||||
func (i *Inventory) outcomeHeard(ctx context.Context, id string) (bool, error) {
|
||||
var heard bool
|
||||
err := i.store.Pool().QueryRow(ctx, `select exists (select 1 from build where id = $1)`, id).Scan(&heard)
|
||||
return heard, err
|
||||
}
|
||||
|
||||
// RequestsNamed is every build request kept whose directory names the module, or whose build said it built
|
||||
// it, newest first, each with its outcome.
|
||||
func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOutcome, error) {
|
||||
@@ -154,6 +185,11 @@ func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) {
|
||||
}
|
||||
|
||||
func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainRequests(ctx, c)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select a.id, a.repository, a.seat, a.source_path, a.ref, a.commit_hash, a.asked_for,
|
||||
coalesce(a.not_asked, ''), coalesce(a.outcome_unknown, ''), a.asked_at,
|
||||
|
||||
@@ -34,7 +34,7 @@ func completeEarlierPins(t *testing.T, inv *Inventory) {
|
||||
}
|
||||
|
||||
func TestAPinMadeBeforeIsCompletedWhenTheNodeAnswersOnce(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
inv := inTheTables(t)
|
||||
ctx := context.Background()
|
||||
for _, n := range []string{"user", "provider"} {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
@@ -61,7 +61,7 @@ func TestAPinMadeBeforeIsCompletedWhenTheNodeAnswersOnce(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAPinMadeBeforeIsLeftOpenWhenTheNodeAnswersTwice(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
inv := inTheTables(t)
|
||||
ctx := context.Background()
|
||||
for _, n := range []string{"user", "provider"} {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
|
||||
@@ -3,14 +3,17 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **A plan's id is a value, never SQL** (novox/hq issue 561): the id reaches PlanByID from a caller of the
|
||||
// **A plan's id is a value, never a query** (novox/hq issue 561): the id reaches PlanByID from a caller of the
|
||||
// controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no
|
||||
// other plan answered for it.
|
||||
// other plan answered for it. On the bus an id is a key, and one the bus takes no spelling of is refused to be kept.
|
||||
func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
@@ -32,14 +35,11 @@ func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) {
|
||||
t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want)
|
||||
}
|
||||
}
|
||||
// A plan whose id holds a quote is found by it, and by nothing else.
|
||||
// A plan whose id holds a quote cannot be a key of the bus: refused by name, never rewritten into another id.
|
||||
quoted := p
|
||||
quoted.ID, quoted.Revision = "plan-'561'", 0
|
||||
if err := inv.SavePlan(ctx, "ed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := inv.PlanByID(ctx, quoted.ID); err != nil || got.ID != quoted.ID {
|
||||
t.Fatalf("the plan with a quote in its id was not read back: %v %+v", err, got.ID)
|
||||
if err := inv.SavePlan(ctx, "ed); !errors.Is(err, chain.ErrBadKey) {
|
||||
t.Fatalf("a plan with a quote in its id was kept, or refused for another reason: %v", err)
|
||||
}
|
||||
if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") {
|
||||
t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent)
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything
|
||||
@@ -66,6 +68,15 @@ type Plan struct {
|
||||
// per machine and module, what was asked and how it went, and the machines with no backup holder. Nil for a
|
||||
// walk that moves no module with an item protected by a backup, and for a walk kept before it was.
|
||||
Backups *PlanBackups `json:"backups,omitempty"`
|
||||
// Links are what the rollout names of the chain (novox/hq ADR 0408 §1): the builds it carried, the machine steps it has
|
||||
// taken and the assignments it changed, each with the build before and after. Kept in the rollout's record beside
|
||||
// the fields above; empty in the controller's store.
|
||||
chain.Links
|
||||
// Held are the merges this plan or batch holds (novox/hq ADR 0276, ADR 0408): what the store kept as batched merges
|
||||
// with this plan's id, and the merges no plan answers that it held last. **Written by the merge operations only**: a
|
||||
// save of the plan keeps what the record holds, never what this copy read, so a copy read before a merge moved cannot
|
||||
// lay it back (issue 1092).
|
||||
Held []chain.HeldMerge `json:"held,omitempty"`
|
||||
// WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308
|
||||
// decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every
|
||||
// other save.
|
||||
@@ -670,6 +681,11 @@ type MergeMove struct {
|
||||
// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once
|
||||
// the transaction is committed, in order.
|
||||
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) (err error) {
|
||||
if c, on, cerr := i.ChainReads(ctx, KindRollouts); cerr != nil {
|
||||
return cerr
|
||||
} else if on {
|
||||
return i.chainSavePlans(ctx, c, plans, moves)
|
||||
}
|
||||
epoch, err := i.actingEpoch(ctx)
|
||||
if err != nil {
|
||||
if len(plans) > 0 {
|
||||
@@ -818,16 +834,35 @@ func mergedAt(t time.Time) *time.Time {
|
||||
|
||||
// OpenPlans is every plan still being worked, oldest first.
|
||||
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
|
||||
}
|
||||
return i.plans(ctx, `where state in ('building', 'rolling') order by created`)
|
||||
}
|
||||
|
||||
// PlansSince is every plan made after a moment, and every plan still being worked, oldest first.
|
||||
func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated)
|
||||
}
|
||||
return i.plans(ctx, `where created > $1 or state in ('building', 'rolling') order by created`, since)
|
||||
}
|
||||
|
||||
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
|
||||
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest)
|
||||
if err == nil && limit >= 0 && len(plans) > limit {
|
||||
plans = plans[:limit]
|
||||
}
|
||||
return plans, err
|
||||
}
|
||||
return i.plans(ctx, `order by created desc limit $1`, limit)
|
||||
}
|
||||
|
||||
@@ -835,6 +870,11 @@ func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error)
|
||||
// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it
|
||||
// recorded when its merge was made.
|
||||
func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return Plan{}, false, err
|
||||
} else if on {
|
||||
return i.chainNewestMergeOf(ctx, c, repository, branch)
|
||||
}
|
||||
plans, err := i.plans(ctx, `where lower(repository) = lower($1) and branch = $2 and merged_at is not null
|
||||
and release is null order by merged_at desc, created desc limit 1`, repository, branch)
|
||||
if err != nil || len(plans) == 0 {
|
||||
@@ -846,6 +886,11 @@ func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string
|
||||
// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a
|
||||
// caller of the plans verb.
|
||||
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return Plan{}, err
|
||||
} else if on {
|
||||
return i.chainPlanByID(ctx, c, id)
|
||||
}
|
||||
plans, err := i.plans(ctx, `where id = $1`, id)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// Unreachable says whether an error means the store could not be asked right now, rather than
|
||||
@@ -29,6 +34,13 @@ func Unreachable(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
// The chain on the bus (novox/hq ADR 0408): a bus that does not answer, a lease this controller no longer holds, a key
|
||||
// somebody else wrote first — each is "not now", as the store being away is.
|
||||
if errors.Is(err, chain.ErrNotHeld) || errors.Is(err, chain.ErrMoved) || errors.Is(err, context.DeadlineExceeded) ||
|
||||
errors.Is(err, nats.ErrTimeout) || errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrConnectionClosed) ||
|
||||
errors.Is(err, nats.ErrDisconnected) || errors.Is(err, nats.ErrNoServers) || errors.Is(err, jetstream.ErrNoStreamResponse) {
|
||||
return true
|
||||
}
|
||||
var pg *pgconn.PgError
|
||||
if errors.As(err, &pg) {
|
||||
switch pg.Code {
|
||||
|
||||
@@ -238,6 +238,20 @@ func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// mustBeAssigned refuses to record a holder that is not an assignment: a seat cannot be handed to something that is not
|
||||
// running anywhere. The store's foreign key said it while the assignments were rows of the store; they are keys of the bus
|
||||
// now, so the controller says it.
|
||||
func (i *Inventory) mustBeAssigned(ctx context.Context, seat, nodeName, module string) error {
|
||||
assigned, err := i.isAssigned(ctx, nodeName, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !assigned {
|
||||
return fmt.Errorf("recording %s on %s as the holder of %s: %s is not assigned to %s", module, nodeName, seat, module, nodeName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HoldSeat records that one assignment holds a seat, replacing whoever held it — every holder, for a
|
||||
// replicated seat (novox/hq ADR 0223) — as one transaction, so the seat is never without a holder in
|
||||
// between (novox/hq ADR 0131, design 28 task 5.3). The assignment must exist; the store refuses
|
||||
@@ -248,6 +262,9 @@ func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := i.mustBeAssigned(ctx, seat, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -276,6 +293,9 @@ func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, mo
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := i.mustBeAssigned(ctx, seat, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := i.store.Pool().Exec(ctx,
|
||||
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
|
||||
on conflict (seat, node, module) do nothing`,
|
||||
|
||||
@@ -174,6 +174,8 @@ func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, bui
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// And what it put on the machine, in the assignments of the modules it carried a build of (issue 1095).
|
||||
saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now()))
|
||||
if recordErr != nil {
|
||||
return &SendNotRecordedError{Err: recordErr}
|
||||
}
|
||||
|
||||
@@ -22,9 +22,8 @@ func generationNow(t *testing.T, inv *Inventory) int64 {
|
||||
return g
|
||||
}
|
||||
|
||||
// **Every change to what is assigned where raises the generation, in its own transaction** — an
|
||||
// assignment, an unassignment, and one taken by the machine's removal, which no writer in Go makes — and
|
||||
// a repeated assignment, which changes nothing, does not.
|
||||
// **Every change to what is assigned where raises the generation** — an assignment and an unassignment — and a repeated
|
||||
// assignment, which changes nothing, does not.
|
||||
func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres", "web")
|
||||
ctx := t.Context()
|
||||
@@ -52,16 +51,8 @@ func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) {
|
||||
if unassigned <= assigned {
|
||||
t.Fatalf("an unassignment left the generation at %d", unassigned)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := generationNow(t, inv)
|
||||
if _, err := inv.RemoveNodeForTest(ctx, node); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed := generationNow(t, inv); removed <= before {
|
||||
t.Errorf("a machine's removal took its assignments and left the generation at %d", removed)
|
||||
}
|
||||
// A machine's removal, which no verb makes, is read from the machines the store knows: an assignment on a machine it
|
||||
// does not know is none (assignments in the bucket have no key of the store to cascade through).
|
||||
}
|
||||
|
||||
// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter
|
||||
@@ -175,22 +166,6 @@ func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **An update that leaves an assignment as it was raises nothing** (the trigger's WHEN).
|
||||
func TestAnUpdateThatChangesNoAssignmentRaisesNothing(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web")
|
||||
ctx := t.Context()
|
||||
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := generationNow(t, inv)
|
||||
if _, err := inv.store.Pool().Exec(ctx, `update assignment set module = module`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after := generationNow(t, inv); after != before {
|
||||
t.Errorf("an update that changed nothing moved the generation from %d to %d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer.
|
||||
func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
|
||||
@@ -32,6 +32,11 @@ type sourceRow struct {
|
||||
// sourceRows is every successful build of each module (of one module, when named), newest request
|
||||
// first.
|
||||
func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][]sourceRow, []string, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, nil, err
|
||||
} else if on {
|
||||
return i.chainSourceRows(ctx, c, module)
|
||||
}
|
||||
query := `select b.module, b.id, b.commit_hash, b.source_fingerprint, b.made, b.manifest,
|
||||
coalesce(g.verdict = 'failed', false)
|
||||
from build b left join build_gate g on g.build = b.id
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// heldControl is a merge announcement that records how it was settled.
|
||||
type heldControl struct {
|
||||
body []byte
|
||||
took, held, drops int
|
||||
}
|
||||
|
||||
func (c *heldControl) Kind() string { return "source-moved" }
|
||||
func (c *heldControl) Body() []byte { return c.body }
|
||||
func (c *heldControl) Subject() string { return "" }
|
||||
func (c *heldControl) Redelivered() bool { return false }
|
||||
func (c *heldControl) HeldFor() time.Duration { return 0 }
|
||||
func (c *heldControl) Answer(context.Context, []byte) error { return nil }
|
||||
func (c *heldControl) Took() error { c.took++; return nil }
|
||||
func (c *heldControl) Hold(time.Duration) error { c.held++; return nil }
|
||||
func (c *heldControl) Drop() error { c.drops++; return nil }
|
||||
|
||||
type sourceMover struct {
|
||||
Upgrader
|
||||
answer error
|
||||
}
|
||||
|
||||
func (s sourceMover) SourceMoved(context.Context, SourceMoved) error { return s.answer }
|
||||
|
||||
// **A merge is acknowledged only after the controller recorded it** (novox/hq ADR 0407 decision 3, issue 1092): an
|
||||
// answer of "not now" — the plan it belongs to could not be saved, the bus away, the lease lost — leaves the announcement
|
||||
// unacknowledged, held to be handed over again; only an answer that is not "not now" settles it.
|
||||
func TestAMergeTheControllerCouldNotRecordIsHeldAndNotAcknowledged(t *testing.T) {
|
||||
body, err := json.Marshal(SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := &Server{log: log.New(io.Discard, "", 0)}
|
||||
for name, tc := range map[string]struct {
|
||||
answer error
|
||||
took, held int
|
||||
}{
|
||||
"recorded": {nil, 1, 0},
|
||||
"not now": {errors.Join(ErrTryAgain, errors.New("the plan is not saved")), 0, 1},
|
||||
"an answer, not a wait": {errors.New("the mesh could not act on it"), 1, 0},
|
||||
} {
|
||||
s.upgrader = sourceMover{answer: tc.answer}
|
||||
m := &heldControl{body: body}
|
||||
s.sourceMoved(context.Background(), m)
|
||||
if m.took != tc.took || m.held != tc.held || m.drops != 0 {
|
||||
t.Errorf("%s: acknowledged %d times and held %d times, want %d and %d", name, m.took, m.held, tc.took, tc.held)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -325,6 +325,10 @@ func validKey(key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChangedFields names the top-level fields whose values differ between two encoded objects, sorted; nil when either is
|
||||
// not an object: what an event of a changed key says of it.
|
||||
func ChangedFields(before, after []byte) []string { return changedFields(before, after) }
|
||||
|
||||
// changedFields names the top-level fields whose values differ between two encoded objects, sorted; nil
|
||||
// when either is not an object.
|
||||
func changedFields(before, after []byte) []string {
|
||||
|
||||
@@ -20,6 +20,9 @@ import (
|
||||
// (internal/catalogue/state.go), and each mirror's own writer (this package and the loop that runs it). The
|
||||
// audit of a mirror against its store (ADR 0289 §4, issue 419) joins this list when it is built.
|
||||
var mayNameAMirror = []string{
|
||||
// The chain opens its buckets, the assignments' among them, which stopped being a mirror when the chain took them
|
||||
// (ADR 0408 §3, issue 1089).
|
||||
"internal/chain/bus.go",
|
||||
"cmd/mesh-controller/record_mirrors.go",
|
||||
"internal/broker/controller_buckets.go",
|
||||
// The grant of a machine's own keys of the assignments mirror to its provisioner client (design 57 P3).
|
||||
|
||||
@@ -2,4 +2,4 @@ module route-proxy
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 h1:Sv1Qzix+U1RpaUkw5IbB0IJEyZ+FqgzHmbZrCZkcWf0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
|
||||
+23
@@ -335,6 +335,9 @@ func (p *Provider) pass(ctx context.Context, assignments, entries map[string][]b
|
||||
var out []Outcome
|
||||
unknown := map[string]bool{} // consumers whose assignment was skipped: their entries are not judged
|
||||
for _, key := range sortedKeys(assignments) {
|
||||
if !Standing(assignments[key]) {
|
||||
continue // ended or pending: read as a removed key (issue 1098)
|
||||
}
|
||||
var a planned
|
||||
if err := json.Unmarshal(assignments[key], &a); err != nil || a.Node+"."+a.Module != key {
|
||||
o := Outcome{Key: key, Status: Failed, Why: fmt.Sprintf("the assignment %s is not the controller's, so "+
|
||||
@@ -744,3 +747,23 @@ func (b stateBus) Delete(_ context.Context, bucket, key string) error {
|
||||
}
|
||||
return s.Delete(key)
|
||||
}
|
||||
|
||||
// Standing is whether an assignment's value is a standing assignment: false when `ended` is set (the module was
|
||||
// taken off its node, novox/hq ADR 0408 clause 4) or `state` is not empty (a pending one). Such a value is read
|
||||
// exactly as a removed key. Every other field, known or not, changes nothing (issue 1098).
|
||||
func Standing(value []byte) bool {
|
||||
var v struct {
|
||||
Ended json.RawMessage `json:"ended"`
|
||||
State json.RawMessage `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &v) != nil {
|
||||
return true // not decided here: the parse that follows refuses it
|
||||
}
|
||||
return !fieldSet(v.Ended) && !fieldSet(v.State)
|
||||
}
|
||||
|
||||
// fieldSet is whether a field carries something: not absent, null or empty.
|
||||
func fieldSet(f json.RawMessage) bool {
|
||||
s := strings.TrimSpace(string(f))
|
||||
return s != "" && s != "null" && s != `""`
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0
|
||||
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6
|
||||
## explicit; go 1.22
|
||||
git.novox.be/novox/mesh-sdk/go
|
||||
git.novox.be/novox/mesh-sdk/go/provider
|
||||
|
||||
Vendored
+16
@@ -139,3 +139,19 @@
|
||||
./cmd/mesh-controller TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate
|
||||
./cmd/mesh-controller TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt
|
||||
./cmd/mesh-controller TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus
|
||||
./internal/inventory TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing
|
||||
./internal/inventory TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing
|
||||
./internal/inventory TestTheMoveRefusesAValueOverTheLargestMessage
|
||||
./internal/inventory TestTheMoveRefusesOnACountThatDiffers
|
||||
./internal/inventory TestTheMoveWithoutTheLeaseWritesNothing
|
||||
./internal/inventory TestBuildsOnTheBusAreReadBackByEveryReader
|
||||
./internal/inventory TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace
|
||||
./internal/inventory TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain
|
||||
./internal/inventory TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame
|
||||
./cmd/mesh-controller TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain
|
||||
./internal/inventory TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt
|
||||
./internal/inventory TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves
|
||||
./internal/inventory TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent
|
||||
./cmd/mesh-controller TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments
|
||||
./cmd/mesh-controller TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment
|
||||
./internal/inventory TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved
|
||||
|
||||
Reference in New Issue
Block a user