Compare commits
39
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fb66608a13 | ||
|
|
7be2df627f | ||
|
|
cb006fc8dd | ||
|
|
0273609090 | ||
|
|
3b49d7044f | ||
|
|
ba59283b4b | ||
|
|
461fa9c7b4 | ||
|
|
06bf8cb2fb | ||
|
|
222c9c1d72 | ||
|
|
0520773412 | ||
|
|
eba864dc5f | ||
|
|
2d3d5ab616 | ||
|
|
dc65fe2319 | ||
|
|
fc43238267 | ||
|
|
8c8bc5021e | ||
|
|
557c11af1e | ||
|
|
661e8eff2f | ||
|
|
8c0fa40e65 | ||
|
|
6746ab6492 | ||
|
|
48e9656c67 | ||
|
|
a825a4c904 | ||
|
|
315c622386 | ||
|
|
d3556f452e | ||
|
|
68db18bdfc | ||
|
|
cbc6bac8c0 | ||
|
|
0b4cd46b3f | ||
|
|
7f074d9c94 | ||
|
|
e76ba966b3 | ||
|
|
da6d038e07 | ||
|
|
c795915315 | ||
|
|
aa02e7bf4e | ||
|
|
4eeba8fb20 | ||
|
|
d095a3dcef | ||
|
|
5b08eb09be | ||
|
|
472744bbaa | ||
|
|
9859923664 | ||
|
|
fdab58ae4f | ||
|
|
3b989d8bb3 | ||
|
|
88ea361bf6 |
@@ -33,7 +33,7 @@ const packageManagerKey = "the package manager ("
|
||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
heard := map[string]bool{}
|
||||
for _, m := range f.machines {
|
||||
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
|
||||
if !m.asleep() && !m.lastHeard.IsZero() && !m.silent(f.now) {
|
||||
heard[m.name] = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,7 +201,7 @@ func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.
|
||||
}
|
||||
|
||||
// asksRecords is the bucket the asks seat's declarer keeps its record of asks in.
|
||||
func asksRecords(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
func asksRecords(ctx context.Context, inv catalogueReader) (string, error) {
|
||||
declared, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
@@ -146,8 +147,18 @@ func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
// What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only
|
||||
// when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown.
|
||||
if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -46,7 +46,21 @@ func pushedBy(t *testing.T, open *stores, node string) string {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
|
||||
// A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was
|
||||
// built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109).
|
||||
builds := declared.Builds
|
||||
if builds == nil {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
builds = carriedBuilds(names, nil, current, nil)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
|
||||
@@ -389,10 +389,31 @@ func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, o
|
||||
// least one required status, no administrator merging past one.
|
||||
//
|
||||
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
||||
// repository and path. path is the module's directory as built; branch the trunk it is registered from.
|
||||
// repository and path; or, for the build of a merge, when mesh-delivery announced it on the operator's Approve of
|
||||
// that head (novox/hq ADR 0420 §7 for a moved module, ADR 0432 for a new one), the forge's checks still holding. path is the module's directory as built; branch the trunk it is registered from.
|
||||
func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source,
|
||||
buildID, path, branch string) (int64, error) {
|
||||
forms := newSourceForms(ctx, inv)
|
||||
return mayRegisterIn(ctx, inv, newSourceForms(ctx, inv), module, built, buildID, path, branch)
|
||||
}
|
||||
|
||||
// catalogueReader is the catalogue's manifests, as the router's records are found in them.
|
||||
type catalogueReader interface {
|
||||
Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error)
|
||||
}
|
||||
|
||||
// registrations is what a registration's decision reads of the mesh: *inventory.Inventory, or a test's stand-in.
|
||||
type registrations interface {
|
||||
catalogueReader
|
||||
SourceOf(ctx context.Context, module string) (inventory.Source, error)
|
||||
SourceIdentity(ctx context.Context, module string) (int64, error)
|
||||
Catalogued(ctx context.Context) ([]inventory.Entry, error)
|
||||
BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error)
|
||||
MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error)
|
||||
}
|
||||
|
||||
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
||||
func mayRegisterIn(ctx context.Context, inv registrations, forms *sourceForms, module string, built inventory.Source,
|
||||
buildID, path, branch string) (int64, error) {
|
||||
was, err := inv.SourceOf(ctx, module)
|
||||
isNew := errors.Is(err, inventory.ErrNoSuchModule)
|
||||
if err != nil && !isNew {
|
||||
@@ -404,7 +425,8 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
}
|
||||
refuse := func(why string) (int64, error) {
|
||||
return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+
|
||||
"controller's terminal, never through a verb: a registered module is what the next push sends, and whoever "+
|
||||
"controller's terminal, or by the build of a merge mesh-delivery announced on the operator's Approve of that "+
|
||||
"head, never through a verb: a registered module is what the next push sends, and whoever "+
|
||||
"may call a verb includes agents (novox/hq ADR 0266)", errNotItsSource, why)
|
||||
}
|
||||
|
||||
@@ -438,8 +460,11 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
// **Or the move the operator approved** (novox/hq ADR 0420 §7, issue 1025): a known module built from another
|
||||
// repository by a merge, whose pull request mesh-delivery merged on the operator's Approve of that head.
|
||||
// A verb alone, or a merge with no such warrant, is refused as before.
|
||||
// **Or a new module from a repository nothing registered is built from, on the same warrant** (novox/hq ADR 0432,
|
||||
// issue 1146): the build of a merge mesh-delivery announced on the operator's Approve of that head registers it.
|
||||
// The forge's protection and the repository's identity are checked below as for every registration.
|
||||
movedWhy := ""
|
||||
if why != "" && !terminal && !isNew {
|
||||
if why != "" && !terminal {
|
||||
accepted, err := approvedMove(ctx, inv, forms, buildID, built)
|
||||
switch {
|
||||
case err == nil:
|
||||
@@ -507,7 +532,10 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
owner, name, m, facts.ID, m, id))
|
||||
}
|
||||
}
|
||||
if movedWhy != "" {
|
||||
if movedWhy != "" && isNew {
|
||||
fmt.Printf("%s: %s is new, from %s/%s — registered at the operator's Approve: %s\n", buildID, module, owner, name,
|
||||
movedWhy)
|
||||
} else if movedWhy != "" {
|
||||
fmt.Printf("%s: %s moved from %s to %s/%s — registered at the operator's Approve: %s\n", buildID, module,
|
||||
sourceWords(was.Repository, was.Seat), owner, name, movedWhy)
|
||||
}
|
||||
@@ -516,7 +544,7 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
|
||||
// askedHereFor says the build was asked at the controller's terminal, for this repository and this path: a kept
|
||||
// request marked so, whose source is the outcome's (novox/hq ADR 0266).
|
||||
func askedHereFor(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
||||
func askedHereFor(ctx context.Context, inv registrations, forms *sourceForms, buildID string,
|
||||
built inventory.Source, path string) (bool, error) {
|
||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||
if err != nil || !found || !r.AtTerminal {
|
||||
|
||||
@@ -317,7 +317,7 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
||||
keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil
|
||||
}
|
||||
routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
routerAskOf = func(_ context.Context, _ catalogueReader, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
if id != a.merge.AskID {
|
||||
t.Errorf("%s: the router was asked for %q", c.what, id)
|
||||
}
|
||||
|
||||
@@ -13,9 +13,7 @@ import (
|
||||
)
|
||||
|
||||
// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
|
||||
// bus's key-value store, written by the controller that holds the lease. The tables of the controller's store are
|
||||
// copied into them once, by the serving controller at its start, and a kind is read from its bucket only once the move
|
||||
// has recorded it (inventory.ChainReads).
|
||||
// bus's key-value store, written by the controller that holds the lease, and the controller's store keeps none of them.
|
||||
|
||||
// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
|
||||
func chainHolds(ctx context.Context) error {
|
||||
@@ -33,97 +31,7 @@ func openTheChain(ctx context.Context) (*chain.Chain, error) {
|
||||
return chain.OnTheBus(ctx, js.Conn(), chainHolds)
|
||||
}
|
||||
|
||||
// kindChainMoveRefused is the condition of a move that stopped without switching a kind.
|
||||
const kindChainMoveRefused = "chain-move-refused"
|
||||
|
||||
// chainMoveRetry is how long a move that failed for want of the bus waits before it tries again.
|
||||
const chainMoveRetry = 30 * time.Second
|
||||
|
||||
// movingTheChain copies the tables into the buckets and records it, at the serving controller's start: after the buckets
|
||||
// exist and the lease is held, before anything reads a kind that has not moved. **The first try is made here, before the
|
||||
// controller serves**, so what it serves is read from one place. A move that fails for want of the bus is tried again
|
||||
// every half minute while the controller serves from the tables; one that is refused — a count that differs, a value
|
||||
// over the bus's largest — is raised as an urgent condition, and the tables stay what is read: nothing is half-switched.
|
||||
func movingTheChain(ctx context.Context, inv *inventory.Inventory) {
|
||||
say := func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }
|
||||
// The first try is made here, before the controller serves; the tries after it, while it serves.
|
||||
if movedOrStopped(ctx, inv, say) {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(chainMoveRetry):
|
||||
}
|
||||
if movedOrStopped(ctx, inv, say) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// movedOrStopped makes one try at the move and says whether there is nothing more to try: moved, refused, or the controller
|
||||
// ending. False for a failure of the kind that is tried again (the bus away, a write not granted yet, the lease not held).
|
||||
func movedOrStopped(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) bool {
|
||||
_, err := inv.MoveChain(ctx, say)
|
||||
var refusal *inventory.Refusal
|
||||
switch {
|
||||
case err == nil:
|
||||
go clearChainMove(ctx)
|
||||
go askAgainSoon(ctx, inv)
|
||||
return true
|
||||
case errors.As(err, &refusal):
|
||||
say("%v", err)
|
||||
go raiseChainMove(ctx, refusal)
|
||||
return true
|
||||
case ctx.Err() != nil:
|
||||
return true
|
||||
}
|
||||
say("the move to the bus did not finish, and the tables are read until it does: %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// chainMoveObservation is the urgent condition for a refused move.
|
||||
func chainMoveObservation(r *inventory.Refusal) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "chain." + r.Kind, Token: kindChainMoveRefused,
|
||||
Kind: kindChainMoveRefused, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Summary: "the controller's " + r.Kind + " were not moved to the bus: " + r.Why, Source: "chain move"}
|
||||
}
|
||||
|
||||
// whenKept runs f once the serving controller keeps conditions: the move runs before they are kept.
|
||||
func whenKept(ctx context.Context, f func(k *conditions.Keeper)) {
|
||||
for conditionsFrom == nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
f(conditionsFrom)
|
||||
}
|
||||
|
||||
func raiseChainMove(ctx context.Context, r *inventory.Refusal) {
|
||||
whenKept(ctx, func(k *conditions.Keeper) {
|
||||
if _, err := k.Observe(ctx, chainMoveObservation(r)); err != nil {
|
||||
fmt.Printf("chain: the condition for the refused move could not be raised: %v\n", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func clearChainMove(ctx context.Context) {
|
||||
whenKept(ctx, func(k *conditions.Keeper) {
|
||||
for _, kind := range []string{inventory.KindBuilds, inventory.KindRollouts, inventory.KindAssignments} {
|
||||
key := chainMoveObservation(&inventory.Refusal{Kind: kind}).Key()
|
||||
if _, err := k.Clear(ctx, key, "the "+kind+" are on the bus"); err != nil {
|
||||
fmt.Printf("chain: the condition for the move of the %s could not be cleared: %v\n", kind, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// attachTheChain gives the inventory of the serving controller the chain on its connection, then moves what has not moved.
|
||||
// attachTheChain gives the inventory of the serving controller the chain on its connection.
|
||||
func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
|
||||
bus, ok := server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
@@ -134,7 +42,19 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
||||
return err
|
||||
}
|
||||
inv.UseChain(c)
|
||||
movingTheChain(ctx, inv)
|
||||
inv.RequireLayout()
|
||||
// And the builds laid out under their modules, once, before they are read (issue 1121) — **before this returns**, so
|
||||
// nothing the start goes on to read meets a chain not laid out yet: on 2026-10-18 the layout ran beside the start, a
|
||||
// read of the start was refused, and the controller exited and was started again every five seconds. The layout of
|
||||
// keys laid out before skips them, so this takes seconds; one that fails is tried again beside the serving, loudly.
|
||||
if err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }); err != nil {
|
||||
fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err)
|
||||
go layBuildsOutSoon(ctx, inv)
|
||||
}
|
||||
// And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving.
|
||||
go askAgainSoon(ctx, inv)
|
||||
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
||||
go copySentBuildsSoon(ctx, inv)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -179,3 +99,79 @@ func askAgainSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
askAgain(ctx, inv)
|
||||
}
|
||||
}
|
||||
|
||||
// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not
|
||||
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
||||
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
||||
|
||||
// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
|
||||
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
|
||||
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
|
||||
// that names a commit and empties each column it has copied.
|
||||
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
failed := false
|
||||
for {
|
||||
_, err := inv.CopySentBuilds(ctx)
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
|
||||
Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
|
||||
if err != nil {
|
||||
fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
|
||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||
o.Summary += err.Error()
|
||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if keeper := conditionsFrom; keeper != nil && failed {
|
||||
_, _ = keeper.Clear(ctx, o.Key(), "copied")
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(chainMoveRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// chainMoveRetry is how long the copy waits before it tries again.
|
||||
const chainMoveRetry = 30 * time.Second
|
||||
|
||||
// kindBuildsNotLaidOut is the condition of the layout of the builds under their modules that has not succeeded: until it
|
||||
// has, no build is read, and a send that needs one is refused with the reason.
|
||||
const kindBuildsNotLaidOut = "builds-not-laid-out"
|
||||
|
||||
// layBuildsOutSoon lays the builds out under their modules as soon as this controller holds the lease (the chain refuses a
|
||||
// write before), and tries again every half minute while it fails, loudly: each failure is printed, and the first raises an
|
||||
// urgent condition until it has worked. It is idempotent (LayBuildsOut).
|
||||
func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
failed := false
|
||||
for {
|
||||
err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) })
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindBuildsNotLaidOut, Token: kindBuildsNotLaidOut,
|
||||
Kind: kindBuildsNotLaidOut, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
|
||||
if err != nil {
|
||||
fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
|
||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||
o.Summary += err.Error()
|
||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if keeper := conditionsFrom; keeper != nil && failed {
|
||||
_, _ = keeper.Clear(ctx, o.Key(), "laid out")
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(chainMoveRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -176,12 +176,12 @@ func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T
|
||||
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
|
||||
t.Fatalf("the assignment names %+v", on)
|
||||
}
|
||||
// A declaration sent by hand with no builds known says nothing of them.
|
||||
// A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit.
|
||||
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c1" {
|
||||
t.Fatalf("a send of unknown builds changed %+v", on)
|
||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() {
|
||||
t.Fatalf("a send of unknown builds left %+v", on)
|
||||
}
|
||||
// A module sent a commit no build is kept of is named by its commit alone.
|
||||
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
|
||||
@@ -207,3 +207,39 @@ func runOf(on []chain.Assignment, module string) chain.Assignment {
|
||||
a, _ := assignmentOf(on, module)
|
||||
return a
|
||||
}
|
||||
|
||||
// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered
|
||||
// through the `assign` verb for a module whose build is in flight, one read of `<node>.<module>` says state pending with the
|
||||
// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied,
|
||||
// and the table of pending assignments is read by nobody. No key is written by hand.
|
||||
func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatalf("an assignment while its build runs was refused: %v", err)
|
||||
}
|
||||
waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
||||
if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() ||
|
||||
waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" ||
|
||||
!strings.HasPrefix(waiting.Pending.ID, "p-") {
|
||||
t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err)
|
||||
}
|
||||
if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") {
|
||||
t.Fatalf("a module only waiting is assigned: %v", got)
|
||||
}
|
||||
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
||||
if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil ||
|
||||
made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID {
|
||||
t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err)
|
||||
}
|
||||
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatal("the build registered sensors and it was not assigned")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR
|
||||
// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's
|
||||
// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the
|
||||
// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays.
|
||||
func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(91*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
||||
}
|
||||
rollout := ws[0].ID
|
||||
asking, err := inv.PlanByID(ctx, rollout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ {
|
||||
advancePlans(ctx, open)
|
||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
buildID := asking.Modules["notes"].Build
|
||||
if buildID == "" || len(*asked) != 1 {
|
||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("b", 64)
|
||||
result := outcome(buildID, "modules/notes", "c-notes", "notes", "")
|
||||
result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}}
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// What the live path made is in the chain.
|
||||
if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" {
|
||||
t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err)
|
||||
}
|
||||
if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout {
|
||||
t.Fatalf("the rollout is not read from the chain: %+v %v", p, err)
|
||||
}
|
||||
if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, ok := assignmentOf(on, "notes"); !ok {
|
||||
t.Fatalf("the assignment is not read from the chain: %+v", on)
|
||||
}
|
||||
|
||||
// And the catalog of the store holds none of them.
|
||||
have := map[string]bool{}
|
||||
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
||||
have[strings.Trim(row[0], `"`)] = true
|
||||
}
|
||||
for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} {
|
||||
if have[table] {
|
||||
t.Errorf("the table %s is in the controller's store", table)
|
||||
}
|
||||
}
|
||||
for _, table := range []string{"build_gate", "assignment_generation"} {
|
||||
if !have[table] {
|
||||
t.Errorf("the table %s, which stays, is gone", table)
|
||||
}
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
|
||||
t.Errorf("the function raise_assignment_generation is in the store: %v", rows)
|
||||
}
|
||||
}
|
||||
@@ -56,13 +56,19 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
// Built from a commit, so that a send carries them: a holder is asked only of what the machine was sent (issue 1109).
|
||||
built := inventory.Source{BuiltFrom: "c1"}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if err := inv.RegisterModule(ctx, house, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
|
||||
@@ -120,6 +120,11 @@ var probeRegistry = []probe{
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// What the watchman says stopped moving (ADR 0433 §2, issue 1156): each key of its stuck bucket not moved, but a
|
||||
// node's presence, which the silent check takes.
|
||||
{ID: probeStuckID, Asserts: "no key the watchman says stopped moving is left unsaid: each key of its stuck bucket " +
|
||||
"not moved is a condition of the delivery scope until it says moved", From: "ADR 0433 §2, issue 1156",
|
||||
Kind: kindStuck, Phase: 3, run: probeStuck},
|
||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
||||
@@ -178,6 +183,10 @@ var probeRegistry = []probe{
|
||||
{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
|
||||
"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
|
||||
run: probeBackupHeld},
|
||||
// A plan's send key the bus refused (ADR 0433 §4, issue 1157): in place of the timer that stood in for it.
|
||||
{ID: sendKeyRefusedID, Asserts: "no delivery plan's key that watches a machine for its report was refused by the " +
|
||||
"bus at the newest look", From: "ADR 0433 §4, issue 1157", Kind: kindSendKeyRefused, Phase: 1,
|
||||
run: probeSendKeyRefused},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
|
||||
@@ -1120,6 +1120,9 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error)
|
||||
drop()
|
||||
return nil, nil, err
|
||||
}
|
||||
// The chain lives on the bus, and the gate has none: this mesh is raised for a moment, with a chain of its own in memory
|
||||
// that no other reads (issue 1121).
|
||||
open.inventory.UseThrowawayChain()
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(ctx, m); err != nil {
|
||||
open.Close()
|
||||
|
||||
@@ -50,9 +50,17 @@ func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) {
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
<-done
|
||||
lookingAtBatches.Wait()
|
||||
setWindowClock(nil)
|
||||
windowLook.Store(nil)
|
||||
})
|
||||
// The forge's merges are taken from a consumer made when the server starts serving: a merge published before it exists
|
||||
// is not delivered to it. With the chain read from the bus the set-up above is quick enough to lose that race, so the
|
||||
// publisher waits for the consumer.
|
||||
becomes(t, 10*time.Second, "the merges' consumer being made", func() bool {
|
||||
_, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
|
||||
return err == nil
|
||||
})
|
||||
return func(m link.SourceMoved) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(m)
|
||||
|
||||
@@ -145,9 +145,15 @@ func (w *busWindow) Forget(ctx context.Context, batch string) {
|
||||
}
|
||||
}
|
||||
|
||||
// lookingAtBatches counts the looks under way, so a test that ends the window's clock can wait for the last of them before the
|
||||
// next test replaces what they read.
|
||||
var lookingAtBatches sync.WaitGroup
|
||||
|
||||
// lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches
|
||||
// whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once.
|
||||
func lookAtBatches(ctx context.Context, open *stores) {
|
||||
lookingAtBatches.Add(1)
|
||||
defer lookingAtBatches.Done()
|
||||
release, err := open.inventory.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err)
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1145/AC1 on a store: declarationWith itself sets the module of each resource and the build of each module for a
|
||||
// node recorded as reading them, and for no other. Skips without a store.
|
||||
func TestAComposedDeclarationCarriesModulesAndBuildsOnlyForANodeThatReadsThem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
start := time.Now().Add(-time.Hour)
|
||||
manifest := resticFixture(false)
|
||||
manifest["module"], manifest["version"] = "restic", "1"
|
||||
raw, _ := json.Marshal(manifest)
|
||||
b := link.BuildResult{ID: link.NewBuildID(start), Repository: "novox/mesh-catalog", Path: "modules/restic",
|
||||
On: "anchor", Module: "restic", Commit: "c1", Manifest: raw,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, "restic"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
anchor, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordReadsModules(ctx, anchor.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantBuild, err := inv.BuildOf(ctx, "restic", "c1")
|
||||
if err != nil || wantBuild == "" {
|
||||
t.Fatalf("the build of restic at c1 is %q (%v)", wantBuild, err)
|
||||
}
|
||||
|
||||
declare := func(node string) sendable {
|
||||
t.Helper()
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := declarationWith(ctx, open, node, plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
reads := bodyKeys(t, declare("anchor"))
|
||||
builds, _ := reads["builds"].(map[string]any)
|
||||
if builds["restic"] != wantBuild {
|
||||
t.Errorf("1145/AC1: anchor reads modules and was sent builds %v, want restic at %s", reads["builds"], wantBuild)
|
||||
}
|
||||
owned := 0
|
||||
for _, r := range reads["resources"].([]any) {
|
||||
r := r.(map[string]any)
|
||||
if strings.HasPrefix(r["id"].(string), "restic.") {
|
||||
owned++
|
||||
if r["module"] != "restic" {
|
||||
t.Errorf("1145/AC1: resource %v carries module %v, want restic", r["id"], r["module"])
|
||||
}
|
||||
}
|
||||
}
|
||||
if owned == 0 {
|
||||
t.Fatal("1145/AC1: no resource of restic was composed")
|
||||
}
|
||||
|
||||
plain := bodyKeys(t, declare("laptop"))
|
||||
if _, there := plain["builds"]; there {
|
||||
t.Errorf("1145/AC1: laptop never said it reads modules and was sent builds %v", plain["builds"])
|
||||
}
|
||||
for _, r := range plain["resources"].([]any) {
|
||||
if _, there := r.(map[string]any)["module"]; there {
|
||||
t.Fatalf("1145/AC1: laptop was sent a module on %v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-host/validate"
|
||||
)
|
||||
|
||||
// The module of each resource and the build of each module (novox/hq issue 1145, design 47 release 1-2). The
|
||||
// composer's owner of an id and the module's build go on the wire only to a machine whose node-engine said it reads
|
||||
// them: an older one decodes strictly and refuses a key it does not know, whole.
|
||||
|
||||
func modulesDeclaration(reads bool) sendable {
|
||||
return sendable{
|
||||
Resources: []map[string]any{
|
||||
{"id": "web.app.conf", "type": "file", "path": "/etc/app.conf", "content": "x\n"},
|
||||
{"id": "web.app.data", "type": "file", "path": "/etc/data", "content": "d\n"},
|
||||
{"id": "mesh.guard", "type": "file", "path": "/etc/guard", "content": "g\n"}},
|
||||
moduleOf: map[string]string{"web.app.conf": "web.app", "web.app.data": "web.app"},
|
||||
buildIDs: map[string]string{"web.app": "build-7"},
|
||||
ReadsModules: reads,
|
||||
}
|
||||
}
|
||||
|
||||
// 1145/AC1
|
||||
func TestAMachineThatReadsModulesIsSentEachResourcesModuleAndEachModulesBuild(t *testing.T) {
|
||||
keys := bodyKeys(t, modulesDeclaration(true))
|
||||
builds, _ := keys["builds"].(map[string]any)
|
||||
if len(builds) != 1 || builds["web.app"] != "build-7" {
|
||||
t.Fatalf("1145/AC1: the body carries builds %v", keys["builds"])
|
||||
}
|
||||
want := map[string]any{"web.app.conf": "web.app", "web.app.data": "web.app", "mesh.guard": nil}
|
||||
for _, r := range keys["resources"].([]any) {
|
||||
r := r.(map[string]any)
|
||||
if r["module"] != want[r["id"].(string)] {
|
||||
t.Errorf("1145/AC1: resource %v carries module %v, want %v", r["id"], r["module"], want[r["id"].(string)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1145/AC2: the body is byte for byte what it is today.
|
||||
func TestAMachineThatDoesNotReadModulesIsSentTheBodyItWasSentBefore(t *testing.T) {
|
||||
d := modulesDeclaration(false)
|
||||
got, err := d.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := sendable{Resources: d.Resources}
|
||||
want, _ := before.Body()
|
||||
if string(got) != string(want) {
|
||||
t.Fatalf("1145/AC2: the body changed:\n%s\nwas\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1145/AC4: the node-engine's own validator, as the gate runs it, takes the two words.
|
||||
func TestTheNodeEnginesValidatorTakesAModulePerResourceAndABuildPerModule(t *testing.T) {
|
||||
raw, err := modulesDeclaration(true).Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var generic map[string]any
|
||||
if err := json.Unmarshal(raw, &generic); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if problems := validate.Declaration(raw); len(problems) != 0 {
|
||||
t.Fatalf("1145/AC4: refused: %q\n%s", problems, fmt.Sprint(generic))
|
||||
}
|
||||
}
|
||||
|
||||
// The build a module carries is the one its assignment names when that is the commit carried, else the one built from
|
||||
// the commit carried; none when neither is known.
|
||||
func TestTheBuildOfACarriedCommitIsTheAssignmentsOrTheOneBuiltFromIt(t *testing.T) {
|
||||
runs := map[string]chain.Ref{"a": {Build: "b-a", Commit: "c1"}, "b": {Build: "b-old", Commit: "c0"}}
|
||||
looked := map[string]string{"b/c2": "b-new"}
|
||||
got, err := wireBuilds(map[string]string{"a": "c1", "b": "c2", "c": "c3", "d": ""}, runs,
|
||||
func(module, commit string) (string, error) { return looked[module+"/"+commit], nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"a": "b-a", "b": "b-new"}
|
||||
if len(got) != len(want) || got["a"] != want["a"] || got["b"] != want["b"] {
|
||||
t.Fatalf("1145: builds %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -168,7 +168,7 @@ var keptDeliveryMerge = func(ctx context.Context, owner, repo string, number int
|
||||
|
||||
// routerAskOf is the router's record of one of mesh-delivery's asks: the ask as published and its warrant. A
|
||||
// variable so a test can hand it one.
|
||||
var routerAskOf = func(ctx context.Context, inv *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
var routerAskOf = func(ctx context.Context, inv catalogueReader, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return nil, nil, errors.New("only the serving controller reads the router's record")
|
||||
@@ -216,7 +216,7 @@ var errNotAMerge = errors.New("the build is not of a merge")
|
||||
// (ADR 0420 §7): the build is of a merge, that merge is a pull request mesh-delivery announced it made, and the
|
||||
// ask it made it on holds an Approve by the operator of that head. Answers why it was accepted; an error says why
|
||||
// it was not, and the refusal of ADR 0266 stands.
|
||||
func approvedMove(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
||||
func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, buildID string,
|
||||
built inventory.Source) (string, error) {
|
||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **The operator's Approve of a merge registers a new module from its repository** (novox/hq ADR 0432, issue 1146).
|
||||
// The mesh as the decision reads it, with nothing registered from the repository, and the build request and merge the
|
||||
// controller kept.
|
||||
type anEmptyMesh struct {
|
||||
request inventory.BuildRequest
|
||||
asked bool
|
||||
merge inventory.BatchedMerge
|
||||
}
|
||||
|
||||
func (m *anEmptyMesh) Catalogue(context.Context) (map[string]catalogue.Manifest, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *anEmptyMesh) SourceOf(context.Context, string) (inventory.Source, error) {
|
||||
return inventory.Source{}, inventory.ErrNoSuchModule
|
||||
}
|
||||
func (m *anEmptyMesh) SourceIdentity(context.Context, string) (int64, error) { return 0, nil }
|
||||
func (m *anEmptyMesh) Catalogued(context.Context) ([]inventory.Entry, error) { return nil, nil }
|
||||
func (m *anEmptyMesh) BuildRequestByID(_ context.Context, id string) (inventory.BuildRequest, bool, error) {
|
||||
return m.request, m.asked && id == m.request.ID, nil
|
||||
}
|
||||
func (m *anEmptyMesh) MergeOf(context.Context, string, string) (inventory.BatchedMerge, bool, error) {
|
||||
return m.merge, true, nil
|
||||
}
|
||||
|
||||
// aDeliveryMergeOf is the Approved merge of a pull request of novox/mesh-delivery, its heads keyed by that repository,
|
||||
// and the ask and warrant bound to exactly those heads (the digest the Approve left covers them).
|
||||
func aDeliveryMergeOf(t *testing.T, head string, covered map[string]string) anApprovedMerge {
|
||||
t.Helper()
|
||||
a := approvedMerge(t)
|
||||
a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-delivery", 7, head
|
||||
a.merge.Heads = covered
|
||||
for n := range a.ask.Options {
|
||||
d, err := asks.ActDigest(mergeAct(a.merge.AskID, a.ask.Options[n].ID, covered))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.ask.Options[n].Binds = d
|
||||
}
|
||||
a.warrant.AskDigest = a.ask.Digest()
|
||||
return a
|
||||
}
|
||||
|
||||
func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwise(t *testing.T) {
|
||||
const commit, head = "c0ffee0123456789", "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||
built := inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
change func(*anApprovedMerge)
|
||||
request string // "merge", or "build" through a verb
|
||||
unkept bool
|
||||
forge any // what the forge says of the repository; nil is protected
|
||||
built *inventory.Source
|
||||
accepted bool
|
||||
ac string
|
||||
}{
|
||||
{what: "the Approve of the merged head, a protected trunk on the forge", request: "merge", accepted: true, ac: "1146/AC1"},
|
||||
{what: "a build asked through a verb", request: "build", ac: "1146/AC2"},
|
||||
{what: "a merge with no warrant", request: "merge", unkept: true, ac: "1146/AC2"},
|
||||
{what: "a merged head the Approve did not cover", request: "merge", ac: "1146/AC2", change: func(a *anApprovedMerge) {
|
||||
*a = aDeliveryMergeOf(t, head, map[string]string{
|
||||
"novox/mesh-delivery@eeeeeeeeeeee": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"})
|
||||
}},
|
||||
{what: "heads other than the ones the Approve was bound to", request: "merge", ac: "1146/AC2", change: func(a *anApprovedMerge) {
|
||||
a.merge.Heads = map[string]string{"novox/mesh-delivery@eeeeeeeeeeee": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"}
|
||||
}},
|
||||
{what: "a repository not on the mesh's forge", request: "merge", ac: "1146/AC3",
|
||||
built: &inventory.Source{Repository: "http://elsewhere.example/x/y.git"}},
|
||||
{what: "a forge that cannot be asked", request: "merge", ac: "1146/AC3", forge: errors.New("connection refused")},
|
||||
{what: "an unprotected trunk", request: "merge", ac: "1146/AC3", forge: forgeFacts{ID: 9, Guarded: false, Why: "direct push"}},
|
||||
} {
|
||||
t.Run(c.ac+" "+c.what, func(t *testing.T) {
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
if c.change != nil {
|
||||
c.change(&a)
|
||||
}
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
if c.forge != nil {
|
||||
theForge.Store("novox/mesh-delivery", c.forge)
|
||||
t.Cleanup(func() { theForge.Delete("novox/mesh-delivery") })
|
||||
}
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||
"head_sha": head, "merge_commit_sha": commit})
|
||||
mesh := &anEmptyMesh{asked: true, merge: inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main",
|
||||
Commit: commit, Merged: time.Now(), Event: event},
|
||||
request: inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "",
|
||||
Ref: "main", Commit: commit, For: c.request}}
|
||||
forms := &sourceForms{bases: map[string]string{}, base: func(seat string) string {
|
||||
if seat == "git" {
|
||||
return "http://forge.internal:20000"
|
||||
}
|
||||
return ""
|
||||
}}
|
||||
src := built
|
||||
if c.built != nil {
|
||||
src = *c.built
|
||||
}
|
||||
id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-view", src, "build-1", "", "main")
|
||||
switch {
|
||||
case c.accepted && err != nil:
|
||||
t.Fatalf("%s: the Approved merge's new module was refused: %v", c.ac, err)
|
||||
case c.accepted && id == 0:
|
||||
t.Fatalf("%s: registered with no forge identity to record", c.ac)
|
||||
case !c.accepted && !errors.Is(err, errNotItsSource):
|
||||
t.Fatalf("%s: %s was not refused: %v", c.ac, c.what, err)
|
||||
case !c.accepted && c.ac == "1146/AC2" &&
|
||||
(!strings.Contains(err.Error(), "controller's terminal") || !strings.Contains(err.Error(), "Approve")):
|
||||
t.Fatalf("%s: the refusal does not name both ways in: %v", c.ac, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A second new module from a repository whose first was registered is the case ADR 0266 §44 already has —
|
||||
// "alongside": registered through a verb with no warrant asked of the router (1146/AC5).
|
||||
type aMeshWithOneAlready struct{ anEmptyMesh }
|
||||
|
||||
func (m *aMeshWithOneAlready) Catalogued(context.Context) ([]inventory.Entry, error) {
|
||||
e := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}}
|
||||
e.Manifest.Module = "mesh-delivery-view"
|
||||
return []inventory.Entry{e}, nil
|
||||
}
|
||||
|
||||
func TestASecondNewModuleFromARepositoryIsJudgedAlongsideAndAsksNoWarrant(t *testing.T) {
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
t.Error("1146/AC5: the announced merges were asked for a module judged alongside")
|
||||
return deliveryMerge{}, false, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
t.Error("1146/AC5: the router was asked for a module judged alongside")
|
||||
return nil, nil, nil
|
||||
}
|
||||
mesh := &aMeshWithOneAlready{anEmptyMesh{asked: true,
|
||||
request: inventory.BuildRequest{ID: "build-2", Repository: "novox/mesh-delivery", Seat: "git", For: "build"}}}
|
||||
forms := &sourceForms{bases: map[string]string{}, base: func(string) string { return "http://forge.internal:20000" }}
|
||||
built := inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}
|
||||
if id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-watchman", built, "build-2", "", "main"); err != nil || id == 0 {
|
||||
t.Fatalf("1146/AC5: a second module from a repository the catalogue builds from was refused (id %d): %v", id, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1146/AC4: a new module from a repository nothing registered is built from, whole against a real bus and the store
|
||||
// (novox/hq ADR 0432, issue 1146; the warrant is read as for a move, ADR 0420 §7, issue 1025): mesh-delivery's word that it
|
||||
// merged a pull request is published on the subject the controller consumes and kept in its bucket; the router's
|
||||
// record of the ask is read back from the router's bucket by a direct get; and the build of a module from the
|
||||
// repository that merge was made in is registered from there only when that record holds the operator's Approve of
|
||||
// the merged head. Store-backed: skips without one.
|
||||
func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
answer bool // the router holds the operator's Approve of the merged heads
|
||||
moved bool
|
||||
}{
|
||||
{what: "the router holds the operator's Approve", answer: true, moved: true},
|
||||
{what: "the router holds no record of the ask"},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
// The module declaring the router's records, as the controller finds them.
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
|
||||
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
a := aDeliveryMergeOf(t, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"})
|
||||
if c.answer {
|
||||
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The events consumer exists, from now on, before the word is published: Receive would make it too late.
|
||||
if err := broker.AssertServingConsumers(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serving, stop := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = server.Serve(serving) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
|
||||
// mesh-delivery says it merged the pull request, and the controller keeps what it said.
|
||||
body, _ := json.Marshal(a.merge)
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubject, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the merge being kept", func() bool {
|
||||
_, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, a.merge.HeadSHA)
|
||||
return err == nil && kept
|
||||
})
|
||||
|
||||
// The forge's merge of that pull request, as the controller heard it, and the build it asked for.
|
||||
const commit = "c0ffee0123456789"
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-delivery", Seat: "git",
|
||||
Path: "modules/mesh-delivery-view", Ref: "main", Commit: commit, For: "merge"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := onTrunk(id, "novox/mesh-delivery", "git", "modules/mesh-delivery-view", map[string]any{"module": "mesh-delivery-view", "version": "2"})
|
||||
moved.Commit = commit
|
||||
_, _, err = takeIn(ctx, inv, moved)
|
||||
src, _ := inv.SourceOf(ctx, "mesh-delivery-view")
|
||||
if c.moved && (err != nil || src.Repository != "novox/mesh-delivery") {
|
||||
t.Fatalf("the Approved move was not registered from its repository: %v (built from %q)", err, src.Repository)
|
||||
}
|
||||
if !c.moved && (!errors.Is(err, errNotItsSource) || src.Repository != "") {
|
||||
t.Fatalf("a new module with no Approve was not refused as it is: %v (built from %q)", err, src.Repository)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -491,8 +491,7 @@ func settlingPending(ctx context.Context, open *stores) {
|
||||
|
||||
// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds;
|
||||
// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound,
|
||||
// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than
|
||||
// KeptFor are deleted. Returns what it did.
|
||||
// ends its assignment; ended ones are raised as conditions and cleared once answered. Returns what it did.
|
||||
func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
||||
inv := open.inventory
|
||||
var said []string
|
||||
@@ -565,11 +564,6 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
||||
}
|
||||
|
||||
said = append(said, raisePendingEnded(ctx, inv)...)
|
||||
if n, err := inv.ForgetEndedPending(ctx, now); err != nil {
|
||||
say("ended pending assignments could not be deleted: %v", err)
|
||||
} else if n > 0 {
|
||||
say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor)
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
@@ -585,7 +579,7 @@ func containsString(xs []string, x string) bool {
|
||||
// pendingObservation is the condition for a pending assignment that was not made: one per pending
|
||||
// assignment, its row's id the last part of its id, so one row's condition is never another's.
|
||||
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID),
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%s", p.Node, p.Module, p.ID),
|
||||
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
|
||||
Resolver: conditions.ResolverOperator, Source: "pending assignments",
|
||||
Summary: p.Note}
|
||||
@@ -671,8 +665,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)}
|
||||
}
|
||||
byID := map[int64]string{}
|
||||
var ids []int64
|
||||
byID := map[string]string{}
|
||||
var ids []string
|
||||
for _, c := range open {
|
||||
if c.Kind != kindPendingEnded {
|
||||
continue
|
||||
@@ -682,8 +676,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
var id int64
|
||||
if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil {
|
||||
id := parts[len(parts)-2]
|
||||
if !strings.HasPrefix(id, "p-") {
|
||||
continue
|
||||
}
|
||||
byID[id] = c.Key
|
||||
@@ -701,7 +695,7 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
|
||||
if known[id] {
|
||||
continue
|
||||
}
|
||||
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil {
|
||||
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed, or the module was assigned and the wait taken back"); err != nil {
|
||||
said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -542,23 +542,26 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
|
||||
t.Fatalf("a module registered meanwhile: %+v", p)
|
||||
}
|
||||
|
||||
// Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is
|
||||
// still open (second review, bug B).
|
||||
// Ended waits are not deleted, for the chain removes no key (ADR 0408 section 4): after KeptFor they are no longer listed, open
|
||||
// ones never are left out, nor one whose condition is still open (second review, bug B).
|
||||
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||
rows, err := inv.Pending(ctx, time.Unix(0, 0))
|
||||
later := time.Now().Add(inventory.KeptFor + time.Hour)
|
||||
settlePending(ctx, open, later)
|
||||
rows, err := inv.Pending(ctx, later.Add(-inventory.KeptFor))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if !r.Open() && (r.Raised == nil || r.Cleared != nil) {
|
||||
t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said)
|
||||
t.Fatalf("an ended pending assignment with no open condition is listed past %s: %+v", inventory.KeptFor, r)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") {
|
||||
t.Fatalf("the applied row was not deleted: %v", said)
|
||||
for _, r := range rows {
|
||||
if r.Module == "heard" {
|
||||
t.Fatalf("an applied wait is listed past %s: %+v", inventory.KeptFor, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -637,8 +640,9 @@ func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) {
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||
if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] {
|
||||
t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err)
|
||||
// No key of the chain is removed (ADR 0408 §4): a cleared wait stays in the value of its key, saying it was cleared.
|
||||
if known, err := inv.PendingKnown(ctx, []string{row.ID}); err != nil || !known[row.ID] || rowOf(t, open, row.ID).Cleared == nil {
|
||||
t.Fatalf("a cleared row is not kept, saying so: %v", err)
|
||||
}
|
||||
|
||||
// On anchor, then anchor removed: the row goes, and its condition with it at the next tick.
|
||||
|
||||
@@ -23,7 +23,7 @@ func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool {
|
||||
return found
|
||||
}
|
||||
|
||||
func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
|
||||
func rowOf(t *testing.T, open *stores, id string) inventory.PendingAssignment {
|
||||
t.Helper()
|
||||
rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0))
|
||||
if err != nil {
|
||||
@@ -34,7 +34,7 @@ func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("pending assignment %d is not on record", id)
|
||||
t.Fatalf("pending assignment %s is not on record", id)
|
||||
return inventory.PendingAssignment{}
|
||||
}
|
||||
|
||||
|
||||
@@ -610,6 +610,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"did not answer in ten minutes, so nothing of the delivery is built or sent.",
|
||||
Resolved: "Resolved: the backup is done and the delivery goes on"}
|
||||
}),
|
||||
kindStuck: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Something in a delivery has stopped moving",
|
||||
Explanation: "The watchman noticed a part of the delivery that did not move when it should have: a build " +
|
||||
"asked and not answered, a check that stopped, or a part that went offline. Nothing is lost, and it is " +
|
||||
"said again until it moves.",
|
||||
Resolved: "Resolved: it moves again"}
|
||||
}),
|
||||
kindSendKeyRefused: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery cannot watch a machine for its report",
|
||||
Explanation: "The bus refused the note that makes the mesh notice a machine that never reports on a send. " +
|
||||
"The delivery goes on, and the note is written again at its next look.",
|
||||
Resolved: "Resolved: the delivery watches the machine for its report again"}
|
||||
}),
|
||||
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Messages pile up for a listener",
|
||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
||||
|
||||
+89
-22
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -500,33 +501,75 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
carried, err := carriedOn(ctx, open, node, plan, composed.LeftOut, true)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
before, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
if !known {
|
||||
before = nil
|
||||
}
|
||||
names := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
||||
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0245).
|
||||
kept, err := recordedKept(ctx, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
for m, was := range kept {
|
||||
if _, carried := out.Builds[m]; carried {
|
||||
out.Builds[m] = was
|
||||
out.Builds = carried
|
||||
out.Bindings = boundToData(plan, composed.LeftOut)
|
||||
}
|
||||
// And the module of each resource and the build of each module, to a machine whose node-engine said it reads them
|
||||
// (novox/hq issue 1145) and to no other; the same whatever the composition is for, so what a question about the
|
||||
// send digests is what the send digests.
|
||||
reads, err := open.inventory.ReadsModules(ctx, record.ID)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
if reads {
|
||||
carried := out.Builds
|
||||
if choosing != Allocating {
|
||||
if carried, err = carriedOn(ctx, open, node, plan, composed.LeftOut, false); err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
}
|
||||
out.Bindings = boundToData(plan, composed.LeftOut)
|
||||
runs, err := open.inventory.AssignmentRuns(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
builds, err := wireBuilds(carried, runs, func(module, commit string) (string, error) {
|
||||
return open.inventory.BuildOf(ctx, module, commit)
|
||||
})
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
out.ReadsModules, out.moduleOf, out.buildIDs = true, composed.Owner, builds
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// carriedOn is the build of each module a declaration for node carries (see carriedBuilds), the module kept at the
|
||||
// build a machine runs included (ADR 0245). Only a send (strict) fails for a kept module that cannot be read: a question
|
||||
// about it is answered without.
|
||||
func carriedOn(ctx context.Context, open *stores, node string, plan catalogue.Resolution, leftOut map[string]string,
|
||||
strict bool) (map[string]string, error) {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
before, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !known {
|
||||
before = nil
|
||||
}
|
||||
names := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
out := carriedBuilds(names, leftOut, current, before)
|
||||
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0245).
|
||||
kept, err := recordedKept(ctx, open, node)
|
||||
if err != nil {
|
||||
if strict {
|
||||
return nil, err
|
||||
}
|
||||
kept = nil
|
||||
}
|
||||
for m, was := range kept {
|
||||
if _, carried := out[m]; carried {
|
||||
out[m] = was
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -2109,3 +2152,27 @@ func sortedNames[V any](m map[string]V) []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// wireBuilds is the build of each carried module that goes on the wire (novox/hq issue 1145): the build its assignment
|
||||
// names when that assignment is of the commit carried, else the build made from that commit; a module with neither
|
||||
// is not named.
|
||||
func wireBuilds(carried map[string]string, runs map[string]chain.Ref, lookup func(module, commit string) (string, error)) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for module, commit := range carried {
|
||||
if commit == "" {
|
||||
continue
|
||||
}
|
||||
if ref, ok := runs[module]; ok && ref.Commit == commit && ref.Build != "" {
|
||||
out[module] = ref.Build
|
||||
continue
|
||||
}
|
||||
build, err := lookup(module, commit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if build != "" {
|
||||
out[module] = build
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -400,33 +400,39 @@ func syncSilentKeys(ctx context.Context, open *stores) error {
|
||||
}
|
||||
|
||||
// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys
|
||||
// cannot be read, one cannot be put — the wait is looked at by a timer of its own, once, and the error is given back so
|
||||
// the look is retried.
|
||||
// cannot be read, one cannot be put — the refusal is raised as a condition naming the plan and the machine, never
|
||||
// stood in for by a timer, and the error is given back so the look is retried and writes the key again (novox/hq ADR
|
||||
// 0433 §4). The set of refusals is what this look found: a key written at the next look is no longer in it.
|
||||
func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error {
|
||||
byTimer := func() {
|
||||
for w, sent := range want {
|
||||
silentByTimer(w, sent, silentTTL(sent, now))
|
||||
refused := func(err error) map[silentWait]error {
|
||||
out := map[silentWait]error{}
|
||||
for w := range want {
|
||||
out[w] = err
|
||||
}
|
||||
return out
|
||||
}
|
||||
if c == nil {
|
||||
byTimer()
|
||||
noteRefusedSendKeys(refused(errors.New("the bus's clock for the waits is not there")))
|
||||
return nil
|
||||
}
|
||||
c = boundedClock{c}
|
||||
held, err := c.Held(ctx)
|
||||
if err != nil {
|
||||
byTimer()
|
||||
noteRefusedSendKeys(refused(err))
|
||||
return err
|
||||
}
|
||||
arm, drop := silentDiff(want, held)
|
||||
var first error
|
||||
notTaken := map[silentWait]error{}
|
||||
for _, w := range arm {
|
||||
if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
|
||||
silentByTimer(w, want[w], silentTTL(want[w], now))
|
||||
notTaken[w] = err
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
}
|
||||
noteRefusedSendKeys(notTaken)
|
||||
for _, w := range drop {
|
||||
if err := c.Reported(ctx, w); err != nil && first == nil {
|
||||
first = err
|
||||
@@ -435,29 +441,49 @@ func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time
|
||||
return first
|
||||
}
|
||||
|
||||
var silentTimers = struct {
|
||||
mu sync.Mutex
|
||||
timers map[silentWait]time.Time
|
||||
}{timers: map[silentWait]time.Time{}}
|
||||
// silentCallBound is how long one call to the silent clock may take while the look holds the plans (novox/hq issue 1121).
|
||||
// The vendored key listing reads its watcher without the context, so a consumer lost when the broker reloaded its user
|
||||
// list left the look waiting for good, with the plans and every node held.
|
||||
var silentCallBound = 20 * time.Second
|
||||
|
||||
// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key
|
||||
// (DegradableGrants says why that can be): never silently never. Once for each send.
|
||||
func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) {
|
||||
silentTimers.mu.Lock()
|
||||
defer silentTimers.mu.Unlock()
|
||||
if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) {
|
||||
return
|
||||
// boundedClock is a clock whose every call is answered within silentCallBound, or fails: a call that never returns is
|
||||
// left behind on its own goroutine.
|
||||
type boundedClock struct{ silentClock }
|
||||
|
||||
func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, silentCallBound)
|
||||
defer cancel()
|
||||
type answer struct {
|
||||
v T
|
||||
err error
|
||||
}
|
||||
silentTimers.timers[w] = sent
|
||||
// retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once.
|
||||
time.AfterFunc(ttl, func() {
|
||||
silentTimers.mu.Lock()
|
||||
if silentTimers.timers[w].Equal(sent) {
|
||||
delete(silentTimers.timers, w)
|
||||
}
|
||||
silentTimers.mu.Unlock()
|
||||
askToLook()
|
||||
done := make(chan answer, 1)
|
||||
go func() { v, err := call(ctx); done <- answer{v, err} }()
|
||||
select {
|
||||
case a := <-done:
|
||||
return a.v, a.err
|
||||
case <-ctx.Done():
|
||||
var zero T
|
||||
return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func (b boundedClock) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error {
|
||||
_, err := boundedCall(ctx, "arming a wait", func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, b.silentClock.Arm(ctx, w, ttl, sent)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (b boundedClock) Held(ctx context.Context) (map[silentWait]time.Time, error) {
|
||||
return boundedCall(ctx, "listing the waits", b.silentClock.Held)
|
||||
}
|
||||
|
||||
func (b boundedClock) Reported(ctx context.Context, w silentWait) error {
|
||||
_, err := boundedCall(ctx, "dropping a wait", func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, b.silentClock.Reported(ctx, w)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
|
||||
|
||||
@@ -3,10 +3,12 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
@@ -159,23 +161,105 @@ func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
||||
return nil, errors.New("the bus is down")
|
||||
}
|
||||
|
||||
// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own
|
||||
// and the error is given back, so the look is marked failed and retried (issue 974, review 1).
|
||||
func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
|
||||
// 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by
|
||||
// nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error
|
||||
// is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4).
|
||||
func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1
|
||||
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
|
||||
err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent)
|
||||
if err == nil {
|
||||
t.Fatal("no error given back")
|
||||
for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{},
|
||||
"no clock": nil} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
noteRefusedSendKeys(nil)
|
||||
t.Cleanup(func() { noteRefusedSendKeys(nil) })
|
||||
err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent)
|
||||
if clock != nil && err == nil {
|
||||
t.Fatal("no error given back")
|
||||
}
|
||||
got := observeRefusedSendKeys()
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("observations %+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused ||
|
||||
!strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") {
|
||||
t.Fatalf("the condition does not name the plan and the machine: %+v", o)
|
||||
}
|
||||
})
|
||||
}
|
||||
silentTimers.mu.Lock()
|
||||
armed := silentTimers.timers[w].Equal(sent)
|
||||
silentTimers.mu.Unlock()
|
||||
if !armed {
|
||||
t.Error("no timer of its own watches the wait")
|
||||
}
|
||||
|
||||
// 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is.
|
||||
func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2
|
||||
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
noteRefusedSendKeys(nil)
|
||||
t.Cleanup(func() { noteRefusedSendKeys(nil) })
|
||||
want := map[silentWait]time.Time{w: sent}
|
||||
if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 {
|
||||
t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys())
|
||||
}
|
||||
clock := &keptClock{}
|
||||
if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !clock.armed[w].Equal(sent) {
|
||||
t.Fatalf("the next look did not write the key again: %v", clock.armed)
|
||||
}
|
||||
if got := observeRefusedSendKeys(); len(got) != 0 {
|
||||
t.Fatalf("the condition stays after the key was written: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type armRefusingClock struct{ silentClock }
|
||||
|
||||
func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
|
||||
func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error {
|
||||
return errors.New("the bus refuses the key")
|
||||
}
|
||||
|
||||
// keptClock is a clock that takes every key.
|
||||
type keptClock struct {
|
||||
silentClock
|
||||
armed map[silentWait]time.Time
|
||||
}
|
||||
|
||||
func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
|
||||
func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error {
|
||||
if c.armed == nil {
|
||||
c.armed = map[silentWait]time.Time{}
|
||||
}
|
||||
c.armed[w] = sent
|
||||
return nil
|
||||
}
|
||||
|
||||
// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
|
||||
// looked at: the controller follows its own plan-moved (issue 974, round 1).
|
||||
|
||||
// A clock whose bus answers nothing, ever: a consumer lost when the broker reloaded its user list, a key listing that
|
||||
// never reaches its end.
|
||||
type hungClock struct{ silentClock }
|
||||
|
||||
func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
||||
select {} // never answers, whatever the context says (the vendored Keys reads its watcher without it)
|
||||
}
|
||||
|
||||
// issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans
|
||||
// and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition.
|
||||
func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) {
|
||||
was := silentCallBound
|
||||
silentCallBound = 50 * time.Millisecond
|
||||
t.Cleanup(func() { silentCallBound = was })
|
||||
w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("a clock that never answered was reported as fine")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the look is still waiting for a clock that never answers")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,9 +12,9 @@ import (
|
||||
)
|
||||
|
||||
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
|
||||
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there
|
||||
// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that
|
||||
// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed.
|
||||
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the
|
||||
// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts
|
||||
// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed.
|
||||
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
|
||||
var found []string
|
||||
@@ -31,11 +31,13 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
return err
|
||||
}
|
||||
text := string(raw)
|
||||
if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
||||
if write.MatchString(text) {
|
||||
found = append(found, path+" writes the plan table")
|
||||
}
|
||||
if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
||||
found = append(found, path+" calls writePlan")
|
||||
// A plan record is written by the bucket's raw create and update, which only the plans' own file calls.
|
||||
if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" &&
|
||||
strings.Contains(text, "chain.RolloutsOf") {
|
||||
found = append(found, path+" reaches the rollouts bucket")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
@@ -51,17 +53,26 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(raw)
|
||||
if n := len(write.FindAllString(text, -1)); n != 1 {
|
||||
t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n)
|
||||
}
|
||||
calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1)
|
||||
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
|
||||
next := strings.Index(text[save:], "\nfunc ")
|
||||
if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next {
|
||||
t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls))
|
||||
if next < 0 {
|
||||
next = len(text) - save
|
||||
}
|
||||
if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") {
|
||||
t.Error("SavePlansMoving does not say each plan it wrote")
|
||||
if !strings.Contains(text[save:save+next], "chainSavePlans(") {
|
||||
t.Error("SavePlansMoving does not write through chainSavePlans")
|
||||
}
|
||||
chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chainText := string(chainRaw)
|
||||
at := strings.Index(chainText, "func (i *Inventory) chainSavePlans")
|
||||
end := strings.Index(chainText[at:], "\nfunc ")
|
||||
if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") {
|
||||
t.Error("chainSavePlans does not say each plan it wrote")
|
||||
}
|
||||
if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 {
|
||||
t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -629,7 +629,7 @@ func heardMachines(d *doctor) map[string]bool {
|
||||
return out
|
||||
}
|
||||
for _, m := range f.machines {
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
||||
if !m.lastHeard.IsZero() && !m.silent(f.now) && !m.asleep() {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,6 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
||||
return e
|
||||
}
|
||||
sources := map[string]func(context.Context) (map[string][]byte, error){
|
||||
recordmirror.Assignments.Name: inv.AssignmentsMirrorToWrite,
|
||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||
}
|
||||
@@ -55,15 +54,11 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
||||
events := &assignmentEvents{}
|
||||
for {
|
||||
if holds() {
|
||||
// The assignments, once they are keys of the bus, are written by the verbs that change them (novox/hq ADR
|
||||
// 0408): this tick keeps the provisioner's plan in their values and says what changed; the mirror of the store
|
||||
// writes them only until the move.
|
||||
onTheBus := keepingAssignments(ctx, inv, announce, epoch, events, failing)
|
||||
// The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick
|
||||
// keeps the provisioner's plan in their values and says what changed. They are no mirror of the store.
|
||||
keepingAssignments(ctx, inv, announce, epoch, events, failing)
|
||||
for _, m := range recordmirror.Mirrors() {
|
||||
if m.Name == recordmirror.Assignments.Name && onTheBus {
|
||||
continue
|
||||
}
|
||||
why := mirrorOnce(ctx, inv, api, m, writers, sources[m.Name], announce, epoch, holds)
|
||||
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
|
||||
if why != failing[m.Name] {
|
||||
if why != "" {
|
||||
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
|
||||
@@ -83,7 +78,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
||||
}
|
||||
|
||||
// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
|
||||
func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
|
||||
func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
|
||||
source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
|
||||
holds func() bool) string {
|
||||
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
@@ -95,9 +90,6 @@ func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.Jet
|
||||
return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
|
||||
}
|
||||
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
|
||||
if m.Name == recordmirror.Assignments.Name {
|
||||
w.Holds = assignmentsHolds(inv, holds)
|
||||
}
|
||||
writers[m.Name] = w
|
||||
}
|
||||
want, err := source(reading)
|
||||
@@ -128,21 +120,10 @@ type assignmentEvents struct {
|
||||
// the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last
|
||||
// look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet.
|
||||
func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64,
|
||||
events *assignmentEvents, failing map[string]string) bool {
|
||||
events *assignmentEvents, failing map[string]string) {
|
||||
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
const name = "assignments"
|
||||
if _, on, err := inv.KeptAssignments(reading); err != nil || !on {
|
||||
if err != nil {
|
||||
// Not known to be off the bus: the store's mirror is not written over what the bus may hold.
|
||||
if failing[name] == "" {
|
||||
failing[name] = err.Error()
|
||||
fmt.Printf("the assignments cannot be read from the bus: %v\n", err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
if _, _, err := inv.KeepAssignmentPlans(reading); err != nil {
|
||||
if failing[name] != err.Error() {
|
||||
failing[name] = err.Error()
|
||||
@@ -152,9 +133,9 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
|
||||
fmt.Printf("the provisioner's plans are kept in the assignments again\n")
|
||||
failing[name] = ""
|
||||
}
|
||||
now, _, err := inv.KeptAssignments(reading)
|
||||
now, err := inv.KeptAssignments(reading)
|
||||
if err != nil {
|
||||
return true
|
||||
return
|
||||
}
|
||||
if events.known != nil {
|
||||
keys := make([]string, 0, len(now)+len(events.known))
|
||||
@@ -194,12 +175,4 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
|
||||
}
|
||||
}
|
||||
events.known = now
|
||||
return true
|
||||
}
|
||||
|
||||
// assignmentsHolds is the assignments mirror's leave to write: the lease's, and the assignments neither being switched to
|
||||
// the bus nor on it. Asked by the writer before every write and purge, so a pass that read its source before the switch
|
||||
// writes nothing after it (ADR 0408 §4).
|
||||
func assignmentsHolds(inv *inventory.Inventory, holds func() bool) func() bool {
|
||||
return func() bool { return holds() && inv.AssignmentsMirrorMayWrite() }
|
||||
}
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/recordmirror"
|
||||
)
|
||||
|
||||
// **A writer of the store's mirror of the assignments, made before the assignments are switched to the bus, writes and
|
||||
// purges nothing after** (ADR 0408 §4, issue 1093): its leave to write is asked before every write, and false once they are
|
||||
// switching or moved.
|
||||
func TestAMirrorWriterMadeBeforeTheSwitchWritesAndPurgesNothingAfter(t *testing.T) {
|
||||
inv := inventory.NewForTest()
|
||||
w := &recordmirror.Writer{Mirror: recordmirror.Assignments, Holds: assignmentsHolds(inv, func() bool { return true })}
|
||||
if !w.Holds() {
|
||||
t.Fatal("the writer may not write before the switch")
|
||||
}
|
||||
inv.BeginSwitchForTest(inventory.KindAssignments)
|
||||
if w.Holds() {
|
||||
t.Fatal("the writer may write while the assignments are switching")
|
||||
}
|
||||
// Refused before it reads the bucket (it has none here) or lays or ends a key.
|
||||
r, err := w.Sync(t.Context(), map[string][]byte{"anchor.notes": []byte(`{}`)})
|
||||
if !errors.Is(err, recordmirror.ErrNotHeld) || r.Written != 0 || r.Removed != 0 {
|
||||
t.Fatalf("the writer wrote after the switch: %+v %v", r, err)
|
||||
}
|
||||
}
|
||||
@@ -1589,7 +1589,9 @@ func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalog
|
||||
var other []string
|
||||
listID := ""
|
||||
for _, m := range plan.Modules {
|
||||
if _, was := sent[m.Module]; !was {
|
||||
// A module no build is recorded for is carried with no commit and names none on its key (issue 1109), so it is "sent"
|
||||
// only by the comparison of the summaries below; a module with a commit that was never sent is a new assignment.
|
||||
if _, was := sent[m.Module]; !was && declared.Builds[m.Module] != "" {
|
||||
other = append(other, m.Module+" newly assigned")
|
||||
}
|
||||
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
|
||||
|
||||
@@ -42,7 +42,7 @@ func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) {
|
||||
return s, heard
|
||||
}
|
||||
|
||||
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) {
|
||||
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { // 1157/AC3: the send keys stay until release 6 (ADR 0433 §5)
|
||||
s, heard := silentOnABus(t)
|
||||
want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
|
||||
if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A plan's send key the bus refuses (novox/hq ADR 0433 §4, issue 1157): the look that could not write it says which, and
|
||||
// a probe of the controller's self-check raises each as a condition naming the plan and the machine. The next look writes
|
||||
// the key again; the look that writes it leaves it out of the set, and the condition clears.
|
||||
const (
|
||||
kindSendKeyRefused = "send-key-refused"
|
||||
sendKeyRefusedID = "DL"
|
||||
)
|
||||
|
||||
var refusedSendKeys = struct {
|
||||
mu sync.Mutex
|
||||
by map[silentWait]error
|
||||
}{}
|
||||
|
||||
// noteRefusedSendKeys sets the keys the newest look could not write, replacing what the look before found.
|
||||
func noteRefusedSendKeys(refused map[silentWait]error) {
|
||||
refusedSendKeys.mu.Lock()
|
||||
defer refusedSendKeys.mu.Unlock()
|
||||
refusedSendKeys.by = refused
|
||||
}
|
||||
|
||||
// observeRefusedSendKeys is one condition per plan and machine whose send key was refused.
|
||||
func observeRefusedSendKeys() []conditions.Observation {
|
||||
refusedSendKeys.mu.Lock()
|
||||
waits := make([]silentWait, 0, len(refusedSendKeys.by))
|
||||
why := map[silentWait]error{}
|
||||
for w, err := range refusedSendKeys.by {
|
||||
waits = append(waits, w)
|
||||
why[w] = err
|
||||
}
|
||||
refusedSendKeys.mu.Unlock()
|
||||
sort.Slice(waits, func(i, j int) bool { return silentKey(waits[i]) < silentKey(waits[j]) })
|
||||
seen := map[[2]string]bool{}
|
||||
var out []conditions.Observation
|
||||
for _, w := range waits {
|
||||
if seen[[2]string{w.Plan, w.Node}] {
|
||||
continue // a plan sends a machine one tier at a time; one condition says it
|
||||
}
|
||||
seen[[2]string{w.Plan, w.Node}] = true
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.Plan, Token: kindSendKeyRefused + "-" + w.Node,
|
||||
Kind: kindSendKeyRefused, Machine: w.Node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the bus refused the key that watches the delivery %s for a report from %s, so nothing "+
|
||||
"names the machine silent if it never reports; the plan writes the key again at its next look", w.Plan, w.Node),
|
||||
Said: fmt.Sprintf("%s: %v", silentKey(w), why[w])})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeSendKeyRefused reads what the newest look of the plans could not write.
|
||||
func probeSendKeyRefused(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
return observeRefusedSendKeys(), nil
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -38,6 +39,13 @@ type sendable struct {
|
||||
// the send; never on the wire.
|
||||
composedFrom int64
|
||||
actingEpoch uint64
|
||||
// ReadsModules says the machine's node-engine reads a module per resource and a build per module (novox/hq issue
|
||||
// 1145): only then are they sent, because an older node-engine refuses a key it does not know, whole.
|
||||
ReadsModules bool
|
||||
// moduleOf is the composer's owner of each resource id, and buildIDs the build each module carries (its id):
|
||||
// kept beside the resources, and on the wire only to a machine that reads them.
|
||||
moduleOf map[string]string
|
||||
buildIDs map[string]string
|
||||
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
||||
modules []string
|
||||
// localNetwork is every port admitted from the local network and what it resolved to (novox/hq ADR
|
||||
@@ -111,6 +119,28 @@ type adoptionEnvelope struct {
|
||||
// Body is the declaration's bytes, as sent and as digested.
|
||||
func (s sendable) Body() ([]byte, error) {
|
||||
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||
if s.ReadsModules {
|
||||
// The module of each resource the composer owns (what the mesh adds of its own has none), and the build of each
|
||||
// module (novox/hq issue 1145): to a machine that said it reads them, and no other.
|
||||
named := make([]map[string]any, len(s.Resources))
|
||||
for n, r := range s.Resources {
|
||||
module, owned := s.moduleOf[fmt.Sprint(r["id"])]
|
||||
if !owned {
|
||||
named[n] = r
|
||||
continue
|
||||
}
|
||||
with := make(map[string]any, len(r)+1)
|
||||
for k, v := range r {
|
||||
with[k] = v
|
||||
}
|
||||
with["module"] = module
|
||||
named[n] = with
|
||||
}
|
||||
envelope["resources"] = named
|
||||
if len(s.buildIDs) > 0 {
|
||||
envelope["builds"] = s.buildIDs
|
||||
}
|
||||
}
|
||||
if s.Adoption != nil {
|
||||
envelope["adoption"] = s.Adoption
|
||||
}
|
||||
|
||||
@@ -85,10 +85,7 @@ func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
mirror, err := inv.AssignmentsMirror(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mirror := inv.PlanValuesForTest(t)
|
||||
var value inventory.AssignmentMirrored
|
||||
if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil {
|
||||
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err)
|
||||
|
||||
@@ -335,7 +335,7 @@ func watchHeartbeats(f *signalFacts) []conditions.Observation {
|
||||
}
|
||||
bound := heartbeatBound(m.every)
|
||||
silent := f.now.Sub(m.lastHeard)
|
||||
if silent <= bound {
|
||||
if !m.silent(f.now) {
|
||||
continue
|
||||
}
|
||||
severity := conditions.Warning
|
||||
@@ -357,7 +357,7 @@ func watchReports(f *signalFacts) []conditions.Observation {
|
||||
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
||||
continue
|
||||
}
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
if m.silent(f.now) {
|
||||
continue // silent: S1 says it, and a silent machine reports nothing
|
||||
}
|
||||
bound := max(reportAtLeast, 3*m.lastApply)
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// What stopped moving (novox/hq ADR 0433 §2, §3, issue 1156). The watchman writes `stuck.<bucket>.<key>` in its own
|
||||
// bucket for each key whose pulse expired, and again as moved when the key moves; it never deletes one. The controller
|
||||
// reads that bucket in two places and holds no timer for either: a probe of the self-check raises each key not moved as
|
||||
// a condition of the delivery scope, and the check that a machine is silent takes a node's presence key as its verdict.
|
||||
const (
|
||||
kindStuck = "stuck"
|
||||
probeStuckID = "DM"
|
||||
)
|
||||
|
||||
// presenceVerdict is the watchman's word on whether a machine is online, from its stuck presence key.
|
||||
type presenceVerdict int
|
||||
|
||||
const (
|
||||
// presenceUnsaid: the watchman has written nothing for the machine; the heartbeat rule judges it, as before.
|
||||
presenceUnsaid presenceVerdict = iota
|
||||
// presenceHeard: the key says moved, the machine is back online.
|
||||
presenceHeard
|
||||
// presenceSilent: the key says not moved, the machine was offline past its bound.
|
||||
presenceSilent
|
||||
)
|
||||
|
||||
// presenceEntry is presence.<node> as the watchman wrote it: whether the machine is online and when the entry was made.
|
||||
// The watchman writes it only when it changes, so for an online machine the time is when it connected.
|
||||
type presenceEntry struct {
|
||||
Online bool
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// applyPresence sets what the watchman says of a machine: the verdict of its stuck presence key (nil when there is none).
|
||||
// A presence entry that says online counts the machine as heard now; otherwise last heard is the later of the store's
|
||||
// last_seen and the entry's time (the time it went offline).
|
||||
func applyPresence(m *machineFacts, stuck *sdkdelivery.Stuck, entry *presenceEntry, now time.Time) {
|
||||
switch {
|
||||
case stuck == nil:
|
||||
m.presence = presenceUnsaid
|
||||
case stuck.Moved:
|
||||
m.presence = presenceHeard
|
||||
default:
|
||||
m.presence = presenceSilent
|
||||
}
|
||||
switch {
|
||||
case entry == nil:
|
||||
case entry.Online:
|
||||
m.lastHeard = now
|
||||
case entry.At.After(m.lastHeard):
|
||||
m.lastHeard = entry.At
|
||||
}
|
||||
}
|
||||
|
||||
// silent says the machine is silent: the watchman's verdict where it gave one, else the heartbeat rule. A machine never
|
||||
// heard from is not silent, it has not joined.
|
||||
func (m machineFacts) silent(now time.Time) bool {
|
||||
switch m.presence {
|
||||
case presenceSilent:
|
||||
return true
|
||||
case presenceHeard:
|
||||
return false
|
||||
}
|
||||
return !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every)
|
||||
}
|
||||
|
||||
// stuckObservations are the conditions of what stopped moving: one for each key not moved, in the delivery scope, kind
|
||||
// stuck, naming the key and what it waited for. A node's presence key is left to the silent check; a moved key raises
|
||||
// nothing, so its condition clears.
|
||||
func stuckObservations(entries map[string]sdkdelivery.Stuck) []conditions.Observation {
|
||||
keys := make([]string, 0, len(entries))
|
||||
for k := range entries {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var out []conditions.Observation
|
||||
for _, k := range keys {
|
||||
s := entries[k]
|
||||
if s.Moved {
|
||||
continue
|
||||
}
|
||||
if s.Bucket == sdkdelivery.PresenceBucket {
|
||||
if node, _, _, err := sdkdelivery.ParsePresenceKey(s.Key); err == nil && node != "" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
waited := s.Waited
|
||||
if waited == "" {
|
||||
waited = "a move"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: k, Kind: kindStuck,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s has not moved since %s: it waited for %s", k, s.Since.UTC().Format("2006-01-02 15:04 MST"),
|
||||
waited),
|
||||
Said: fmt.Sprintf("%s: waiting for %s since %s", k, waited, s.Since.UTC().Format(time.RFC3339))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// readStuck is every key of the stuck bucket and its value; none when the bucket is not there yet (the watchman not
|
||||
// delivered). A value of another version is left out, said.
|
||||
func readStuck(ctx context.Context, conn *nats.Conn) (map[string]sdkdelivery.Stuck, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watchman's stuck bucket: %w", err)
|
||||
}
|
||||
keys, err := boundedCall(ctx, "listing the stuck keys", func(ctx context.Context) ([]string, error) { return kv.Keys(ctx) })
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]sdkdelivery.Stuck{}
|
||||
for _, key := range keys {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s, err := sdkdelivery.DecodeStuck(e.Value())
|
||||
if err != nil {
|
||||
fmt.Printf("the stuck key %s is not read: %v\n", key, err)
|
||||
continue
|
||||
}
|
||||
out[key] = s
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// readPresence is, by node, presence.<node>'s entry; a node without an entry is left out.
|
||||
func readPresence(ctx context.Context, conn *nats.Conn, nodes []string) (map[string]presenceEntry, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.PresenceBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watchman's presence bucket: %w", err)
|
||||
}
|
||||
out := map[string]presenceEntry{}
|
||||
for _, node := range nodes {
|
||||
key, err := sdkdelivery.PresenceNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := sdkdelivery.DecodePresence(e.Value())
|
||||
if err != nil {
|
||||
fmt.Printf("the presence key %s is not read: %v\n", key, err)
|
||||
continue
|
||||
}
|
||||
out[node] = presenceEntry{Online: p.Online, At: e.Created()}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// withPresence says of each machine what the watchman says: its stuck presence key and its presence entry's time.
|
||||
func withPresence(ctx context.Context, conn *nats.Conn, machines []machineFacts, now time.Time) error {
|
||||
stuck, err := readStuck(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, len(machines))
|
||||
for i, m := range machines {
|
||||
names[i] = m.name
|
||||
}
|
||||
entries, err := readPresence(ctx, conn, names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range machines {
|
||||
var said *sdkdelivery.Stuck
|
||||
if key, err := sdkdelivery.StuckKey(sdkdelivery.PresenceBucket, machines[i].name); err == nil {
|
||||
if s, ok := stuck[key]; ok {
|
||||
said = &s
|
||||
}
|
||||
}
|
||||
var entry *presenceEntry
|
||||
if e, ok := entries[machines[i].name]; ok {
|
||||
entry = &e
|
||||
}
|
||||
applyPresence(&machines[i], said, entry, now)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeStuck is DM: every key the watchman says stopped moving is raised, but a node's presence, which the silent check
|
||||
// takes.
|
||||
func probeStuck(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, nil
|
||||
}
|
||||
entries, err := readStuck(ctx, d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stuckObservations(entries), nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1156/AC4: on the embedded bus with the watchman's buckets, a stuck key written raises a condition and written as moved
|
||||
// clears it; a node's presence key not moved says the machine silent through the facts, and moved says it heard.
|
||||
func TestAStuckKeyWrittenAndThenMovedRaisesAndClearsACondition(t *testing.T) { // 1156/AC4
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if _, err := broker.RaiseBuckets(js, []broker.Bucket{
|
||||
{Module: sdkdelivery.WatchmanModule, Name: "stuck"},
|
||||
{Module: sdkdelivery.WatchmanModule, Name: "presence", History: sdkdelivery.PresenceHistory},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write := func(bucket, key string, moved bool) string {
|
||||
t.Helper()
|
||||
stuckKey, err := sdkdelivery.StuckKey(bucket, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := sdkdelivery.EncodeStuck(sdkdelivery.Stuck{Bucket: bucket, Key: key, Waited: "an answer", Since: time.Now().UTC(), Moved: moved})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, stuckKey, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return stuckKey
|
||||
}
|
||||
d := &doctor{js: js}
|
||||
stuckKey := write(sdkdelivery.PulseBucket, "builds.x", false)
|
||||
write(sdkdelivery.PresenceBucket, "ace", false)
|
||||
got, err := probeStuck(ctx, d)
|
||||
if err != nil || len(got) != 1 || got[0].ID != stuckKey || got[0].Kind != kindStuck {
|
||||
t.Fatalf("a stuck key written: %+v %v", got, err)
|
||||
}
|
||||
machines := []machineFacts{{name: "ace", lastHeard: time.Now()}}
|
||||
if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !machines[0].silent(time.Now()) {
|
||||
t.Fatal("a presence key not moved did not say the machine silent")
|
||||
}
|
||||
write(sdkdelivery.PulseBucket, "builds.x", true)
|
||||
write(sdkdelivery.PresenceBucket, "ace", true)
|
||||
if got, err := probeStuck(ctx, d); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a stuck key written as moved: %+v %v", got, err)
|
||||
}
|
||||
machines = []machineFacts{{name: "ace", lastHeard: time.Now().Add(-time.Hour), every: time.Minute}}
|
||||
if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if machines[0].silent(time.Now()) {
|
||||
t.Fatal("a presence key that moved still says the machine silent")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func aStuckKey(t *testing.T, bucket, key string) string {
|
||||
t.Helper()
|
||||
k, err := sdkdelivery.StuckKey(bucket, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// 1156/AC1: each not-moved stuck key that is not a node's presence is an observation of the delivery scope, kind stuck,
|
||||
// naming the key and what it waited for; a moved key raises nothing, so its condition clears.
|
||||
func TestEachStuckKeyThatHasNotMovedIsAConditionOfTheDeliveryScope(t *testing.T) { // 1156/AC1
|
||||
since := time.Date(2026, 10, 18, 19, 0, 0, 0, time.UTC)
|
||||
pulse := aStuckKey(t, sdkdelivery.PulseBucket, "x")
|
||||
moved := aStuckKey(t, sdkdelivery.HaveBucket, "ace.zsh")
|
||||
node := aStuckKey(t, sdkdelivery.PresenceBucket, "ace")
|
||||
part := aStuckKey(t, sdkdelivery.PresenceBucket, "part.mesh-controller.one")
|
||||
entries := map[string]sdkdelivery.Stuck{
|
||||
pulse: {Bucket: sdkdelivery.PulseBucket, Key: "x", Waited: "the build of zsh to be answered", Since: since},
|
||||
moved: {Bucket: sdkdelivery.HaveBucket, Key: "ace.zsh", Waited: "a want", Since: since, Moved: true},
|
||||
node: {Bucket: sdkdelivery.PresenceBucket, Key: "ace", Waited: "its return", Since: since},
|
||||
part: {Bucket: sdkdelivery.PresenceBucket, Key: "part.mesh-controller.one", Waited: "its return", Since: since},
|
||||
}
|
||||
got := stuckObservations(entries)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("observations %+v, want the pulse key and the part's presence", got)
|
||||
}
|
||||
byID := map[string]conditions.Observation{}
|
||||
for _, o := range got {
|
||||
if o.Scope != conditions.ScopeDelivery || o.Kind != kindStuck {
|
||||
t.Errorf("%+v is not a stuck condition of the delivery scope", o)
|
||||
}
|
||||
byID[o.ID] = o
|
||||
}
|
||||
o, ok := byID[pulse]
|
||||
if !ok || !strings.Contains(o.Summary, pulse) || !strings.Contains(o.Summary, "the build of zsh to be answered") {
|
||||
t.Fatalf("the condition does not name the key and what it waited for: %+v", got)
|
||||
}
|
||||
if _, ok := byID[part]; !ok {
|
||||
t.Errorf("a part's presence is left out: %v", byID)
|
||||
}
|
||||
// Moved again: the same key, written as moved, raises nothing.
|
||||
entries[pulse] = sdkdelivery.Stuck{Bucket: sdkdelivery.PulseBucket, Key: "x", Since: since, Moved: true}
|
||||
if got := stuckObservations(entries); len(got) != 1 {
|
||||
t.Fatalf("a moved key still raises: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func aMachineHeardAt(now time.Time, ago time.Duration) machineFacts {
|
||||
return machineFacts{name: "laptop", lastHeard: now.Add(-ago), every: time.Minute}
|
||||
}
|
||||
|
||||
// 1156/AC2: a node's stuck presence key not moved makes the machine silent, unless it sleeps; moved, it is heard; no
|
||||
// timer is armed (the verdict is the key's, whatever the time of the last word).
|
||||
func TestAStuckPresenceKeyIsTheSilentVerdict(t *testing.T) { // 1156/AC2
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
f.machines = append(f.machines, aMachineHeardAt(now, 10*time.Second))
|
||||
notMoved := sdkdelivery.Stuck{Bucket: sdkdelivery.PresenceBucket, Key: "laptop", Waited: "its return", Since: now.Add(-3 * time.Minute)}
|
||||
applyPresence(&f.machines[1], ¬Moved, nil, now)
|
||||
got := watchHeartbeats(f)
|
||||
if len(got) != 1 || got[0].Machine != "laptop" || got[0].Kind != "silent" {
|
||||
t.Fatalf("a presence key not moved: %+v", got)
|
||||
}
|
||||
// Asleep: not silent.
|
||||
f.machines[1].power = link.PowerState{State: "sleeping", At: now.Add(-time.Hour)}
|
||||
if got := watchHeartbeats(f); len(got) != 0 {
|
||||
t.Fatalf("a sleeping machine raised %+v", got)
|
||||
}
|
||||
// Moved: heard, whatever the store's last word says.
|
||||
f.machines[1].power = link.PowerState{}
|
||||
f.machines[1].lastHeard = now.Add(-2 * time.Hour)
|
||||
moved := notMoved
|
||||
moved.Moved = true
|
||||
applyPresence(&f.machines[1], &moved, nil, now)
|
||||
if got := watchHeartbeats(f); len(got) != 0 {
|
||||
t.Fatalf("a presence key that moved raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1156/AC3: with a presence entry that says offline, last heard is the later of last_seen and the entry's time; with none,
|
||||
// last_seen. An entry that says online counts as heard now: the watchman writes it only when it changes, so its time is
|
||||
// when the machine connected, and a machine online for hours with no stuck key is not silent, whatever last_seen says.
|
||||
func TestLastHeardFollowsThePresenceEntry(t *testing.T) { // 1156/AC3
|
||||
now := time.Now()
|
||||
seen := now.Add(-time.Hour)
|
||||
m := machineFacts{name: "laptop", lastHeard: seen, every: time.Minute}
|
||||
applyPresence(&m, nil, nil, now)
|
||||
if !m.lastHeard.Equal(seen) {
|
||||
t.Fatalf("with no presence entry last heard is %v, want last_seen %v", m.lastHeard, seen)
|
||||
}
|
||||
offlineAt := now.Add(-5 * time.Minute)
|
||||
applyPresence(&m, nil, &presenceEntry{Online: false, At: offlineAt}, now)
|
||||
if !m.lastHeard.Equal(offlineAt) {
|
||||
t.Fatalf("offline since %v: last heard is %v", offlineAt, m.lastHeard)
|
||||
}
|
||||
m = machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute), every: time.Minute}
|
||||
applyPresence(&m, nil, &presenceEntry{Online: false, At: now.Add(-time.Hour)}, now)
|
||||
if !m.lastHeard.Equal(now.Add(-time.Minute)) {
|
||||
t.Fatalf("an older entry replaced a newer last_seen: %v", m.lastHeard)
|
||||
}
|
||||
for _, lastSeen := range []time.Duration{10 * time.Second, 5 * time.Hour} {
|
||||
m := machineFacts{name: "laptop", lastHeard: now.Add(-lastSeen), every: time.Minute}
|
||||
applyPresence(&m, nil, &presenceEntry{Online: true, At: now.Add(-6 * time.Hour)}, now)
|
||||
if m.silent(now) {
|
||||
t.Fatalf("a machine online since hours ago (last_seen %v ago) with no stuck key is silent", lastSeen)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -155,6 +155,8 @@ type machineFacts struct {
|
||||
tools bool
|
||||
toolsHeard time.Time
|
||||
toolsEvery time.Duration
|
||||
// presence is the watchman's word on whether it is online (novox/hq ADR 0433 §3); unsaid, the heartbeat rule judges.
|
||||
presence presenceVerdict
|
||||
}
|
||||
|
||||
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||
@@ -470,7 +472,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
m.reportedCurrent = r.Current
|
||||
}
|
||||
if !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
if m.silent(now) {
|
||||
anyPast = true
|
||||
}
|
||||
if m.tools && now.Sub(later(m.toolsHeard, link.ToolsBeats.Started())) > heartbeatBound(m.toolsEvery) {
|
||||
@@ -478,6 +480,19 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
// What the watchman says of each machine: its stuck presence key is the silent verdict, and presence.<node>'s time when
|
||||
// it was last heard (ADR 0433 §3). Unreadable is said, and the heartbeat rule judges as before.
|
||||
if w.js != nil {
|
||||
if err := withPresence(ctx, w.js.Conn(), out, now); err != nil {
|
||||
fmt.Printf("what the watchman says of the machines' presence cannot be read: %v\n", err)
|
||||
} else {
|
||||
for _, m := range out {
|
||||
if m.silent(now) {
|
||||
anyPast = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// What a machine past its bound last said of its power, read only then: a machine that said it
|
||||
// is asleep is not lost (ADR 0211). Unreadable is said: a sleeping laptop is then called silent,
|
||||
// which is the louder mistake and the right one to make.
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
@@ -36,4 +36,4 @@ require (
|
||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||
// `go mod vendor` fails loudly, at once, everywhere.
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261017162006-6469e8238443
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261017162006-6469e8238443 h1:zUtpGbiH9DLjCPY+KJsm+PZ25yW6k/NmGVS3haGFfMM=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261017162006-6469e8238443/go.mod h1:5QOm3cIfu/k9fN6QqgDbKGCcDst7Ul1UlcfsiMVm9j4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4 h1:yYDxFRJoH59kiCoEulD+FBwctyf9CrtKIxlEAsVL39Q=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1155/AC1: a module whose manifest reads mesh-controller.checks-running may watch and get every key of the bucket and
|
||||
// write none.
|
||||
func TestAModuleReadingChecksRunningReadsAndWritesNone(t *testing.T) { // 1155/AC1
|
||||
if !slices.Contains(ControllerMirrors(), "checks-running") || BucketName(ControllerSeat, "checks-running") != CheckRunningBucket {
|
||||
t.Fatalf("a module cannot name %s in `reads`: %v", CheckRunningBucket, ControllerMirrors())
|
||||
}
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-watchman", PasswordHash: "x",
|
||||
Reads: []string{"mesh-controller.checks-running"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "KV_" + CheckRunningBucket
|
||||
has(t, perms.Publish, "$JS.API.STREAM.INFO."+stream)
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET."+stream+".>")
|
||||
has(t, perms.Publish, "$JS.API.CONSUMER.CREATE."+stream+".>")
|
||||
for _, p := range perms.Publish {
|
||||
if SubjectsOverlap(p, "$KV."+CheckRunningBucket+".any.key") {
|
||||
t.Errorf("the reading module may write %s through %q", CheckRunningBucket, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1155/AC3: the writers table names checks-running's writer as the build seat's holders, and the rule that a mirror is
|
||||
// written by the controller alone names it as the exception, with its reason.
|
||||
func TestChecksRunningIsAMirrorWrittenByTheBuildSeatsHolders(t *testing.T) { // 1155/AC3
|
||||
why := MirrorsWrittenByAnother["checks-running"]
|
||||
if why == "" {
|
||||
t.Fatal("the mirrors written by another do not name checks-running with a reason")
|
||||
}
|
||||
subject := "$KV." + CheckRunningBucket + ".novox.a.7"
|
||||
var rows int
|
||||
for _, row := range WritersTable {
|
||||
for _, s := range row.Subjects {
|
||||
if s != "$KV."+CheckRunningBucket+".>" {
|
||||
continue
|
||||
}
|
||||
rows++
|
||||
holder := Principal{Kind: KindModule, Module: "builder", Node: "one", Holds: []Seat{{Name: TheBuildSeat}}}
|
||||
other := Principal{Kind: KindModule, Module: "mesh-delivery-watchman", Node: "one"}
|
||||
if row.Writes == nil || !row.Writes(holder, subject) || row.Writes(other, subject) {
|
||||
t.Errorf("%q is not written by the build seat's holders alone", row.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("%d rows of the writers table write %s, want one", rows, CheckRunningBucket)
|
||||
}
|
||||
for m := range MirrorsWrittenByAnother {
|
||||
if m != "checks-running" {
|
||||
t.Errorf("the mirror %q is written by another and is not checks-running", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
@@ -55,7 +56,7 @@ var (
|
||||
// check ends; a key not renewed within the bucket's age is a builder that died, and is gone. A builder that takes a
|
||||
// waiting request for a head and trunk commit a key names drops it. **Written by the build seat's holders, several by
|
||||
// design** — each the key of the check it runs.
|
||||
CheckRunningBucket = BucketName(ControllerSeat, "checks-running")
|
||||
CheckRunningBucket = BucketName(ControllerSeat, ChecksRunning)
|
||||
// DeliveryMergesBucket holds the merges mesh-delivery announced it made, each with the ask whose Approve it merged
|
||||
// on (novox/hq ADR 0420 §7, issue 1025): per merge a key `<owner>/<repo>/<number>/<head sha>` holding the
|
||||
// announcement, kept DeliveryMergesKeptFor. What lets the build of a moved module register it from its new
|
||||
@@ -132,9 +133,22 @@ func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, Ass
|
||||
// ControllerMirrors are the local names of the controller's buckets a module may read: the record mirrors and the
|
||||
// chain's buckets.
|
||||
func ControllerMirrors() []string {
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts}
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, ChecksRunning}
|
||||
}
|
||||
|
||||
// MirrorsWrittenByAnother are the controller's mirrors whose writer is not the controller, each with its reason: the rule
|
||||
// that a mirror is written by the controller alone (TestTheMirrorsListAndTheWritersTableAgree) has these exceptions and no
|
||||
// other.
|
||||
var MirrorsWrittenByAnother = map[string]string{
|
||||
ChecksRunning: "the checks running are said by the builders themselves: every machine holding the build seat puts the " +
|
||||
"key of the check it runs, at its start and at each heartbeat, and deletes it when the check ends, so the " +
|
||||
"controller asserts the bucket and writes none of its keys (ADR 0406 §3); it is readable through `reads` so " +
|
||||
"the watchman sees a check that stopped (ADR 0433 §6)",
|
||||
}
|
||||
|
||||
// ChecksRunning is the local name of the bucket of the checks running, as a reader names it in `reads`.
|
||||
const ChecksRunning = "checks-running"
|
||||
|
||||
// The mirrors' caps (design 54 §5): one value per key, no age, 64 KiB a value, 64 MiB a bucket.
|
||||
const (
|
||||
MirrorValueBytes = 64 << 10
|
||||
@@ -189,14 +203,15 @@ func IsControllerBucket(bucket string) bool {
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
|
||||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
|
||||
bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket
|
||||
bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket ||
|
||||
bucket == delivery.HaveBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
|
||||
DeliveryMergesBucket, BuildsBucket, RolloutsBucket}
|
||||
DeliveryMergesBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
@@ -370,6 +385,12 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
return fmt.Errorf("asserting bucket %s: %w", c.bucket, err)
|
||||
}
|
||||
}
|
||||
// **The have bucket: what each machine has of each module** (ADR 0429 §1, issue 1136). Twenty revisions of each key and
|
||||
// no age. Asserted by the controller, written by each machine's node-engine under its own node's keys and by nobody
|
||||
// else: the controller's grant leaves it out.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, HaveBucketConfig()); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", delivery.HaveBucket, err)
|
||||
}
|
||||
// **The assignments keep the marker age they had as a mirror.** A stream that allows messages an age cannot be
|
||||
// made to disallow them (the server refuses: "message TTL status can not be disabled"), so the bucket that was
|
||||
// the assignments mirror is asserted with the age it was made with. Nothing is purged from it, so no marker is ever
|
||||
@@ -439,3 +460,18 @@ func EnsureLeaseBucket(ctx context.Context, js jetstream.JetStream) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HaveBucketConfig is the options of the have bucket (novox/hq ADR 0429 §1): delivery.HaveHistory revisions of each
|
||||
// key, no age, a value under a kilobyte (a have is JSON of about 500 bytes), 64 MiB on disk like the assignments.
|
||||
func HaveBucketConfig() jetstream.KeyValueConfig {
|
||||
return jetstream.KeyValueConfig{
|
||||
Bucket: delivery.HaveBucket,
|
||||
Description: "what each machine has of each module, one key `<node>.<module>` (and `<node>.node` for the machine's " +
|
||||
"own facts) (novox/hq ADR 0429, design 47): written by each machine's node-engine under its own node's keys " +
|
||||
"alone, never by the controller; read by the delivery view and the watchman",
|
||||
History: delivery.HaveHistory,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 64 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,6 +135,12 @@ func TestTheMirrorsListAndTheWritersTableAgree(t *testing.T) {
|
||||
for _, m := range ControllerMirrors() {
|
||||
bucket := BucketName(ControllerSeat, m)
|
||||
listed[bucket] = true
|
||||
if why, other := MirrorsWrittenByAnother[m]; other { // 1155/AC3: the exception, with its reason
|
||||
if why == "" {
|
||||
t.Errorf("the mirror %s is written by another and says no reason", bucket)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !rowed[bucket] {
|
||||
t.Errorf("the mirror %s is listed and no row of the writers table writes it", bucket)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
//go:build integration
|
||||
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// **Each node's credential writes its own have keys and no other** (novox/hq ADR 0429 §1, issue 1136, 1136/AC4): on a
|
||||
// real server whose users are composed as the controller composes them, node alpha's put of alpha.zsh is taken, its
|
||||
// put of beta.zsh is refused by the server, a module reading node-engine.have is refused any put and its watch sees
|
||||
// alpha.zsh.
|
||||
func TestNatsANodePutsOnlyItsOwnHaveKeys(t *testing.T) { // 1136/AC4
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
users, err := Users(Records{Nodes: []string{"alpha", "beta"}, Assigned: map[string][]Declared{
|
||||
"alpha": {{Module: RuntimeModule}, {Module: "mesh-delivery-view", Reads: []string{"node-engine.have"}}},
|
||||
"beta": {{Module: RuntimeModule}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range users {
|
||||
users[i].PasswordHash = string(hash)
|
||||
}
|
||||
accounts, err := ComposeAccounts(users)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
connect := func(user string) (*nats.Conn, chan error) {
|
||||
t.Helper()
|
||||
refused := make(chan error, 8)
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
|
||||
if err != nil {
|
||||
t.Fatalf("%s cannot connect: %v", user, err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
return nc, refused
|
||||
}
|
||||
controller, _ := connect(ControllerName)
|
||||
cjs, err := jetstream.New(controller)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := cjs.CreateOrUpdateKeyValue(ctx, HaveBucketConfig()); err != nil {
|
||||
t.Fatalf("the controller cannot make %s: %v", delivery.HaveBucket, err)
|
||||
}
|
||||
|
||||
alpha, _ := connect("node.alpha")
|
||||
ajs, err := jetstream.New(alpha)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
akv, err := ajs.KeyValue(ctx, delivery.HaveBucket)
|
||||
if err != nil {
|
||||
t.Fatalf("node alpha cannot bind %s: %v", delivery.HaveBucket, err)
|
||||
}
|
||||
if _, err := akv.Put(ctx, "alpha.zsh", []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatalf("node alpha cannot put alpha.zsh: %v", err)
|
||||
}
|
||||
if e, err := akv.Get(ctx, "alpha.zsh"); err != nil || string(e.Value()) != `{"v":1}` {
|
||||
t.Errorf("node alpha reads %v (%v) of its own key", e, err)
|
||||
}
|
||||
|
||||
reader, readerRefused := connect("alpha.mesh-delivery-view")
|
||||
for _, other := range []struct {
|
||||
name string
|
||||
nc *nats.Conn
|
||||
ch chan error
|
||||
key string
|
||||
}{{"node alpha putting beta.zsh", alpha, nil, "beta.zsh"},
|
||||
{"the reading module putting alpha.zsh", reader, readerRefused, "alpha.zsh"}} {
|
||||
ch := other.ch
|
||||
if ch == nil {
|
||||
// alpha's own refusals arrive on its own handler: reconnect with one that is watched.
|
||||
var nc *nats.Conn
|
||||
nc, ch = connect("node.alpha")
|
||||
other.nc = nc
|
||||
}
|
||||
if err := other.nc.Publish("$KV."+delivery.HaveBucket+"."+other.key, []byte(`{"v":99}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = other.nc.Flush()
|
||||
select {
|
||||
case err := <-ch:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("%s: the server said %v, want a permissions violation", other.name, err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Errorf("the server did not refuse %s", other.name)
|
||||
}
|
||||
}
|
||||
|
||||
// The reader watches and sees alpha.zsh, and beta.zsh was never taken.
|
||||
rjs, err := jetstream.New(reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rkv, err := rjs.KeyValue(ctx, delivery.HaveBucket)
|
||||
if err != nil {
|
||||
t.Fatalf("a module reading node-engine.have cannot bind it: %v", err)
|
||||
}
|
||||
w, err := rkv.WatchAll(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("the reading module cannot watch: %v", err)
|
||||
}
|
||||
defer w.Stop()
|
||||
seen := map[string]string{}
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
break
|
||||
}
|
||||
seen[e.Key()] = string(e.Value())
|
||||
}
|
||||
if seen["alpha.zsh"] != `{"v":1}` || len(seen) != 1 {
|
||||
t.Errorf("the reading module's watch sees %v, want alpha.zsh alone", seen)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// 1136/AC1: the controller's start asserts node-engine_have with the history delivery keeps, and the bucket is one of
|
||||
// the controller's own, so raising the buckets does not name it undeclared (ADR 0429 §1).
|
||||
func TestTheHaveBucketIsTheControllersAndKeepsTwentyRevisions(t *testing.T) { // 1136/AC1
|
||||
cfg := HaveBucketConfig()
|
||||
if cfg.Bucket != delivery.HaveBucket || cfg.History != delivery.HaveHistory || delivery.HaveHistory != 20 {
|
||||
t.Fatalf("the have bucket is asserted as %q with history %d, want %q with 20", cfg.Bucket, cfg.History, delivery.HaveBucket)
|
||||
}
|
||||
if cfg.Storage != jetstream.FileStorage || cfg.MaxBytes <= 0 || cfg.MaxValueSize <= 0 {
|
||||
t.Fatalf("the have bucket is not bounded on disk: %+v", cfg)
|
||||
}
|
||||
if !IsControllerBucket(delivery.HaveBucket) || !slices.Contains(ControllerBuckets(), delivery.HaveBucket) {
|
||||
t.Fatalf("%s is not among the controller's buckets", delivery.HaveBucket)
|
||||
}
|
||||
undeclared, err := RaiseBuckets(fakeBuckets{names: []string{delivery.HaveBucket}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(undeclared) != 0 {
|
||||
t.Fatalf("%v are raised as undeclared", undeclared)
|
||||
}
|
||||
}
|
||||
|
||||
// 1136/AC2: a node may write and read the keys of its own node of the have bucket, with what a get, create, update and
|
||||
// watch of them need, and no other key of it; the controller does not write it; an unsafe node name is refused.
|
||||
func TestANodeWritesAndReadsOnlyItsOwnHaveKeys(t *testing.T) { // 1136/AC2
|
||||
perms, err := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, stream := delivery.HaveBucket, "KV_"+delivery.HaveBucket
|
||||
for _, want := range []string{
|
||||
"$KV." + b + ".one.>",
|
||||
"$JS.API.STREAM.INFO." + stream,
|
||||
"$JS.API.DIRECT.GET." + stream + ".$KV." + b + ".one.>",
|
||||
"$JS.API.CONSUMER.CREATE." + stream + ".*.$KV." + b + ".one.>",
|
||||
"$JS.API.CONSUMER.DELETE." + stream + ".>",
|
||||
"$JS.FC." + stream + ".>",
|
||||
} {
|
||||
has(t, perms.Publish, want)
|
||||
}
|
||||
for _, p := range perms.Publish {
|
||||
if !strings.Contains(p, delivery.HaveBucket) {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(p, "KV."+b+".>") || strings.Contains(p, "KV."+b+".two") || strings.HasSuffix(p, "."+b+".*") ||
|
||||
strings.Contains(p, "DIRECT.GET."+stream+".>") || strings.Contains(p, "CONSUMER.CREATE."+stream+".>") {
|
||||
t.Errorf("the node is granted %q, which reaches keys of other nodes", p)
|
||||
}
|
||||
}
|
||||
if err := CheckWriters(Principal{Kind: KindNode, Node: "one"}, perms.Publish); err != nil {
|
||||
t.Fatalf("the node's own have keys are refused as a second writer: %v", err)
|
||||
}
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range controller.Publish {
|
||||
if SubjectsOverlap(p, "$KV."+b+".one.zsh") {
|
||||
t.Errorf("the controller may write the have keys through %q", p)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"on.e", "one.>", "*", "o>ne"} {
|
||||
if _, err := PermissionsFor(Principal{Kind: KindNode, Node: bad, PasswordHash: "x"}); err == nil {
|
||||
t.Errorf("node %q was accepted as part of a have subject", bad)
|
||||
}
|
||||
}
|
||||
// A principal that is not a node writes no have key: the writers table refuses it.
|
||||
if err := CheckWriters(Principal{Kind: KindModule, Module: "billing", Node: "one"}, []string{"$KV." + b + ".one.>"}); err == nil {
|
||||
t.Error("a module granted the have keys of a node is not refused as a second writer")
|
||||
}
|
||||
if err := CheckWriters(Principal{Kind: KindNode, Node: "two"}, []string{"$KV." + b + ".one.>"}); err == nil {
|
||||
t.Error("a node granted another node's have keys is not refused as a second writer")
|
||||
}
|
||||
}
|
||||
|
||||
// 1136/AC3: a module whose manifest reads node-engine.have, mesh-controller.assignments and mesh-controller.builds may
|
||||
// watch and get every key of the three and write none.
|
||||
func TestAModuleReadingHaveAssignmentsAndBuildsReadsAndWritesNone(t *testing.T) { // 1136/AC3
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-view", PasswordHash: "x",
|
||||
Reads: []string{"node-engine.have", "mesh-controller.assignments", "mesh-controller.builds"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, b := range []string{delivery.HaveBucket, AssignmentsBucket, BuildsBucket} {
|
||||
stream := "KV_" + b
|
||||
has(t, perms.Publish, "$JS.API.STREAM.INFO."+stream)
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET."+stream+".>")
|
||||
has(t, perms.Publish, "$JS.API.CONSUMER.CREATE."+stream+".>")
|
||||
for _, p := range perms.Publish {
|
||||
if SubjectsOverlap(p, "$KV."+b+".any.key") {
|
||||
t.Errorf("the reading module may write %s through %q", b, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -436,30 +436,50 @@ func retentionOf(r Retention) nats.RetentionPolicy {
|
||||
// discards what the bucket holds, and what a module's state holds is data. The mesh's caps are
|
||||
// asserted with the owner's options, so a bucket made by hand converges to them.
|
||||
func (j *JetStream) EnsureBucket(b Bucket) error {
|
||||
history := b.History
|
||||
if history == 0 {
|
||||
history = 1
|
||||
}
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: b.Bucket(),
|
||||
Description: b.Why(),
|
||||
History: uint8(history),
|
||||
TTL: time.Duration(b.TTLSeconds) * time.Second,
|
||||
MaxValueSize: StateMaxValueBytes,
|
||||
MaxBytes: StateMaxBytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
// A stream that allows a time to live per message cannot be made to disallow it (the server refuses the update),
|
||||
// so a bucket that has it keeps it whatever the owner now declares (novox/hq ADR 0433 §1).
|
||||
var held time.Duration
|
||||
if kv, err := js.KeyValue(ctx, b.Bucket()); err == nil {
|
||||
if st, err := kv.Status(ctx); err == nil {
|
||||
held = st.LimitMarkerTTL()
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, BucketConfig(b, held)); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", b.Bucket(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BucketConfig is the options a module's bucket is asserted with. held is the marker age the bucket already has, zero
|
||||
// when it is absent or has none: **a bucket that allows a time to live per message keeps it** (the server refuses to
|
||||
// turn it off), so without key-ttl it is asserted with the age it holds, and with key-ttl with KeyTTLMarkerTTL.
|
||||
func BucketConfig(b Bucket, held time.Duration) jetstream.KeyValueConfig {
|
||||
history := b.History
|
||||
if history == 0 {
|
||||
history = 1
|
||||
}
|
||||
markers := held
|
||||
if b.KeyTTL {
|
||||
markers = KeyTTLMarkerTTL
|
||||
}
|
||||
return jetstream.KeyValueConfig{
|
||||
Bucket: b.Bucket(),
|
||||
Description: b.Why(),
|
||||
History: uint8(history),
|
||||
TTL: time.Duration(b.TTLSeconds) * time.Second,
|
||||
MaxValueSize: StateMaxValueBytes,
|
||||
MaxBytes: StateMaxBytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
LimitMarkerTTL: markers,
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNames is every key-value bucket on the server, the mesh's and anybody else's.
|
||||
func (j *JetStream) BucketNames() ([]string, error) {
|
||||
js, err := jetstream.New(j.conn)
|
||||
|
||||
+55
-2
@@ -21,6 +21,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
@@ -712,6 +713,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// hand, which it alone writes. A put is a publish to the bucket's subject, which `$JS.API.>`
|
||||
// does not cover; each bucket named, not `$KV.>`, which would let it write any module's state.
|
||||
for _, bucket := range ControllerBuckets() {
|
||||
if bucket == delivery.HaveBucket {
|
||||
continue // each node's node-engine writes its own keys of it, the controller none (ADR 0429 §1)
|
||||
}
|
||||
pub = append(pub, "$KV."+bucket+".>")
|
||||
}
|
||||
// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
|
||||
@@ -800,6 +804,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// contract is lease/witness.go): it asks its own machine's node tools PING, and, where the
|
||||
// machine runs the controller, reads the lease's one key — read, never written.
|
||||
pub = append(pub, WitnessSubjects(p)...)
|
||||
// And the keys of its own node in the have bucket, which it alone writes (novox/hq ADR 0429 §1, issue 1136):
|
||||
// what the node has of each module, read by the view and the watchman. Never the controller's.
|
||||
haveKeys, err := NodeKeysGrants(delivery.HaveBucket, p.Node)
|
||||
if err != nil {
|
||||
return Permissions{}, err
|
||||
}
|
||||
pub = append(pub, haveKeys...)
|
||||
// And, while the vault serves it, the vault's move-in in its own name and nobody else's (issue 676): the
|
||||
// values sealed to this machine, which only it opens, handed in sealed to the vault's key for the call.
|
||||
if p.MovesIntoTheVault {
|
||||
@@ -812,6 +823,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
}
|
||||
|
||||
case KindModule:
|
||||
// 0. The watchman alone hears who connects and disconnects, and asks who is connected (ADR 0431 §2).
|
||||
if p.Module == WatchmanModule {
|
||||
sub = append(sub, BusConnectSubject, BusDisconnectSubject)
|
||||
pub = append(pub, BusConnzSubject)
|
||||
}
|
||||
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
|
||||
// else may publish into it, so an event's source is a fact the server enforces rather
|
||||
// than a claim in the body (design 29 §2).
|
||||
@@ -1436,7 +1452,9 @@ func ComposeAccounts(principals []Principal) (string, error) {
|
||||
// JetStream is enabled per account once accounts exist at all: with only the global block set,
|
||||
// a user in MESH is told "JetStream not enabled for account" the first time it binds a
|
||||
// consumer, which is the first thing every host does (2026-09-28).
|
||||
b.WriteString("accounts {\n MESH {\n jetstream: enabled\n users = [\n")
|
||||
b.WriteString("accounts {\n MESH {\n jetstream: enabled\n")
|
||||
b.WriteString(meshSystemImports)
|
||||
b.WriteString(" users = [\n")
|
||||
for _, p := range sorted {
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
@@ -1457,10 +1475,45 @@ func ComposeAccounts(principals []Principal) (string, error) {
|
||||
}
|
||||
b.WriteString(" } }\n")
|
||||
}
|
||||
b.WriteString(" ]\n }\n}\n")
|
||||
b.WriteString(" ]\n }\n")
|
||||
b.WriteString(systemAccount)
|
||||
b.WriteString("}\n")
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// WatchmanModule is the one module whose principal hears the bus's connect and disconnect events and asks its list of
|
||||
// connections (novox/hq ADR 0431 §2, issue 1147): the watchman of the repository mesh-delivery, which writes presence
|
||||
// from them. No other principal is granted the subjects below, and nothing is a wildcard over them.
|
||||
const WatchmanModule = "mesh-delivery-watchman"
|
||||
|
||||
// The bus's own events, imported into MESH from the default system account `$SYS` under subjects of MESH's own
|
||||
// (novox/hq ADR 0431 §1, issue 1147): the server says an account's connections opening and closing only to the system
|
||||
// account, and the list of an account's connections only to a request made there. `$SYS` holds no user, so nobody
|
||||
// gains its other requests and events; it exports exactly these three, to MESH alone.
|
||||
const (
|
||||
BusConnectSubject = "mesh.bus.connect"
|
||||
BusDisconnectSubject = "mesh.bus.disconnect"
|
||||
BusConnzSubject = "mesh.bus.connz"
|
||||
)
|
||||
|
||||
// meshSystemImports is what MESH imports from $SYS: the two event streams and the one service.
|
||||
const meshSystemImports = " imports = [\n" +
|
||||
" { stream: { account: \"$SYS\", subject: \"$SYS.ACCOUNT.MESH.CONNECT\" }, to: \"" + BusConnectSubject + "\" }\n" +
|
||||
" { stream: { account: \"$SYS\", subject: \"$SYS.ACCOUNT.MESH.DISCONNECT\" }, to: \"" + BusDisconnectSubject + "\" }\n" +
|
||||
" { service: { account: \"$SYS\", subject: \"$SYS.REQ.ACCOUNT.MESH.CONNZ\" }, to: \"" + BusConnzSubject + "\" }\n" +
|
||||
" ]\n"
|
||||
|
||||
// systemAccount is the account `$SYS`, the server's default system account by its own name (a config that declares an
|
||||
// account of that name makes it the system account), with no user and the three exports to MESH. Quoted, because an
|
||||
// unquoted `$` begins a variable in the server's config grammar.
|
||||
const systemAccount = " \"$SYS\" {\n" +
|
||||
" exports = [\n" +
|
||||
" { stream: \"$SYS.ACCOUNT.MESH.CONNECT\", accounts: [\"MESH\"] }\n" +
|
||||
" { stream: \"$SYS.ACCOUNT.MESH.DISCONNECT\", accounts: [\"MESH\"] }\n" +
|
||||
" { service: \"$SYS.REQ.ACCOUNT.MESH.CONNZ\", accounts: [\"MESH\"] }\n" +
|
||||
" ]\n" +
|
||||
" }\n"
|
||||
|
||||
// unique is a sorted list with each subject once. Two carried modules holding seats with the same
|
||||
// verb, or the runtime module itself carried beside the others, would otherwise write a grant twice
|
||||
// — harmless to the server, and noise in a file that is read as the mesh's authority model.
|
||||
|
||||
@@ -28,6 +28,8 @@ func TestTheComposedConfigMatchesTheGolden(t *testing.T) {
|
||||
Emits: []string{"order.placed"}, PasswordHash: "$2a$11$ssssssssssssssssssssss"},
|
||||
{Kind: KindModule, Node: "two", Module: "audit",
|
||||
Consumes: []string{"shop.order.placed"}, PasswordHash: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa"},
|
||||
// The watchman alone hears the bus's connections (novox/hq ADR 0431 §2, issue 1147).
|
||||
{Kind: KindModule, Node: "one", Module: WatchmanModule, PasswordHash: "$2a$11$wwwwwwwwwwwwwwwwwwwwww"},
|
||||
// The bus's own module: the snapshot API and its inbox, nothing else (novox/hq ADR 0235).
|
||||
{Kind: KindModule, Node: "one", Module: "nats", SnapshotsTheBus: true,
|
||||
Serves: []string{"nats_streams"}, PasswordHash: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb"},
|
||||
|
||||
+39
-16
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A module's state on the bus (novox/hq ADR 0201, design 32 §4, design 25 §3).
|
||||
@@ -35,8 +36,14 @@ type Bucket struct {
|
||||
// PerMachine says each machine's instance writes only the key named for its machine (novox/hq ADR
|
||||
// 0260), and is granted that key and no other.
|
||||
PerMachine bool
|
||||
// KeyTTL says the bucket allows a time to live per key (novox/hq ADR 0433 §1): its owner writes each key with its
|
||||
// own, and the server writes a delete marker, kept KeyTTLMarkerTTL, when it ends.
|
||||
KeyTTL bool
|
||||
}
|
||||
|
||||
// KeyTTLMarkerTTL is how long the marker an expired key leaves is kept: long enough for a watcher to hear it.
|
||||
const KeyTTLMarkerTTL = time.Minute
|
||||
|
||||
// BucketName is the bucket a module's state lives in: the module and the local name joined by an
|
||||
// underscore, which neither may contain, so two modules can never derive one bucket.
|
||||
func BucketName(module, name string) string { return module + "_" + name }
|
||||
@@ -65,6 +72,37 @@ func bucketOfRead(read string) (string, bool) {
|
||||
return BucketName(module, name), true
|
||||
}
|
||||
|
||||
// readGrants is what reaching a bucket's keys needs: binding to it, a direct get, an ordered consumer for listing and
|
||||
// watching and the deletion of that consumer and its flow control. key is one key's subject tail, a pattern such as
|
||||
// `one.>` for a prefix of keys, or empty for every key. The bucket is made with AllowDirect, so a get is the direct get
|
||||
// and never the stream's message get.
|
||||
func readGrants(bucket, key string) []string {
|
||||
stream := "KV_" + bucket
|
||||
out := []string{"$JS.API.STREAM.INFO." + stream}
|
||||
if key == "" {
|
||||
out = append(out,
|
||||
"$JS.API.DIRECT.GET."+stream+".>",
|
||||
"$JS.API.CONSUMER.CREATE."+stream+".>")
|
||||
} else {
|
||||
out = append(out,
|
||||
"$JS.API.DIRECT.GET."+stream+".$KV."+bucket+"."+key,
|
||||
"$JS.API.CONSUMER.CREATE."+stream+".*.$KV."+bucket+"."+key)
|
||||
}
|
||||
return append(out,
|
||||
"$JS.API.CONSUMER.DELETE."+stream+".>",
|
||||
"$JS.FC."+stream+".>")
|
||||
}
|
||||
|
||||
// NodeKeysGrants is what a node's node-engine is granted of a bucket it writes by node, `<node>.>` and no other key
|
||||
// (novox/hq ADR 0429 §1: each key one writer, enforced by credentials): the writes of those keys and what a get,
|
||||
// create, update and watch of them need. The node name is checked as every subject token is.
|
||||
func NodeKeysGrants(bucket, node string) ([]string, error) {
|
||||
if !safeSubject.MatchString(node) {
|
||||
return nil, fmt.Errorf("%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", node)
|
||||
}
|
||||
return append([]string{"$KV." + bucket + "." + node + ".>"}, readGrants(bucket, node+".>")...), nil
|
||||
}
|
||||
|
||||
// stateGrants is what a principal publishes to reach the state its modules keep and read: for every
|
||||
// bucket, binding to it, reading a key directly, and an ordered consumer for listing and watching,
|
||||
// created and deleted on the bucket's own stream, with its flow control answered; for a bucket an
|
||||
@@ -84,22 +122,7 @@ func bucketOfRead(read string) (string, bool) {
|
||||
// consumer (named at random by the client) and flow control; none of them reads or writes a value.
|
||||
func stateGrants(a stateAccess) []string {
|
||||
var out []string
|
||||
read := func(bucket, key string) {
|
||||
stream := "KV_" + bucket
|
||||
out = append(out, "$JS.API.STREAM.INFO."+stream)
|
||||
if key == "" {
|
||||
out = append(out,
|
||||
"$JS.API.DIRECT.GET."+stream+".>",
|
||||
"$JS.API.CONSUMER.CREATE."+stream+".>")
|
||||
} else {
|
||||
out = append(out,
|
||||
"$JS.API.DIRECT.GET."+stream+".$KV."+bucket+"."+key,
|
||||
"$JS.API.CONSUMER.CREATE."+stream+".*.$KV."+bucket+"."+key)
|
||||
}
|
||||
out = append(out,
|
||||
"$JS.API.CONSUMER.DELETE."+stream+".>",
|
||||
"$JS.FC."+stream+".>")
|
||||
}
|
||||
read := func(bucket, key string) { out = append(out, readGrants(bucket, key)...) }
|
||||
machineKey := ""
|
||||
if safeSubject.MatchString(a.Node) {
|
||||
machineKey = a.Node
|
||||
|
||||
@@ -3,9 +3,14 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
func TestABucketIsAssertedInPlace(t *testing.T) {
|
||||
@@ -53,3 +58,48 @@ func TestABucketIsAssertedOverAHeldConnection(t *testing.T) {
|
||||
t.Fatalf("the bucket is not there: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1154/AC4: on the pinned server a bucket declared with key-ttl takes a put with a time to live of a second, the key is
|
||||
// gone after it and the server has written a delete marker; and the bucket asserted again without key-ttl is not refused.
|
||||
func TestAKeyTTLBucketExpiresAKeyAndIsAssertedAgain(t *testing.T) { // 1154/AC4
|
||||
js := aLiveBus(t)
|
||||
b := Bucket{Module: "statetest", Name: "pulse", KeyTTL: true}
|
||||
if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
|
||||
t.Fatalf("a real server refused a key-ttl bucket: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
kv, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket, err := kv.KeyValue(ctx, b.Bucket())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := bucket.Create(ctx, "presence.alpha", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
|
||||
t.Fatalf("the bucket refused a key with a time to live: %v", err)
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
if _, err := bucket.Get(ctx, "presence.alpha"); errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the key is still there after its time to live")
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
hist, err := bucket.History(ctx, "presence.alpha")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if last := hist[len(hist)-1]; last.Operation() == jetstream.KeyValuePut {
|
||||
t.Fatalf("no delete marker was left: last entry is %v", last.Operation())
|
||||
}
|
||||
// The owner no longer says key-ttl: the server cannot turn the age off, so the bucket keeps it and is not refused.
|
||||
b.KeyTTL = false
|
||||
if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
|
||||
t.Fatalf("asserting the bucket without key-ttl was refused, so a restart would fail: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 1154/AC2: a bucket declared with key-ttl is asserted with the marker age that allows a time to live per message;
|
||||
// one without is asserted with none.
|
||||
func TestAKeyTTLBucketIsAssertedWithTheMarkerAge(t *testing.T) { // 1154/AC2
|
||||
with := BucketConfig(Bucket{Module: "mesh-delivery-watchman", Name: "pulse", KeyTTL: true}, 0)
|
||||
if with.LimitMarkerTTL != KeyTTLMarkerTTL || KeyTTLMarkerTTL <= 0 {
|
||||
t.Fatalf("a key-ttl bucket is asserted with marker age %v, want %v", with.LimitMarkerTTL, KeyTTLMarkerTTL)
|
||||
}
|
||||
without := BucketConfig(Bucket{Module: "mesh-delivery-watchman", Name: "stuck"}, 0)
|
||||
if without.LimitMarkerTTL != 0 {
|
||||
t.Fatalf("a bucket without key-ttl is asserted with marker age %v, want none", without.LimitMarkerTTL)
|
||||
}
|
||||
if with.Bucket != "mesh-delivery-watchman_pulse" || with.History != 1 || with.MaxBytes != StateMaxBytes {
|
||||
t.Fatalf("the rest of the options changed: %+v", with)
|
||||
}
|
||||
}
|
||||
|
||||
// 1154/AC3: a bucket that already allows a time to live per message keeps it when its owner no longer says key-ttl,
|
||||
// because the server refuses to turn it off; it is not asserted without it.
|
||||
func TestABucketThatAllowsAKeyTTLKeepsIt(t *testing.T) { // 1154/AC3
|
||||
got := BucketConfig(Bucket{Module: "m", Name: "s"}, 5*time.Minute)
|
||||
if got.LimitMarkerTTL != 5*time.Minute {
|
||||
t.Fatalf("a bucket that held marker age 5m is asserted with %v, which the server refuses", got.LimitMarkerTTL)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
//go:build integration
|
||||
|
||||
package broker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// **The system account's exports reach MESH on a reload, with no restart** (novox/hq ADR 0431 §3, issue 1147,
|
||||
// 1147/AC3): the embedded pinned server starts with today's composed accounts (no `$SYS`, no import); the new file is
|
||||
// written and the server reloaded; the reload succeeds; a user with the watchman's grants hears another user's connect
|
||||
// and disconnect and gets an answer to `mesh.bus.connz` that lists the open connections; another module's user is
|
||||
// refused a subscribe on `mesh.bus.connect`.
|
||||
func TestNatsTheSystemAccountsExportsReachMeshOnAReload(t *testing.T) { // 1147/AC3
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
users, err := Users(Records{Nodes: []string{"one"}, Assigned: map[string][]Declared{
|
||||
"one": {{Module: RuntimeModule}, {Module: WatchmanModule}, {Module: "gitea"}, {Module: "shop"}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range users {
|
||||
users[i].PasswordHash = string(hash)
|
||||
}
|
||||
accounts, err := ComposeAccounts(users)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Today's file: the same accounts without the system account's exports and MESH's imports.
|
||||
today := strings.Replace(strings.Replace(accounts, meshSystemImports, "", 1), systemAccount, "", 1)
|
||||
if today == accounts || strings.Contains(today, "$SYS") {
|
||||
t.Fatal("today's composed file could not be derived from the new one")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
conf := filepath.Join(dir, "bus.conf")
|
||||
write := func(accounts string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+`"`+dir+`/js"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write(today)
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("today's accounts do not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
|
||||
connect := func(user string) (*nats.Conn, chan error) {
|
||||
t.Helper()
|
||||
refused := make(chan error, 8)
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
|
||||
if err != nil {
|
||||
t.Fatalf("%s cannot connect: %v", user, err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
return nc, refused
|
||||
}
|
||||
// A connection open across the reload: the reload must not drop it.
|
||||
shop, _ := connect("one.shop")
|
||||
|
||||
write(accounts)
|
||||
if err := s.Reload(); err != nil {
|
||||
t.Fatalf("the server does not take the system account's exports on a reload: %v", err)
|
||||
}
|
||||
if !shop.IsConnected() {
|
||||
t.Error("the reload dropped a connection that was open")
|
||||
}
|
||||
|
||||
watchman, _ := connect("one." + WatchmanModule)
|
||||
connects := make(chan *nats.Msg, 16)
|
||||
disconnects := make(chan *nats.Msg, 16)
|
||||
if _, err := watchman.ChanSubscribe(BusConnectSubject, connects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := watchman.ChanSubscribe(BusDisconnectSubject, disconnects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := watchman.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Another user connects and disconnects: the watchman hears both, and the events name that user.
|
||||
gitea, _ := connect("one.gitea")
|
||||
hears := func(ch chan *nats.Msg, what string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case m := <-ch:
|
||||
if !strings.Contains(string(m.Data), "one.gitea") {
|
||||
t.Errorf("the %s event does not name the user that connected: %s", what, m.Data)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Errorf("the watchman did not hear another user's %s", what)
|
||||
}
|
||||
}
|
||||
hears(connects, "connect")
|
||||
// The answer to the connections request lists the open connections, the shop's among them. The server names the
|
||||
// account from the subject the export clamps (MESH), but leaves the user's name out unless the request asks for it
|
||||
// with "auth": true (monitor.go ConnzOptions.Username), so the watchman asks so.
|
||||
reply, err := watchman.Request(BusConnzSubject, []byte(`{"auth":true}`), 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh.bus.connz is not answered: %v", err)
|
||||
}
|
||||
var answer struct {
|
||||
Data struct {
|
||||
Connections []struct {
|
||||
AuthorizedUser string `json:"authorized_user"`
|
||||
} `json:"connections"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &answer); err != nil {
|
||||
t.Fatalf("the connections answer is not JSON: %v: %s", err, reply.Data)
|
||||
}
|
||||
var listed bool
|
||||
for _, c := range answer.Data.Connections {
|
||||
listed = listed || c.AuthorizedUser == "one.shop"
|
||||
}
|
||||
if !listed {
|
||||
t.Errorf("the connections answer does not list the shop's connection: %s", reply.Data)
|
||||
}
|
||||
gitea.Close()
|
||||
hears(disconnects, "disconnect")
|
||||
|
||||
// Another module's user is refused a subscribe on the events and a publish on the request.
|
||||
other, refused := connect("one.shop")
|
||||
for _, act := range []func() error{
|
||||
func() error { _, err := other.SubscribeSync(BusConnectSubject); return err },
|
||||
func() error { _, err := other.SubscribeSync(BusDisconnectSubject); return err },
|
||||
func() error { return other.Publish(BusConnzSubject, []byte(`{}`)) },
|
||||
} {
|
||||
if err := act(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = other.Flush()
|
||||
select {
|
||||
case err := <-refused:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("the server said %v, want a permissions violation", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("the server did not refuse another module's user the bus's connection subjects")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
)
|
||||
|
||||
// 1147/AC1: the composed accounts file declares $SYS with no user, exporting only the connect and disconnect events of
|
||||
// MESH and the request listing MESH's connections, and MESH imports exactly those three as mesh.bus.connect,
|
||||
// mesh.bus.disconnect and mesh.bus.connz (novox/hq ADR 0431 §1); the server's own grammar takes it with $SYS as its
|
||||
// system account; the golden file shows it (TestTheComposedConfigMatchesTheGolden).
|
||||
func TestTheComposedAccountsExportMeshsConnectionEventsFromTheSystemAccount(t *testing.T) { // 1147/AC1
|
||||
text, err := ComposeAccounts([]Principal{{Kind: KindNode, Node: "one", PasswordHash: "x"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sys := between(text, `"$SYS" {`, "\n }\n")
|
||||
if sys == "" {
|
||||
t.Fatalf("no account $SYS in the composed accounts:\n%s", text)
|
||||
}
|
||||
if strings.Contains(sys, "users") || strings.Contains(sys, "user:") {
|
||||
t.Errorf("$SYS holds a user:\n%s", sys)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`{ stream: "$SYS.ACCOUNT.MESH.CONNECT", accounts: ["MESH"] }`,
|
||||
`{ stream: "$SYS.ACCOUNT.MESH.DISCONNECT", accounts: ["MESH"] }`,
|
||||
`{ service: "$SYS.REQ.ACCOUNT.MESH.CONNZ", accounts: ["MESH"] }`,
|
||||
} {
|
||||
if !strings.Contains(sys, want) {
|
||||
t.Errorf("$SYS does not export %s:\n%s", want, sys)
|
||||
}
|
||||
}
|
||||
if n := strings.Count(sys, "{ stream:") + strings.Count(sys, "{ service:"); n != 3 {
|
||||
t.Errorf("$SYS exports %d things, want exactly 3:\n%s", n, sys)
|
||||
}
|
||||
mesh := between(text, " MESH {", " users = [")
|
||||
for _, want := range []string{
|
||||
`{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.CONNECT" }, to: "mesh.bus.connect" }`,
|
||||
`{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.DISCONNECT" }, to: "mesh.bus.disconnect" }`,
|
||||
`{ service: { account: "$SYS", subject: "$SYS.REQ.ACCOUNT.MESH.CONNZ" }, to: "mesh.bus.connz" }`,
|
||||
} {
|
||||
if !strings.Contains(mesh, want) {
|
||||
t.Errorf("MESH does not import %s:\n%s", want, mesh)
|
||||
}
|
||||
}
|
||||
if n := strings.Count(mesh, "{ stream:") + strings.Count(mesh, "{ service:"); n != 3 {
|
||||
t.Errorf("MESH imports %d things, want exactly 3:\n%s", n, mesh)
|
||||
}
|
||||
|
||||
// The pinned server's grammar takes it, and makes $SYS its system account.
|
||||
path := filepath.Join(t.TempDir(), "accounts.conf")
|
||||
if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("the server refuses the composed accounts: %v", err)
|
||||
}
|
||||
var found bool
|
||||
for _, a := range opts.Accounts {
|
||||
if a.Name != "$SYS" {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if !a.IsExportService("$SYS.REQ.ACCOUNT.MESH.CONNZ") {
|
||||
t.Error("$SYS does not export the connections request as a service")
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("the server read no account $SYS")
|
||||
}
|
||||
for _, u := range opts.Users {
|
||||
if u.Account != nil && u.Account.Name == "$SYS" {
|
||||
t.Errorf("user %s is in $SYS", u.Username)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// between is the text from the first start to the first end after it, or empty.
|
||||
func between(s, start, end string) string {
|
||||
i := strings.Index(s, start)
|
||||
if i < 0 {
|
||||
return ""
|
||||
}
|
||||
rest := s[i:]
|
||||
j := strings.Index(rest[len(start):], end)
|
||||
if j < 0 {
|
||||
return rest
|
||||
}
|
||||
return rest[:len(start)+j]
|
||||
}
|
||||
|
||||
// 1147/AC2: only mesh-delivery-watchman's principal may subscribe to mesh.bus.connect and mesh.bus.disconnect and
|
||||
// publish to mesh.bus.connz; no other principal's grant covers them by a wildcard (ADR 0431 §2).
|
||||
func TestOnlyTheWatchmanHearsTheBusConnectionEvents(t *testing.T) { // 1147/AC2
|
||||
builder := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Emits: []string{"built"}}
|
||||
watchman := Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-watchman", PasswordHash: "x"}
|
||||
others := []Principal{
|
||||
{Kind: KindController, PasswordHash: "x"},
|
||||
{Kind: KindNode, Node: "one", PasswordHash: "x"},
|
||||
{Kind: KindEnrolment, Node: "two", PasswordHash: "x"},
|
||||
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
|
||||
{Kind: KindProvisioner, Node: "one", PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "gitea", Emits: []string{"pull.merged"}, PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "mesh-delivery-view", PasswordHash: "x",
|
||||
Reads: []string{"node-engine.have"}, Consumes: []string{"mesh-delivery-watchman.presence"}},
|
||||
{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{builder}, PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "nats", SnapshotsTheBus: true, PasswordHash: "x"},
|
||||
{Kind: KindNodeTools, Node: "one", Module: RuntimeModule, PasswordHash: "x", Carries: []Declared{
|
||||
{Module: "mesh-delivery-watchman"}, {Module: "build-agent", Holds: []Seat{builder}}}},
|
||||
}
|
||||
heard := []string{"mesh.bus.connect", "mesh.bus.disconnect"}
|
||||
perms, err := PermissionsFor(watchman)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range heard {
|
||||
has(t, perms.Subscribe, s)
|
||||
}
|
||||
has(t, perms.Publish, "mesh.bus.connz")
|
||||
for _, s := range append(heard, "mesh.bus.connz") {
|
||||
for _, p := range perms.Subscribe {
|
||||
if p != s && SubjectsOverlap(p, s) && !strings.HasPrefix(p, "_INBOX.") {
|
||||
t.Errorf("the watchman's subscription %q reaches %s beyond naming it", p, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range others {
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p.Username(), err)
|
||||
}
|
||||
for _, s := range append(heard, "mesh.bus.connz") {
|
||||
for _, g := range append(append([]string(nil), perms.Publish...), perms.Subscribe...) {
|
||||
if SubjectsOverlap(g, s) {
|
||||
t.Errorf("%s is granted %q, which covers %s", p.Username(), g, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+18
-1
@@ -22,6 +22,11 @@ jetstream {
|
||||
accounts {
|
||||
MESH {
|
||||
jetstream: enabled
|
||||
imports = [
|
||||
{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.CONNECT" }, to: "mesh.bus.connect" }
|
||||
{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.DISCONNECT" }, to: "mesh.bus.disconnect" }
|
||||
{ service: { account: "$SYS", subject: "$SYS.REQ.ACCOUNT.MESH.CONNZ" }, to: "mesh.bus.connz" }
|
||||
]
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
@@ -33,10 +38,15 @@ accounts {
|
||||
subscribe: { allow: ["_INBOX.enrol.one.>"] }
|
||||
} }
|
||||
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "one.mesh-delivery-watchman", password: "$2a$11$wwwwwwwwwwwwwwwwwwwwww", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_mesh-delivery-watchman.>", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.mesh-delivery-watchman", "mesh.bus.connz"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.mesh-delivery-watchman", "$SRV.INFO.mesh-delivery-watchman.>", "$SRV.PING", "$SRV.PING.mesh-delivery-watchman", "$SRV.PING.mesh-delivery-watchman.>", "$SRV.STATS", "$SRV.STATS.mesh-delivery-watchman", "$SRV.STATS.mesh-delivery-watchman.>", "_INBOX.one.mesh-delivery-watchman.>", "mesh.assignment.one.mesh-delivery-watchman", "mesh.bus.connect", "mesh.bus.disconnect", "mesh.mod.mesh-delivery-watchman.call.>", "mesh.mod.mesh-delivery-watchman.tool.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: {
|
||||
publish: { allow: ["$JS.API.STREAM.INFO.*", "$JS.API.STREAM.NAMES", "$JS.API.STREAM.SNAPSHOT.*", "$JS.SNAPSHOT.ACK.>"] }
|
||||
subscribe: { allow: ["_INBOX.one.nats.>"] }
|
||||
@@ -58,4 +68,11 @@ accounts {
|
||||
} }
|
||||
]
|
||||
}
|
||||
"$SYS" {
|
||||
exports = [
|
||||
{ stream: "$SYS.ACCOUNT.MESH.CONNECT", accounts: ["MESH"] }
|
||||
{ stream: "$SYS.ACCOUNT.MESH.DISCONNECT", accounts: ["MESH"] }
|
||||
{ service: "$SYS.REQ.ACCOUNT.MESH.CONNZ", accounts: ["MESH"] }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The writers table (novox/hq to-be 45 §1, ADR 0227 rule 1), compiled in.
|
||||
@@ -92,6 +94,13 @@ func ownProvisioner(p Principal, pattern string) bool {
|
||||
return p.Kind == KindProvisioner && len(tokens) > 3 && tokens[2] == p.Node
|
||||
}
|
||||
|
||||
// ownNodeKeys is a node writing the have keys of its own node: `$KV.<bucket>.<node>.…`, the node in the third token
|
||||
// and no wildcard there (novox/hq ADR 0429 §1).
|
||||
func ownNodeKeys(p Principal, pattern string) bool {
|
||||
tokens := strings.Split(pattern, ".")
|
||||
return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
|
||||
}
|
||||
|
||||
// kvOf is a bucket's write subjects.
|
||||
func kvOf(bucket string) []string { return []string{"$KV." + bucket + ".>"} }
|
||||
|
||||
@@ -205,6 +214,11 @@ var WritersTable = []WriterRow{
|
||||
{State: "the rollouts of the chain", Writer: "controller (lease holder), compare-and-set on the key's revision",
|
||||
KeptIn: "key-value " + RolloutsBucket + ", sixty-four revisions of each key, none removed",
|
||||
Others: "read through `reads` as mesh-controller." + ChainRollouts, Subjects: kvOf(RolloutsBucket), Writes: isController},
|
||||
// What each machine has of each module (novox/hq ADR 0429 §1, design 47): written by that machine's node-engine under
|
||||
// its own node's keys, never by the controller; the view and the watchman read it.
|
||||
{State: "a machine's have", Writer: "the node-engine of the machine, under its own keys `<node>.>`",
|
||||
KeptIn: "key-value " + delivery.HaveBucket + ", twenty revisions of each key",
|
||||
Others: "read through `reads` as node-engine.have", Subjects: kvOf(delivery.HaveBucket), Writes: ownNodeKeys},
|
||||
{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
|
||||
KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
|
||||
Subjects: kvOf(DeclaredConfigBucket), Writes: isController},
|
||||
|
||||
@@ -41,6 +41,8 @@ var designRows = []string{
|
||||
// The chain beside the assignments, the record itself and no mirror of one (ADR 0408, design 47, issue 1089).
|
||||
"the builds of the chain",
|
||||
"the rollouts of the chain",
|
||||
// What each machine has of each module, written by its node-engine under its own keys (ADR 0429 §1, issue 1136).
|
||||
"a machine's have",
|
||||
"the declared-config mirror",
|
||||
"the predefined-keys mirror",
|
||||
// The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6).
|
||||
|
||||
@@ -36,6 +36,9 @@ type StateDeclaration struct {
|
||||
// PerMachine says the state is keyed by machine: each machine's instance writes the key named for
|
||||
// its machine and no other, and the bus grants it that key alone (novox/hq ADR 0260).
|
||||
PerMachine bool `json:"per-machine,omitempty"`
|
||||
// KeyTTL says each key of the bucket lives the time its writer gives it, and the server writes a delete marker
|
||||
// when it ends (novox/hq ADR 0433 §1): the bucket is made with the server's time to live per message.
|
||||
KeyTTL bool `json:"key-ttl,omitempty"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads a bucket as its bare name, or as {name, history, ttl-seconds}.
|
||||
@@ -49,7 +52,7 @@ func (s *StateDeclaration) UnmarshalJSON(raw []byte) error {
|
||||
dec := json.NewDecoder(bytes.NewReader(trimmed))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return fmt.Errorf("a state is either a name or {name, history, ttl-seconds, per-machine}: %w", typedUnknown(err))
|
||||
return fmt.Errorf("a state is either a name or {name, history, ttl-seconds, per-machine, key-ttl}: %w", typedUnknown(err))
|
||||
}
|
||||
*s = StateDeclaration(full)
|
||||
return nil
|
||||
@@ -57,7 +60,7 @@ func (s *StateDeclaration) UnmarshalJSON(raw []byte) error {
|
||||
|
||||
// MarshalJSON writes back the short form when there is nothing else to say.
|
||||
func (s StateDeclaration) MarshalJSON() ([]byte, error) {
|
||||
if s.History == 0 && s.TTLSeconds == 0 && !s.PerMachine {
|
||||
if s.History == 0 && s.TTLSeconds == 0 && !s.PerMachine && !s.KeyTTL {
|
||||
return json.Marshal(s.Name)
|
||||
}
|
||||
type plain StateDeclaration
|
||||
@@ -130,7 +133,7 @@ func StateProblems(m Manifest) []string {
|
||||
// controller's account alone (the writers table); declared as a module's state, they would be asserted a
|
||||
// second way and granted to the module's own principal as well. Named here so a read of one is a read of
|
||||
// kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing.
|
||||
var ControllerMirrors = []string{"assignments", "declared-config", "predefined", "builds", "rollouts"}
|
||||
var ControllerMirrors = []string{"assignments", "declared-config", "predefined", "builds", "rollouts", "checks-running"}
|
||||
|
||||
// StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no
|
||||
// such state. An absent owner says nothing — a module may be installed long before the one whose
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestAStateNameIsLocalAndOneToken(t *testing.T) {
|
||||
{`{"module":"a.b","version":"1","state":["s"]}`, `no dot`},
|
||||
{`{"module":"a","version":"1","reads":["bindings"]}`, `a read is <module>.<name>`},
|
||||
{`{"module":"a","version":"1","reads":["a.s"]}`, `which is its own state`},
|
||||
{`{"module":"a","version":"1","state":[{"name":"s","shared":true}]}`, `{name, history, ttl-seconds, per-machine}`},
|
||||
{`{"module":"a","version":"1","state":[{"name":"s","shared":true}]}`, `{name, history, ttl-seconds, per-machine, key-ttl}`},
|
||||
} {
|
||||
_, err := ParseManifest([]byte(c.manifest))
|
||||
if err == nil {
|
||||
@@ -107,3 +107,19 @@ func TestTheCataloguePassNamesAReadItsOwnerDoesNotKeep(t *testing.T) {
|
||||
t.Fatalf("the catalogue pass said %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
// 1154/AC1: a state entry may say key-ttl; it is read, written back, and a bare name or an entry without it reads as before.
|
||||
func TestAStateMayHaveKeysThatExpireOneByOne(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"mesh-delivery-watchman","version":"1",` +
|
||||
`"state":[{"name":"pulse","key-ttl":true},"stuck",{"name":"seen","history":5}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.State[0].KeyTTL || m.State[1].KeyTTL || m.State[2].KeyTTL {
|
||||
t.Fatalf("key-ttl not read as said: %+v", m.State)
|
||||
}
|
||||
out, _ := json.Marshal(m.State)
|
||||
if string(out) != `[{"name":"pulse","key-ttl":true},"stuck",{"name":"seen","history":5}]` {
|
||||
t.Fatalf("written back as %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -33,6 +34,9 @@ type Backend interface {
|
||||
Keys(ctx context.Context) ([]string, error)
|
||||
// Watch sends every key's current value, then a Caught entry, then each later write, until the context ends.
|
||||
Watch(ctx context.Context) (<-chan Entry, error)
|
||||
// WatchFiltered is Watch of the keys a subject pattern matches (`*` one token, `>` the rest), the filter kept by the
|
||||
// server: a read of one module's builds reads that module's keys and no other (novox/hq issue 1121).
|
||||
WatchFiltered(ctx context.Context, filter string) (<-chan Entry, error)
|
||||
// History is a key's earlier values and its current one, oldest first, at most the bucket keeps.
|
||||
History(ctx context.Context, key string) ([]Entry, error)
|
||||
}
|
||||
@@ -58,6 +62,7 @@ type Memory struct {
|
||||
values map[string][]Entry
|
||||
revision uint64
|
||||
watchers []*watcher
|
||||
filters map[string]int
|
||||
// Now is the clock the entries are dated by; nil is time.Now.
|
||||
Now func() time.Time
|
||||
// writes counts the creates and updates taken.
|
||||
@@ -155,7 +160,9 @@ func (m *Memory) write(key string, value []byte) uint64 {
|
||||
}
|
||||
m.values[key] = h
|
||||
for _, w := range m.watchers {
|
||||
w.push(clone(e))
|
||||
if w.match(key) {
|
||||
w.push(clone(e))
|
||||
}
|
||||
}
|
||||
return e.Revision
|
||||
}
|
||||
@@ -178,9 +185,10 @@ func (m *Memory) Keys(_ context.Context) ([]string, error) {
|
||||
|
||||
// watcher is one watch of a Memory: an unbounded queue, so a watch that is slow is behind and never misses a write.
|
||||
type watcher struct {
|
||||
mu sync.Mutex
|
||||
q []Entry
|
||||
wake chan struct{}
|
||||
match func(string) bool
|
||||
mu sync.Mutex
|
||||
q []Entry
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
func (w *watcher) push(e Entry) {
|
||||
@@ -205,18 +213,54 @@ func (w *watcher) pop() (Entry, bool) {
|
||||
}
|
||||
|
||||
func (m *Memory) Watch(ctx context.Context) (<-chan Entry, error) {
|
||||
return m.WatchFiltered(ctx, ">")
|
||||
}
|
||||
|
||||
// SubjectMatches is whether a key matches a subject pattern: `*` stands for one token, `>` for one or more at the end.
|
||||
func SubjectMatches(pattern, key string) bool {
|
||||
p, k := strings.Split(pattern, "."), strings.Split(key, ".")
|
||||
for n, t := range p {
|
||||
if t == ">" {
|
||||
return len(k) > n
|
||||
}
|
||||
if n >= len(k) || (t != "*" && t != k[n]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(k)
|
||||
}
|
||||
|
||||
// Filtered counts the filtered watches a Memory took, by filter, for a test that says what a read touched.
|
||||
func (m *Memory) Filtered() map[string]int {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
out := map[string]int{}
|
||||
for k, v := range m.filters {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Memory) WatchFiltered(ctx context.Context, filter string) (<-chan Entry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.fail != nil {
|
||||
return nil, m.fail
|
||||
}
|
||||
w := &watcher{wake: make(chan struct{}, 1)}
|
||||
w := &watcher{wake: make(chan struct{}, 1), match: func(k string) bool { return SubjectMatches(filter, k) }}
|
||||
if m.filters == nil {
|
||||
m.filters = map[string]int{}
|
||||
}
|
||||
m.filters[filter]++
|
||||
keys := make([]string, 0, len(m.values))
|
||||
for k := range m.values {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if !w.match(k) {
|
||||
continue
|
||||
}
|
||||
if e, held := m.last(k); held {
|
||||
w.q = append(w.q, clone(e))
|
||||
}
|
||||
|
||||
+12
-2
@@ -38,7 +38,7 @@ func OnTheBus(ctx context.Context, conn *nats.Conn, holds Holds) (*Chain, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(builds, rollouts, assignments, holds), nil
|
||||
return New(builds, rollouts, assignments, holds).WithManifests(builds), nil
|
||||
}
|
||||
|
||||
// busBucket is a Backend on a key-value bucket. It calls no delete and no purge: the type has none to call (ADR 0408 §4).
|
||||
@@ -85,7 +85,17 @@ func (b busBucket) Keys(ctx context.Context) ([]string, error) {
|
||||
}
|
||||
|
||||
func (b busBucket) Watch(ctx context.Context) (<-chan Entry, error) {
|
||||
w, err := b.kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
return b.WatchFiltered(ctx, ">")
|
||||
}
|
||||
|
||||
func (b busBucket) WatchFiltered(ctx context.Context, filter string) (<-chan Entry, error) {
|
||||
var w jetstream.KeyWatcher
|
||||
var err error
|
||||
if filter == ">" {
|
||||
w, err = b.kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
} else {
|
||||
w, err = b.kv.WatchFiltered(ctx, []string{filter}, jetstream.IgnoreDeletes())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+68
-1
@@ -128,6 +128,60 @@ func (s Bucket[T]) All(ctx context.Context) (map[string]Kept[T], error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Filtered is every record whose key matches a subject pattern, by key, read in one pass of a watch the server filters.
|
||||
func (s Bucket[T]) Filtered(ctx context.Context, filter string) (map[string]Kept[T], error) {
|
||||
ctx, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
w, err := s.b.WatchFiltered(ctx, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]Kept[T]{}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("reading %s: %w", filter, ctx.Err())
|
||||
case e, open := <-w:
|
||||
if !open {
|
||||
return nil, errors.New("the bucket stopped answering before it was read whole")
|
||||
}
|
||||
if e.Caught {
|
||||
return out, nil
|
||||
}
|
||||
k, err := decode[T](e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[e.Key] = k
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FilteredRaw is Filtered for keys that hold bytes of their own (a pointer holds a build's id), by key.
|
||||
func (s Bucket[T]) FilteredRaw(ctx context.Context, filter string) (map[string]Entry, error) {
|
||||
ctx, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
w, err := s.b.WatchFiltered(ctx, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]Entry{}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("reading %s: %w", filter, ctx.Err())
|
||||
case e, open := <-w:
|
||||
if !open {
|
||||
return nil, errors.New("the bucket stopped answering before it was read whole")
|
||||
}
|
||||
if e.Caught {
|
||||
return out, nil
|
||||
}
|
||||
out[e.Key] = e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// History is a key's earlier records and its current one, oldest first.
|
||||
func (s Bucket[T]) History(ctx context.Context, key string) ([]Kept[T], error) {
|
||||
all, err := s.b.History(ctx, key)
|
||||
@@ -182,10 +236,22 @@ type Chain struct {
|
||||
Rollouts Bucket[Rollout]
|
||||
Assignments Bucket[Assignment]
|
||||
|
||||
// Manifests holds each build's manifest once, under the digest of its bytes (ManifestKey); Kept says it is on the bus. A
|
||||
// chain without it keeps the manifest in the build's record, as the tests' chains do.
|
||||
Manifests Bucket[json.RawMessage]
|
||||
Kept bool
|
||||
|
||||
rollouts Backend
|
||||
holds Holds
|
||||
}
|
||||
|
||||
// WithManifests gives the chain its manifests bucket.
|
||||
func (c *Chain) WithManifests(b Backend) *Chain {
|
||||
c.Manifests = Bucket[json.RawMessage]{b: b, holds: c.holds}
|
||||
c.Kept = true
|
||||
return c
|
||||
}
|
||||
|
||||
// RolloutsOf is the rollouts bucket typed as the controller's plan, which carries more than the chain's Rollout names:
|
||||
// the same keys and values, read and written as the plan.
|
||||
func RolloutsOf[T any](c *Chain) Bucket[T] { return Bucket[T]{c.rollouts, c.holds} }
|
||||
@@ -206,5 +272,6 @@ func InMemory(holds Holds) *Chain {
|
||||
if holds == nil {
|
||||
holds = func(context.Context) error { return nil }
|
||||
}
|
||||
return New(NewMemory(), NewMemory(), NewMemory(), holds)
|
||||
builds := NewMemory()
|
||||
return New(builds, NewMemory(), NewMemory(), holds).WithManifests(builds)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The builds bucket's keys (novox/hq issue 1121). A build is under its module, so a read of one module is a read of the keys
|
||||
// `<module>.>` and no other, and a lookup that has what it looks up by is one Get of a pointer key, whose value is a build's id.
|
||||
//
|
||||
// <module>.build.<id> a build's record, its outcome (a records-only read is the filter `<module>.build.>`)
|
||||
// req.<id> a request, and a build that knew no module
|
||||
// id.<id> the module of the build with that id (the pointer a lookup by id alone reads)
|
||||
// <module>.newest the newest build of the module, of any state
|
||||
// <module>.worked the newest successful build of the module
|
||||
// <module>.held the build standing for the module's newest successful build (issue 280)
|
||||
// <module>.commit.<sha> the newest successful build from that commit: a CommitPointer
|
||||
// <module>.source.<key> the newest successful build with that reuse key (ADR 0350 §1), spelled by SpellReuse
|
||||
// manifest.sha256-<hex> a manifest, once, by the digest of its bytes
|
||||
//
|
||||
// **A pointer is a hint** (issue 1121): the record is written first, then each pointer by compare-and-set; a reader Gets the
|
||||
// record a pointer names and checks it, and when the pointer is missing or wrong reads the module's records, repairs the
|
||||
// pointers and says so.
|
||||
//
|
||||
// No module's name contains a dot. A key is never removed (ADR 0408 §4); the bare `<id>` keys of before the move stay, and are
|
||||
// never read once the kind `builds-layout` is moved.
|
||||
const (
|
||||
RequestPrefix = "req"
|
||||
IDPrefix = "id"
|
||||
)
|
||||
|
||||
// BuildKey is a build's record under its module; a build that knew none is the request's key.
|
||||
func BuildKey(module, id string) string {
|
||||
if module == "" {
|
||||
return RequestKey(id)
|
||||
}
|
||||
return module + ".build." + id
|
||||
}
|
||||
|
||||
func RequestKey(id string) string { return RequestPrefix + "." + id }
|
||||
func IDKey(id string) string { return IDPrefix + "." + id }
|
||||
func NewestKey(module string) string { return module + ".newest" }
|
||||
func WorkedKey(module string) string { return module + ".worked" }
|
||||
func HeldKey(module string) string { return module + ".held" }
|
||||
func CommitKey(module, sha string) string { return module + ".commit." + sha }
|
||||
func SourceKey(module, fp string) string { return module + ".source." + fp }
|
||||
func ModuleFilter(module string) string { return module + ".build.>" }
|
||||
|
||||
// ManifestPrefix is where a build's manifest is kept, in the builds bucket itself: a new bucket needs a grant the bus gets
|
||||
// only at a later push, and the push needs the builds. No module is named so.
|
||||
const ManifestPrefix = "manifest"
|
||||
|
||||
// ReservedModules are the names that are not a module's: a key under one is not a build's record.
|
||||
var ReservedModules = []string{ManifestPrefix, RequestPrefix, IDPrefix, LayoutPrefix}
|
||||
|
||||
// LayoutPrefix is the name of the key that says the builds are laid out under their modules: LayoutMarkerKey, written once by
|
||||
// the controller's start after it has laid the old keys out. Nothing reads the builds before it exists.
|
||||
const LayoutPrefix = "layout"
|
||||
|
||||
// LayoutMarkerKey is the key whose existence says the layout is made.
|
||||
const LayoutMarkerKey = LayoutPrefix + ".done"
|
||||
|
||||
// ManifestKey is where a manifest is kept: `manifest.sha256-<hex of its bytes>`.
|
||||
func ManifestKey(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return ManifestPrefix + ".sha256-" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// CommitPointer is the value of `<module>.commit.<sha>`: the build, and what the surveys of every module's commits read
|
||||
// without a Get of each build: its source fingerprint and what it would put on a machine.
|
||||
type CommitPointer struct {
|
||||
Build string `json:"build"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
}
|
||||
|
||||
// IsRecordKey is whether a key of the builds bucket is a build's record under a module: `<module>.build.<id>`.
|
||||
func IsRecordKey(key string) bool {
|
||||
parts := strings.Split(key, ".")
|
||||
return len(parts) == 3 && parts[1] == "build" && !slices.Contains(ReservedModules, parts[0])
|
||||
}
|
||||
|
||||
// SpellReuse is a reuse key (`v1:<hex>`) as a part of a key the bus takes: every character it does not take becomes `-`.
|
||||
func SpellReuse(key string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' {
|
||||
return r
|
||||
}
|
||||
return '-'
|
||||
}, key)
|
||||
}
|
||||
|
||||
// ModuleOf is the module a record key is under.
|
||||
func ModuleOf(key string) string { m, _, _ := strings.Cut(key, "."); return m }
|
||||
|
||||
// DecodeKept is a watched entry as a typed record, for a read that took the entries of several kinds and decodes those
|
||||
// that hold a record.
|
||||
func DecodeKept[T any](e Entry) (Kept[T], error) { return decode[T](e) }
|
||||
|
||||
// AsAnswered is the state of a request record whose outcome has been heard: the record keeps the request and says which
|
||||
// build answered it, so a read of the requests needs no read of the builds.
|
||||
const BuildAnswered = "answered"
|
||||
@@ -0,0 +1,38 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSubjectsMatchAsTheBusFiltersThem(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
pattern, key string
|
||||
want bool
|
||||
}{
|
||||
{"alpha.>", "alpha.build-1", true}, {"alpha.>", "alpha.commit.c1", true}, {"alpha.>", "beta.build-1", false},
|
||||
{"alpha.>", "alpha", false}, {"*.newest", "alpha.newest", true}, {"*.newest", "alpha.commit.newest", false},
|
||||
{"*", "build-1", true}, {"*", "alpha.build-1", false}, {">", "a.b.c", true}, {"req.*", "req.build-1", true},
|
||||
} {
|
||||
if got := SubjectMatches(c.pattern, c.key); got != c.want {
|
||||
t.Errorf("SubjectMatches(%q, %q) = %v", c.pattern, c.key, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheKeysOfAFilterAreWatched(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
for _, k := range []string{"alpha.build.b1", "alpha.build.b2", "beta.build.b3", "alpha.newest"} {
|
||||
if _, err := c.Builds.CreateRaw(ctx, k, []byte(`{"id":"x"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := c.Builds.FilteredRaw(ctx, ModuleFilter("alpha"))
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("alpha.> gave %v: %v", got, err)
|
||||
}
|
||||
if !IsRecordKey("alpha.build.b1") || IsRecordKey("alpha.newest") || IsRecordKey("alpha.commit.c") || IsRecordKey("manifest.build.x") || IsRecordKey("req.b1") || IsRecordKey("id.b1") {
|
||||
t.Error("IsRecordKey tells a record from a pointer wrongly")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A pending value is not a standing assignment** (ADR 0261, ADR 0408, issue 1094): it names its node and module, holds no
|
||||
// time assigned, says `state: pending`, and every reader that skips a value with a state skips it.
|
||||
func TestAPendingValueIsNotAStandingAssignment(t *testing.T) {
|
||||
since := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
a := Assignment{Node: "anchor", Module: "notes", State: StatePending,
|
||||
Pending: &Pending{ID: "p-1", State: PendingWaiting, Build: "build-1", Since: since}}
|
||||
if a.Stands() {
|
||||
t.Fatal("a pending value stands")
|
||||
}
|
||||
raw, _ := json.Marshal(a)
|
||||
for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"state":"pending"`, `"pending":{"id":"p-1","state":"waiting"`} {
|
||||
if !strings.Contains(string(raw), want) {
|
||||
t.Errorf("the value lacks %s: %s", want, raw)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(raw), `"assigned"`) {
|
||||
t.Errorf("a pending value names a time assigned: %s", raw)
|
||||
}
|
||||
// The wait carried into the value that stands: no state, so it stands, and the record stays.
|
||||
made := Assignment{Node: "anchor", Module: "notes", Assigned: since, Pending: &Pending{ID: "p-1", State: PendingApplied}}
|
||||
if !made.Stands() || !a.Pending.Open() || made.Pending.Open() {
|
||||
t.Fatalf("stands %v, open before %v, open after %v", made.Stands(), a.Pending.Open(), made.Pending.Open())
|
||||
}
|
||||
var back Assignment
|
||||
if err := json.Unmarshal(raw, &back); err != nil || back.Pending == nil || back.Pending.Build != "build-1" || !back.Pending.Since.Equal(since) {
|
||||
t.Fatalf("read back %+v: %v", back, err)
|
||||
}
|
||||
now := since
|
||||
raised := Pending{ID: "p-2", State: PendingExpired, Raised: &now, Acknowledged: &now, Cleared: &now}
|
||||
body, _ := json.Marshal(raised)
|
||||
if !strings.Contains(string(body), `"raised"`) || !strings.Contains(string(body), `"acknowledged"`) || !strings.Contains(string(body), `"cleared"`) {
|
||||
t.Errorf("the marks of a wait: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTransitionsOfAPendingAssignment(t *testing.T) {
|
||||
ok := [][2]string{{PendingWaiting, PendingExpired}, {PendingWaiting, PendingWithdrawn}, {PendingApplying, PendingApplied},
|
||||
{PendingApplying, PendingRefused}}
|
||||
for _, p := range ok {
|
||||
if !CanSettle(p[0], p[1]) {
|
||||
t.Errorf("%s may end as %s", p[0], p[1])
|
||||
}
|
||||
}
|
||||
for _, p := range [][2]string{{PendingWaiting, PendingApplied}, {PendingWaiting, PendingRefused}, {PendingApplying, PendingExpired},
|
||||
{PendingApplying, PendingWithdrawn}, {PendingApplied, PendingExpired}, {PendingExpired, PendingWithdrawn},
|
||||
{PendingWaiting, PendingWaiting}} {
|
||||
if CanSettle(p[0], p[1]) {
|
||||
t.Errorf("%s may not end as %s", p[0], p[1])
|
||||
}
|
||||
}
|
||||
if !CanMove(PendingWaiting, PendingApplying) || !CanMove(PendingApplying, PendingWaiting) || CanMove(PendingApplied, PendingWaiting) {
|
||||
t.Error("a claim moves a wait to applying and back")
|
||||
}
|
||||
}
|
||||
|
||||
// **A module only pulled in is a standing value marked pulled-in, never a state, and a hand send makes a key say its commit
|
||||
// unknown with a time** (issue 1109): every reader treats a non-empty state as no assignment, so a pulled-in key must have
|
||||
// none; and unknown is never written in `commit`.
|
||||
func TestAPulledInValueStandsAndAHandSendIsATimeNotACommit(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
pulled := Assignment{Node: "anchor", Module: "mesh-wireguard", PulledIn: true, Commit: "c1", Build: "b1", Since: at}
|
||||
if !pulled.Stands() || pulled.State != "" {
|
||||
t.Fatalf("a pulled-in value does not stand: %+v", pulled)
|
||||
}
|
||||
raw, _ := json.Marshal(pulled)
|
||||
if !strings.Contains(string(raw), `"pulled-in":true`) || strings.Contains(string(raw), `"state"`) || strings.Contains(string(raw), `"assigned"`) {
|
||||
t.Fatalf("the pulled-in value is %s", raw)
|
||||
}
|
||||
unknown := Assignment{Node: "anchor", Module: "notes", Assigned: at, SentUnknown: at}
|
||||
raw, _ = json.Marshal(unknown)
|
||||
if !strings.Contains(string(raw), `"sent-unknown":"2026-10-18T03:00:00Z"`) || strings.Contains(string(raw), `"commit"`) || !unknown.Stands() {
|
||||
t.Fatalf("the hand-sent value is %s", raw)
|
||||
}
|
||||
plain, _ := json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at})
|
||||
if strings.Contains(string(plain), "pulled-in") || strings.Contains(string(plain), "sent-unknown") {
|
||||
t.Fatalf("a value that is neither says so: %s", plain)
|
||||
}
|
||||
}
|
||||
+77
-10
@@ -87,15 +87,17 @@ type Request struct {
|
||||
// Build is one attempt, whichever way it went, kept under its id. It names its module, its repository and commit, its
|
||||
// reuse key (novox/hq ADR 0350) and what it published (ADR 0408 §1).
|
||||
type Build struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state,omitempty"`
|
||||
Repository string `json:"repository"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
On string `json:"on,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Manifest json.RawMessage `json:"manifest,omitempty"`
|
||||
ID string `json:"id"`
|
||||
State string `json:"state,omitempty"`
|
||||
Repository string `json:"repository"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
On string `json:"on,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Manifest json.RawMessage `json:"manifest,omitempty"`
|
||||
// ManifestDigest names the manifest in the manifests bucket (ManifestKey); a record that has it keeps no Manifest.
|
||||
ManifestDigest string `json:"manifest-digest,omitempty"`
|
||||
Against []string `json:"against,omitempty"`
|
||||
Mirrored []string `json:"mirrored,omitempty"`
|
||||
Read []ReadRepository `json:"read,omitempty"`
|
||||
@@ -189,9 +191,74 @@ type Assignment struct {
|
||||
|
||||
// Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed**
|
||||
// (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a
|
||||
// State, as no assignment. State is, in a later delivery, the pending assignment waiting for its build (issue 1094).
|
||||
// State, as no assignment. State is StatePending while the key holds an assignment waiting for its build, or one that
|
||||
// was never made (issue 1094).
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
State string `json:"state,omitempty"`
|
||||
// Pending is the assignment's wait for its build (novox/hq ADR 0261), kept in the value of the key it waits for and, when
|
||||
// it is made, carried into the value that stands: the record of what became of the wait.
|
||||
Pending *Pending `json:"pending,omitempty"`
|
||||
|
||||
// PulledIn marks a standing value of a module that is on the machine only because another requires it, or because a
|
||||
// send carried it: no time assigned, but a key all the same, so what the machine was sent of it is kept and read
|
||||
// (issue 1109). Never a state: every reader treats a non-empty state as no assignment, and would take it off the machine.
|
||||
PulledIn bool `json:"pulled-in,omitempty"`
|
||||
// SentUnknown is when a send by hand made what the machine runs unknown (a declaration sent with no builds known): the
|
||||
// commit is cleared with it. Absent normally; a send that names its builds clears it.
|
||||
SentUnknown time.Time `json:"sent-unknown,omitzero"`
|
||||
}
|
||||
|
||||
// StatePending is the State of a value that holds no assignment but a pending one.
|
||||
const StatePending = "pending"
|
||||
|
||||
// The states of a pending assignment (novox/hq ADR 0261).
|
||||
const (
|
||||
PendingWaiting = "waiting"
|
||||
PendingApplying = "applying"
|
||||
PendingApplied = "applied"
|
||||
PendingRefused = "refused"
|
||||
PendingExpired = "expired"
|
||||
PendingWithdrawn = "withdrawn"
|
||||
)
|
||||
|
||||
// Pending is an assignment kept until its module is registered: the build it waits for, where that build is from, and the
|
||||
// times of its claim and its end. Its id is p-<unix nanoseconds of Since>, unique for a key, which holds one at a time.
|
||||
type Pending struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
Build string `json:"build,omitempty"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Note string `json:"note,omitempty"`
|
||||
Settled *time.Time `json:"settled,omitempty"`
|
||||
Raised *time.Time `json:"raised,omitempty"`
|
||||
Acknowledged *time.Time `json:"acknowledged,omitempty"`
|
||||
Cleared *time.Time `json:"cleared,omitempty"`
|
||||
// Earlier are the waits of this key that ended unmade and were replaced by this one before their condition was answered
|
||||
// (taken back or cleared): each stays, with its own marks, so a replaced wait never leaves the operator without the
|
||||
// condition it raised or was to raise. Dropped once answered.
|
||||
Earlier []Pending `json:"earlier,omitempty"`
|
||||
}
|
||||
|
||||
// Open says the assignment is still to be made.
|
||||
func (p Pending) Open() bool { return p.State == PendingWaiting || p.State == PendingApplying }
|
||||
|
||||
// CanSettle says a pending assignment in state from may end in state to: a wait expires or is withdrawn, a claim becomes
|
||||
// applied or refused, and nothing else.
|
||||
func CanSettle(from, to string) bool {
|
||||
switch from {
|
||||
case PendingWaiting:
|
||||
return to == PendingExpired || to == PendingWithdrawn
|
||||
case PendingApplying:
|
||||
return to == PendingApplied || to == PendingRefused
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CanMove says a pending assignment in state from may go to state to without ending: claimed, or released again.
|
||||
func CanMove(from, to string) bool {
|
||||
return (from == PendingWaiting && to == PendingApplying) || (from == PendingApplying && to == PendingWaiting)
|
||||
}
|
||||
|
||||
// Stands says the value is an assignment that holds: it has not ended and is not waiting in a state of its own.
|
||||
|
||||
@@ -3,11 +3,8 @@ package inventory
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// The merges a batch or walk answers (novox/hq ADR 0276): every merge the controller heard into a branch a
|
||||
@@ -33,138 +30,78 @@ type BatchedMerge struct {
|
||||
|
||||
// AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan.
|
||||
func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return false, err
|
||||
} else if on {
|
||||
return i.chainAddMerge(ctx, c, m)
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`insert into batched_merge (repository, branch, commit_hash, merged_at, heard_at, event, plan_id, alone)
|
||||
values (lower($1), $2, $3, $4, $5, $6, nullif($7, ''), $8) on conflict do nothing`,
|
||||
m.Repository, m.Branch, m.Commit, mergedAt(m.Merged), m.Heard, []byte(m.Event), m.Plan, m.Alone)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
return i.chainAddMerge(ctx, c, m)
|
||||
}
|
||||
|
||||
// MergeOf is one kept merge; false when it was never heard.
|
||||
func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return BatchedMerge{}, false, err
|
||||
} else if on {
|
||||
return i.chainMergeOf(ctx, c, repository, commit)
|
||||
}
|
||||
ms, err := i.merges(ctx, `where repository = lower($1) and commit_hash = $2`, repository, commit)
|
||||
if err != nil || len(ms) == 0 {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return BatchedMerge{}, false, err
|
||||
}
|
||||
return ms[0], true, nil
|
||||
return i.chainMergeOf(ctx, c, repository, commit)
|
||||
}
|
||||
|
||||
// MergesOf is every merge a plan answers, oldest merge first.
|
||||
func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainMergesOf(ctx, c, plan)
|
||||
}
|
||||
return i.merges(ctx, `where plan_id = $1 order by merged_at nulls last, heard_at`, plan)
|
||||
return i.chainMergesOf(ctx, c, plan)
|
||||
}
|
||||
|
||||
// LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the
|
||||
// newest first.
|
||||
func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainLaterMergesOf(ctx, c, repository, branch, after)
|
||||
}
|
||||
return i.merges(ctx, `where repository = lower($1) and branch = $2 and merged_at > $3 and plan_id is not null
|
||||
order by merged_at desc`, repository, branch, after)
|
||||
return i.chainLaterMergesOf(ctx, c, repository, branch, after)
|
||||
}
|
||||
|
||||
// KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a
|
||||
// module's last look says (novox/hq ADR 0276).
|
||||
func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return false, time.Time{}, err
|
||||
} else if on {
|
||||
return i.chainKeptOnBranch(ctx, c, repository, branch)
|
||||
}
|
||||
var first *time.Time
|
||||
var kept bool
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select min(heard_at), count(*) > 0 from batched_merge where repository = lower($1) and branch = $2`,
|
||||
repository, branch).Scan(&first, &kept)
|
||||
if err != nil || first == nil {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return false, time.Time{}, err
|
||||
}
|
||||
return kept, first.UTC(), nil
|
||||
return i.chainKeptOnBranch(ctx, c, repository, branch)
|
||||
}
|
||||
|
||||
// AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3).
|
||||
func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainAloneMerges(ctx, c)
|
||||
}
|
||||
return i.merges(ctx, `where plan_id is null and alone order by merged_at desc nulls last, heard_at desc`)
|
||||
return i.chainAloneMerges(ctx, c)
|
||||
}
|
||||
|
||||
// AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to
|
||||
// be walked on its own commit.
|
||||
func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if on {
|
||||
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`,
|
||||
repository, commit, plan, alone)
|
||||
return err
|
||||
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
|
||||
}
|
||||
|
||||
// Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more
|
||||
// are read so a fault is seen.
|
||||
func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated)
|
||||
}
|
||||
return i.plans(ctx, `where state in ('assembling', 'queued') order by created`)
|
||||
}
|
||||
|
||||
func (i *Inventory) merges(ctx context.Context, tail string, args ...any) ([]BatchedMerge, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select repository, branch, commit_hash, merged_at, heard_at, event, coalesce(plan_id, ''), alone
|
||||
from batched_merge `+tail, args...)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []BatchedMerge
|
||||
for rows.Next() {
|
||||
var m BatchedMerge
|
||||
var merged *time.Time
|
||||
var event []byte
|
||||
if err := rows.Scan(&m.Repository, &m.Branch, &m.Commit, &merged, &m.Heard, &event, &m.Plan, &m.Alone); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if merged != nil {
|
||||
m.Merged = merged.UTC()
|
||||
}
|
||||
m.Heard = m.Heard.UTC()
|
||||
m.Event = event
|
||||
out = append(out, m)
|
||||
}
|
||||
if errors.Is(rows.Err(), pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return out, rows.Err()
|
||||
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated)
|
||||
}
|
||||
|
||||
// SameRepository says two spellings of a repository are one.
|
||||
|
||||
+17
-286
@@ -2,14 +2,9 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
@@ -22,9 +17,11 @@ import (
|
||||
|
||||
// Build is one attempt, whichever way it went.
|
||||
type Build struct {
|
||||
ID string
|
||||
Repository string
|
||||
Ref string
|
||||
ID string
|
||||
// ManifestDigest names the manifest in the manifests bucket, when a chain keeps it apart from the record (issue 1121).
|
||||
ManifestDigest string
|
||||
Repository string
|
||||
Ref string
|
||||
// Module is empty for a build that failed before it knew what it was building.
|
||||
Module string
|
||||
Commit string
|
||||
@@ -134,51 +131,11 @@ func (b Build) Worked() bool { return b.Failed == "" }
|
||||
// whoever asked and once on the exchange when nobody was. Recording both would show one build as
|
||||
// two, and which of the two is real is not a question anybody could answer afterwards.
|
||||
func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return err
|
||||
} else if on {
|
||||
return i.chainRecordBuild(ctx, c, b)
|
||||
}
|
||||
made, err := json.Marshal(b.Made)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
against, err := json.Marshal(b.Against)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
read, err := json.Marshal(b.Read)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var sources any
|
||||
if len(b.Sources) > 0 {
|
||||
raw, err := json.Marshal(b.Sources)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sources = raw
|
||||
}
|
||||
var module *string
|
||||
if b.Module != "" {
|
||||
module = &b.Module
|
||||
}
|
||||
var asked *time.Time
|
||||
if !b.Asked.IsZero() {
|
||||
asked = &b.Asked
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
|
||||
source_path, manifest, built_against, built_contexts, asked, source_fingerprint,
|
||||
build_sources)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
|
||||
on conflict (id) do nothing`,
|
||||
b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
|
||||
b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint, sources)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
|
||||
return i.chainRecordBuild(ctx, c, b)
|
||||
}
|
||||
|
||||
// RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257):
|
||||
@@ -221,79 +178,22 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainBuildList(ctx, c, module, limit)
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
|
||||
from build order by at desc limit $1`
|
||||
args := []any{limit}
|
||||
if module != "" {
|
||||
query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
|
||||
from build where module = $2 order by at desc limit $1`
|
||||
args = append(args, module)
|
||||
}
|
||||
|
||||
rows, err := i.store.Pool().Query(ctx, query, args...)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Build
|
||||
for rows.Next() {
|
||||
var b Build
|
||||
var made []byte
|
||||
var asked *time.Time
|
||||
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
|
||||
&b.On, &b.Failed, &made, &asked, &b.At); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if asked != nil {
|
||||
b.Asked = *asked
|
||||
}
|
||||
if err := json.Unmarshal(made, &b.Made); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out, rows.Err()
|
||||
return i.chainBuildList(ctx, c, module, limit)
|
||||
}
|
||||
|
||||
// BuildByID is one build's record, by the id its request carried — what a plan matches its outcome
|
||||
// by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the
|
||||
// repository, path, ref and commit from. False when no outcome with that id was recorded.
|
||||
func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return Build{}, false, err
|
||||
} else if on {
|
||||
return i.chainBuildByID(ctx, c, id)
|
||||
}
|
||||
var b Build
|
||||
var made []byte
|
||||
var asked *time.Time
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made,
|
||||
coalesce(source_path,''), asked, at
|
||||
from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
|
||||
&b.On, &b.Failed, &made, &b.Path, &asked, &b.At)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Build{}, false, nil
|
||||
}
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return Build{}, false, err
|
||||
}
|
||||
if asked != nil {
|
||||
b.Asked = *asked
|
||||
}
|
||||
if err := json.Unmarshal(made, &b.Made); err != nil {
|
||||
return Build{}, false, err
|
||||
}
|
||||
return b, true, nil
|
||||
return i.chainBuildByID(ctx, c, id)
|
||||
}
|
||||
|
||||
// Held is every artifact this mesh has built, keyed "<module>/<artifact>".
|
||||
@@ -375,40 +275,11 @@ func registeredOnly(held, registered map[string]string) map[string]string {
|
||||
// edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
|
||||
// module standing on nothing looks like: an edge the mesh has not derived is not an edge.
|
||||
func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainBuiltAgainst(ctx, c)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select distinct on (module) module, built_against
|
||||
from build
|
||||
where module is not null and module <> '' and failed = ''
|
||||
order by module, `+newestRequestFirst)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
against := map[string][]string{}
|
||||
for rows.Next() {
|
||||
var module string
|
||||
var raw []byte
|
||||
if err := rows.Scan(&module, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
var refs []string
|
||||
if err := json.Unmarshal(raw, &refs); err != nil {
|
||||
continue
|
||||
}
|
||||
if len(refs) > 0 {
|
||||
against[module] = refs
|
||||
}
|
||||
}
|
||||
return against, rows.Err()
|
||||
return i.chainBuiltAgainst(ctx, c)
|
||||
}
|
||||
|
||||
// ReadRepositories is what each module's newest successful build read source from besides its own
|
||||
@@ -423,95 +294,11 @@ func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, erro
|
||||
// none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read
|
||||
// as before: every file of a context, and its own directory or root.
|
||||
func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainReadRepositories(ctx, c)
|
||||
}
|
||||
// The newest build of each module whatever its outcome: a failed one newer than the newest that
|
||||
// worked leaves that one's build source stale — the merge it was asked for may have changed the closure
|
||||
// (novox/hq ADR 0267), so its module is read whole until a build works again.
|
||||
newest := map[string]struct {
|
||||
at time.Time
|
||||
failed bool
|
||||
}{}
|
||||
tried, err := i.store.Pool().Query(ctx,
|
||||
`select distinct on (module) module, coalesce(asked, at), failed <> ''
|
||||
from build
|
||||
where module is not null and module <> ''
|
||||
order by module, `+newestRequestFirst)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for tried.Next() {
|
||||
var module string
|
||||
var at time.Time
|
||||
var failed bool
|
||||
if err := tried.Scan(&module, &at, &failed); err != nil {
|
||||
tried.Close()
|
||||
return nil, err
|
||||
}
|
||||
newest[module] = struct {
|
||||
at time.Time
|
||||
failed bool
|
||||
}{at, failed}
|
||||
}
|
||||
tried.Close()
|
||||
if err := tried.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select distinct on (module) module, built_contexts, build_sources, coalesce(asked, at),
|
||||
(select source from module where module.name = build.module)
|
||||
from build
|
||||
where module is not null and module <> '' and failed = ''
|
||||
order by module, `+newestRequestFirst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
read := map[string][]ReadRepository{}
|
||||
for rows.Next() {
|
||||
var module string
|
||||
var raw, rawSources []byte
|
||||
var built time.Time
|
||||
var own *string
|
||||
if err := rows.Scan(&module, &raw, &rawSources, &built, &own); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var of []ReadRepository
|
||||
if len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, &of); err != nil {
|
||||
of = nil
|
||||
}
|
||||
}
|
||||
var sources []BuildSource
|
||||
if len(rawSources) > 0 {
|
||||
if err := json.Unmarshal(rawSources, &sources); err != nil {
|
||||
sources = nil
|
||||
}
|
||||
}
|
||||
whole := ""
|
||||
if n, known := newest[module]; known && n.failed && n.at.After(built) && len(sources) > 0 {
|
||||
sources, whole = nil, "a newer build of it failed"
|
||||
}
|
||||
of = WithBuildSources(of, sources)
|
||||
if own != nil {
|
||||
of = OwnRepositoryAsOwn(of, *own)
|
||||
}
|
||||
if whole != "" {
|
||||
of = append(of, ReadRepository{Own: true, Whole: whole})
|
||||
}
|
||||
if len(of) > 0 {
|
||||
for k := range of {
|
||||
of[k].Built, of[k].Looked = built, newest[module].at
|
||||
}
|
||||
read[module] = of
|
||||
}
|
||||
}
|
||||
return read, rows.Err()
|
||||
return i.chainReadRepositories(ctx, c)
|
||||
}
|
||||
|
||||
// WithBuildSources lays a build's build sources over the repositories it read: a context's paths on its
|
||||
@@ -588,18 +375,6 @@ func namesRepository(registered, named string) bool {
|
||||
return reg == name || strings.HasSuffix(reg, "/"+name)
|
||||
}
|
||||
|
||||
// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept".
|
||||
//
|
||||
// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one
|
||||
// whose manifest was empty. A replay can then say which it is holding instead of inventing an
|
||||
// empty declaration for a module that certainly had one.
|
||||
func manifestOrNil(raw []byte) any {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// Announceable is every build worth telling a catalogue about, oldest first.
|
||||
//
|
||||
// **Oldest first, because a graph is built in the order things happened.** Registering a module
|
||||
@@ -614,53 +389,9 @@ func manifestOrNil(raw []byte) any {
|
||||
// One row per module and commit: a module built twice at the same commit is one fact, and the
|
||||
// row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219).
|
||||
func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainAnnounceable(ctx, c)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select distinct on (module, commit_hash)
|
||||
id, repository, ref, module, commit_hash, built_on, failed, made,
|
||||
source_path, manifest, built_against, at
|
||||
from build
|
||||
where failed = '' and module is not null and module <> '' and commit_hash <> ''
|
||||
order by module, commit_hash, `+newestRequestFirst)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Build
|
||||
for rows.Next() {
|
||||
var b Build
|
||||
var made []byte
|
||||
var manifest, against []byte
|
||||
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
|
||||
&b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(made, &b.Made); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Null rather than empty is a build recorded before the mesh kept these, and saying so is
|
||||
// the point of keeping them nullable: the replay carries nothing rather than an empty
|
||||
// declaration for a module that certainly had one.
|
||||
if len(manifest) > 0 {
|
||||
b.Manifest = manifest
|
||||
}
|
||||
if len(against) > 0 {
|
||||
if err := json.Unmarshal(against, &b.Against); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and
|
||||
// the order that matters to a catalogue is a different one.
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) })
|
||||
return out, nil
|
||||
return i.chainAnnounceable(ctx, c)
|
||||
}
|
||||
|
||||
@@ -216,7 +216,7 @@ func TestABuildsBuildSourceComesBackOverWhatItRead(t *testing.T) {
|
||||
if got := read["build-agent"]; len(got) != 1 || got[0].Paths != nil || got[0].Own {
|
||||
t.Fatalf("a build that said no build source reads as %+v", got)
|
||||
}
|
||||
kept, found, err := inv.chain.Builds.Get(ctx, "said")
|
||||
kept, found, err := inv.chain.Builds.Get(ctx, "route-proxy.build.said")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the build is not on the bus: %v %v", found, err)
|
||||
}
|
||||
|
||||
@@ -269,7 +269,7 @@ func bucketsOf(m catalogue.Manifest) []broker.Bucket {
|
||||
var out []broker.Bucket
|
||||
for _, s := range m.State {
|
||||
out = append(out, broker.Bucket{Module: m.Module, Name: s.Name, History: s.History, TTLSeconds: s.TTLSeconds,
|
||||
PerMachine: s.PerMachine})
|
||||
PerMachine: s.PerMachine, KeyTTL: s.KeyTTL})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -258,31 +258,17 @@ func (i *Inventory) SourceOf(ctx context.Context, module string) (Source, error)
|
||||
// machines are running last week's version" is the question somebody actually has, and it is the
|
||||
// one novox/hq ADR 0010 names as the thing that must not be lost.
|
||||
func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) {
|
||||
query := `select m.name, coalesce(n.name, '')
|
||||
from module m
|
||||
left join assignment a on a.module = m.name
|
||||
left join node n on n.id = a.node
|
||||
where m.source is not null
|
||||
and m.source_head is not null
|
||||
and coalesce(m.built_from, '') is distinct from m.source_head
|
||||
order by m.name, n.name`
|
||||
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
|
||||
assigned, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var assigned []assignRow
|
||||
if onTheBus {
|
||||
query = `select m.name, ''
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select m.name
|
||||
from module m
|
||||
where m.source is not null
|
||||
and m.source_head is not null
|
||||
and coalesce(m.built_from, '') is distinct from m.source_head
|
||||
order by m.name`
|
||||
if assigned, err = i.assignments(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx, query)
|
||||
order by m.name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -290,16 +276,11 @@ func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) {
|
||||
|
||||
out := map[string][]string{}
|
||||
for rows.Next() {
|
||||
var module, node string
|
||||
if err := rows.Scan(&module, &node); err != nil {
|
||||
var module string
|
||||
if err := rows.Scan(&module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, seen := out[module]; !seen {
|
||||
out[module] = nil
|
||||
}
|
||||
if node != "" {
|
||||
out[module] = append(out[module], node)
|
||||
}
|
||||
out[module] = nil
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
@@ -654,21 +635,11 @@ func (i *Inventory) Assign(ctx context.Context, nodeName, module string) (bool,
|
||||
if err := i.runsSomewhere(ctx, module); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
|
||||
return false, err
|
||||
} else if on {
|
||||
return i.chainAssign(ctx, c, node, module)
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`insert into assignment (node, module) values ($1, $2) on conflict do nothing`,
|
||||
node.ID, module)
|
||||
if err != nil && strings.Contains(err.Error(), "assignment_module_fkey") {
|
||||
return false, fmt.Errorf("%w: %s", ErrNoSuchModule, module)
|
||||
}
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() > 0, nil
|
||||
return i.chainAssign(ctx, c, node, module)
|
||||
}
|
||||
|
||||
// Unassign takes a module off a node.
|
||||
@@ -677,21 +648,12 @@ func (i *Inventory) Unassign(ctx context.Context, nodeName, module string) error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := i.chainUnassign(ctx, c, node, module); err != nil {
|
||||
return err
|
||||
} else if on {
|
||||
if err := i.chainUnassign(ctx, c, node, module); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`delete from assignment where node = $1 and module = $2`, node.ID, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%s is not assigned to %s", module, nodeName)
|
||||
}
|
||||
}
|
||||
// The seat it held falls back to derivation rather than pointing at nothing: what the store's foreign key did.
|
||||
if _, err := i.store.Pool().Exec(ctx, `delete from seat_holding where node = $1 and module = $2`, node.ID, module); err != nil {
|
||||
@@ -1362,39 +1324,21 @@ type Entry struct {
|
||||
// catalogue nobody lists, and the questions here — what is this, where did it come from, who is
|
||||
// running it — are asked together every time.
|
||||
func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
|
||||
query := `select m.name, m.manifest,
|
||||
coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''),
|
||||
coalesce(m.built_from, ''), coalesce(m.source_head, ''),
|
||||
coalesce(m.source_seen, to_timestamp(0)),
|
||||
coalesce(array_agg(n.name order by n.name) filter (where n.name is not null), '{}')
|
||||
from module m
|
||||
left join assignment a on a.module = m.name
|
||||
left join node n on n.id = a.node
|
||||
group by m.name, m.manifest, m.source, m.source_path, m.source_seat, m.ref, m.built_from,
|
||||
m.source_head, m.source_seen
|
||||
order by m.name`
|
||||
// On the bus the assignments are read from the bucket, and the machines of a module put beside it here.
|
||||
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
|
||||
// The machines of a module are read from the assignments bucket and put beside it here.
|
||||
assigned, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var assignedBy map[string][]string
|
||||
if onTheBus {
|
||||
query = `select m.name, m.manifest,
|
||||
assignedBy := map[string][]string{}
|
||||
for _, a := range assigned {
|
||||
assignedBy[a.Module] = append(assignedBy[a.Module], a.Node)
|
||||
}
|
||||
const query = `select m.name, m.manifest,
|
||||
coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''),
|
||||
coalesce(m.built_from, ''), coalesce(m.source_head, ''),
|
||||
coalesce(m.source_seen, to_timestamp(0)), '{}'::text[]
|
||||
from module m
|
||||
order by m.name`
|
||||
rows, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
assignedBy = map[string][]string{}
|
||||
for _, a := range rows {
|
||||
assignedBy[a.Module] = append(assignedBy[a.Module], a.Node)
|
||||
}
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1414,9 +1358,7 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
|
||||
if source.Seen.Unix() == 0 {
|
||||
source.Seen = time.Time{}
|
||||
}
|
||||
if onTheBus {
|
||||
on = assignedBy[name]
|
||||
}
|
||||
on = assignedBy[name]
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, err
|
||||
|
||||
+76
-94
@@ -5,55 +5,56 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **The chain moves to the bus kind by kind** (novox/hq ADR 0408 §8, issue 981). The tables of builds, rollouts and
|
||||
// assignments are copied once into the chain's buckets by the serving controller at its start (MoveChain), which then
|
||||
// records each kind it moved in chain_moved (migration 0104). **Until a kind is recorded there nothing reads its bucket;
|
||||
// once it is, nothing reads its tables**: ChainReads answers which, and every method of a kind asks it first.
|
||||
|
||||
// The kinds of the chain, as chain_moved names them.
|
||||
const (
|
||||
KindBuilds = "builds"
|
||||
KindRollouts = "rollouts"
|
||||
)
|
||||
// **The chain lives on the bus** (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
|
||||
// bus's key-value store, the controller's store keeps none of them, and every method of a kind reads and writes the chain this
|
||||
// inventory is given (theChain).
|
||||
|
||||
// testChains are the chains in memory a test made for its store, by the store's address: every inventory opened on that
|
||||
// store is given the same chain, as every process of a mesh is on the same bus. Empty outside a test.
|
||||
var testChains sync.Map
|
||||
|
||||
// ErrNoChain is a kind moved to the bus read by an inventory that has no chain attached: a process that did not
|
||||
// open the bus. Refused loudly, never answered from the tables the move left behind.
|
||||
// ErrNoChain is a read or write of the chain by an inventory that has none attached: a process that did not open the bus.
|
||||
// Refused loudly.
|
||||
var ErrNoChain = errors.New("this kind of record lives on the bus, and this process has not opened the bus")
|
||||
|
||||
// movedKinds is which kinds this process has read as moved: once moved, always moved, so it is read from the store
|
||||
// until it is true and not after.
|
||||
type movedKinds struct {
|
||||
// chainState is what an inventory keeps of the chain it is given.
|
||||
type chainState struct {
|
||||
mu sync.Mutex
|
||||
kind map[string]bool
|
||||
open ChainOpener
|
||||
// view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that
|
||||
// is not saved yet.
|
||||
view *rolloutView
|
||||
pending map[string]pendingMerge
|
||||
// switching is set when the assignments begin to be switched to the bus and kept once they are: the store's mirror of
|
||||
// them writes nothing from then on (AssignmentsMirrorMayWrite). Cleared only if the switch fails.
|
||||
switching atomic.Bool
|
||||
// requireLayout is set for a chain on the bus: its builds are read only once the controller's start has laid them out
|
||||
// under their modules (issue 1121), and laid is that seen.
|
||||
requireLayout, laid bool
|
||||
}
|
||||
|
||||
// AssignmentsMirrorMayWrite says the store's mirror of the assignments may still write the bucket: the assignments are
|
||||
// neither being switched to the bus nor on it. Asked before every write of the mirror's writer, not once a pass.
|
||||
func (i *Inventory) AssignmentsMirrorMayWrite() bool {
|
||||
i.moved.mu.Lock()
|
||||
moved := i.moved.kind[KindAssignments]
|
||||
i.moved.mu.Unlock()
|
||||
return !moved && !i.moved.switching.Load()
|
||||
// RequireLayout makes every read of the chain refuse (ErrNotLaidOut) until the builds are laid out under their modules.
|
||||
func (i *Inventory) RequireLayout() {
|
||||
i.state.mu.Lock()
|
||||
i.state.requireLayout = true
|
||||
i.state.mu.Unlock()
|
||||
}
|
||||
|
||||
// layingOut lets one read at a time lay the builds out; the others wait for it and then find the marker.
|
||||
var layingOut sync.Mutex
|
||||
|
||||
// layOutOnce lays the builds out under their modules unless another read did so meanwhile.
|
||||
func (i *Inventory) layOutOnce(ctx context.Context) error {
|
||||
layingOut.Lock()
|
||||
defer layingOut.Unlock()
|
||||
return i.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) })
|
||||
}
|
||||
|
||||
func (i *Inventory) markLaidOut() {
|
||||
i.state.mu.Lock()
|
||||
i.state.laid = true
|
||||
i.state.mu.Unlock()
|
||||
}
|
||||
|
||||
// ChainOpener opens the chain on the bus when a process first needs it: a command that reads and never moved anything has
|
||||
@@ -62,88 +63,69 @@ type ChainOpener func(ctx context.Context) (*chain.Chain, error)
|
||||
|
||||
// UseChainOpener gives the inventory the means to open the chain when it needs it and none is attached.
|
||||
func (i *Inventory) UseChainOpener(open ChainOpener) {
|
||||
i.moved.mu.Lock()
|
||||
i.moved.open = open
|
||||
i.moved.mu.Unlock()
|
||||
i.state.mu.Lock()
|
||||
i.state.open = open
|
||||
i.state.requireLayout = true
|
||||
i.state.mu.Unlock()
|
||||
}
|
||||
|
||||
// UseChain attaches the chain the moved kinds are read and written through. The serving controller attaches the bus's,
|
||||
// holding the lease for its writes; a test attaches one in memory.
|
||||
func (i *Inventory) UseChain(c *chain.Chain) {
|
||||
i.moved.mu.Lock()
|
||||
defer i.moved.mu.Unlock()
|
||||
i.state.mu.Lock()
|
||||
defer i.state.mu.Unlock()
|
||||
i.chain = c
|
||||
if i.moved.view != nil {
|
||||
i.moved.view.stop()
|
||||
i.moved.view = nil
|
||||
if i.state.view != nil {
|
||||
i.state.view.stop()
|
||||
i.state.view = nil
|
||||
}
|
||||
}
|
||||
|
||||
// UseThrowawayChain gives the inventory a chain in memory that is its own, laid out already: for a mesh raised for a moment
|
||||
// beside the real one, as the merge gate raises the facts snapshot's mesh in a throwaway store (issue 1121). Nothing of it
|
||||
// reaches the bus.
|
||||
func (i *Inventory) UseThrowawayChain() {
|
||||
i.UseChain(chain.InMemory(nil))
|
||||
i.markLaidOut()
|
||||
}
|
||||
|
||||
// attached is the chain this inventory has, nil when none.
|
||||
func (i *Inventory) attached() *chain.Chain {
|
||||
i.moved.mu.Lock()
|
||||
defer i.moved.mu.Unlock()
|
||||
i.state.mu.Lock()
|
||||
defer i.state.mu.Unlock()
|
||||
return i.chain
|
||||
}
|
||||
|
||||
// ChainReads says whether a kind of the chain is read from its bucket: chain_moved names it. It answers the chain to read
|
||||
// through, and ErrNoChain when the kind is moved and none is attached.
|
||||
func (i *Inventory) ChainReads(ctx context.Context, kind string) (*chain.Chain, bool, error) {
|
||||
i.moved.mu.Lock()
|
||||
known := i.moved.kind[kind]
|
||||
i.moved.mu.Unlock()
|
||||
if !known {
|
||||
var moved bool
|
||||
err := i.store.Pool().QueryRow(ctx, `select counts ? $1 from chain_moved`, kind).Scan(&moved)
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.Is(err, pgx.ErrNoRows) || (errors.As(err, &pgErr) && pgErr.Code == "42P01") {
|
||||
// No row, or a store older than the migration that makes the table: nothing moved.
|
||||
return nil, false, nil
|
||||
}
|
||||
// theChain is the chain this inventory writes and reads, opened if a command has not yet. On the bus it is refused until the
|
||||
// builds are laid out under their modules.
|
||||
func (i *Inventory) theChain(ctx context.Context) (*chain.Chain, error) {
|
||||
i.state.mu.Lock()
|
||||
if i.chain == nil && i.state.open != nil {
|
||||
c, err := i.state.open(ctx)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("whether %s moved to the bus cannot be read: %w", kind, err)
|
||||
}
|
||||
if !moved {
|
||||
return nil, false, nil
|
||||
}
|
||||
i.moved.mu.Lock()
|
||||
if i.moved.kind == nil {
|
||||
i.moved.kind = map[string]bool{}
|
||||
}
|
||||
i.moved.kind[kind] = true
|
||||
i.moved.mu.Unlock()
|
||||
}
|
||||
i.moved.mu.Lock()
|
||||
defer i.moved.mu.Unlock()
|
||||
if i.chain == nil && i.moved.open != nil {
|
||||
c, err := i.moved.open(ctx)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%w: %s: %v", ErrNoChain, kind, err)
|
||||
i.state.mu.Unlock()
|
||||
return nil, fmt.Errorf("%w: %v", ErrNoChain, err)
|
||||
}
|
||||
i.chain = c
|
||||
}
|
||||
if i.chain == nil {
|
||||
return nil, false, fmt.Errorf("%w: %s", ErrNoChain, kind)
|
||||
c, check := i.chain, i.state.requireLayout && !i.state.laid
|
||||
i.state.mu.Unlock()
|
||||
if c == nil {
|
||||
return nil, ErrNoChain
|
||||
}
|
||||
return i.chain, true, nil
|
||||
}
|
||||
|
||||
// ChainMoved is, per kind moved, how many records the move wrote; empty before any is.
|
||||
func (i *Inventory) ChainMoved(ctx context.Context) (map[string]int, error) {
|
||||
var counts map[string]int
|
||||
err := i.store.Pool().QueryRow(ctx, `select counts from chain_moved`).Scan(&counts)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return map[string]int{}, nil
|
||||
}
|
||||
return counts, err
|
||||
}
|
||||
|
||||
// NewForTest is an inventory with no store, for a test of what needs none.
|
||||
func NewForTest() *Inventory { return &Inventory{moved: &movedKinds{}} }
|
||||
|
||||
// BeginSwitchForTest sets the flag a kind's switch sets, as a switch under way does.
|
||||
func (i *Inventory) BeginSwitchForTest(kind string) {
|
||||
if kind == KindAssignments {
|
||||
i.moved.switching.Store(true)
|
||||
if check {
|
||||
if _, found, err := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); err != nil {
|
||||
return nil, err
|
||||
} else if !found {
|
||||
// **The first read lays them out** (issue 1121): on 2026-10-18 a read of the controller's start came before
|
||||
// the layout and was refused, and the controller exited and was started again every five seconds. A
|
||||
// process that may write (the lease holder) lays them out here, once; one that may not is refused, loudly.
|
||||
if err := i.layOutOnce(ctx); err != nil {
|
||||
return nil, fmt.Errorf("%w (laying them out now failed: %v)", ErrNotLaidOut, err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
i.markLaidOut()
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
// here (chain.Assignment.Stands). No `pending` state is written yet (issue 1094).
|
||||
|
||||
// KindAssignments is the kind of the chain that is each module assigned to a node, with the build it runs there.
|
||||
const KindAssignments = "assignments"
|
||||
|
||||
func assignmentKey(node, module string) string { return node + "." + module }
|
||||
|
||||
@@ -31,32 +30,15 @@ type assignRow struct {
|
||||
Assigned time.Time
|
||||
}
|
||||
|
||||
// assignments is every assignment, by node then module: from the bucket when the kind is moved, else from the store.
|
||||
// assignments is every assignment, by node then module, read from the assignments bucket.
|
||||
// A key whose value names no time assigned is a module only pulled in by another (issue 765), and one that has ended or
|
||||
// waits in a state is gone: no assignment. One on a machine the store does not know is none either, as the store took a
|
||||
// removed machine's.
|
||||
func (i *Inventory) assignments(ctx context.Context) ([]assignRow, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !on {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node order by n.name, a.module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []assignRow
|
||||
for rows.Next() {
|
||||
var a assignRow
|
||||
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the assignments cannot be read from the bus: %w", err)
|
||||
@@ -172,10 +154,12 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node,
|
||||
switch {
|
||||
case found && cur.Value.Stands():
|
||||
// A module only pulled in by another becomes an assignment, keeping what it holds.
|
||||
cur.Value.Assigned = now
|
||||
cur.Value.Assigned, cur.Value.PulledIn = now, false
|
||||
_, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision)
|
||||
case found:
|
||||
// One that ended is assigned again: a fresh value over it, the ended one staying in history.
|
||||
// One that ended is assigned again, and one that was pending is made: a fresh value over it, the earlier one
|
||||
// staying in history. The wait is carried into the value that stands (issue 1094).
|
||||
fresh.Pending = carriedPending(cur.Value, now)
|
||||
_, err = c.Assignments.Update(ctx, key, fresh, cur.Revision)
|
||||
default:
|
||||
_, err = c.Assignments.Create(ctx, key, fresh)
|
||||
@@ -253,8 +237,8 @@ func (i *Inventory) chainAwaitingSince(ctx context.Context, name string) (time.T
|
||||
// and rollout another write put in the same value, and never ending an assignment. A plan the provisioner's rules refuse is
|
||||
// refused by name and nothing is written (AssignmentValues).
|
||||
func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed int, err error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
var plans map[string]Provisioning
|
||||
@@ -289,11 +273,11 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i
|
||||
case keepSkip:
|
||||
continue
|
||||
case keepCreate:
|
||||
next.Assigned = time.Time{}
|
||||
next.Assigned, next.PulledIn = time.Time{}, true
|
||||
_, err = c.Assignments.Create(ctx, key, next)
|
||||
case keepFresh:
|
||||
// A value that ended, pulled in again: a fresh one over it, the ended one staying in history.
|
||||
next.Assigned = time.Time{}
|
||||
next.Assigned, next.PulledIn = time.Time{}, true
|
||||
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
|
||||
default:
|
||||
cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning
|
||||
@@ -308,8 +292,8 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i
|
||||
written++
|
||||
}
|
||||
for key, cur := range have {
|
||||
if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() {
|
||||
continue
|
||||
if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() || cur.Value.Commit != "" {
|
||||
continue // one a send still carries is on the machine, and ends when a send does not carry it any more
|
||||
}
|
||||
// A value of a module no longer pulled in, which no assignment names: it ends, and stays.
|
||||
cur.Value.Ended = time.Now().UTC()
|
||||
@@ -340,17 +324,16 @@ type KeptValue struct {
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// KeptAssignments is every key of the assignments bucket that holds a value that stands, and false when the assignments are
|
||||
// not on the bus.
|
||||
// What the serving controller compares from one look to the next to say a key changed (`assignment-changed`).
|
||||
func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, bool, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on {
|
||||
return nil, false, err
|
||||
// KeptAssignments is every key of the assignments bucket that holds a value that stands. What the serving controller
|
||||
// compares from one look to the next to say a key changed (`assignment-changed`).
|
||||
func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]KeptValue, len(all))
|
||||
for key, k := range all {
|
||||
@@ -359,11 +342,11 @@ func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue,
|
||||
}
|
||||
body, err := json.Marshal(k.Value)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
return nil, err
|
||||
}
|
||||
out[key] = KeptValue{Value: body, Revision: k.Revision}
|
||||
}
|
||||
return out, true, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// What a pass that keeps the plans does with one key.
|
||||
@@ -384,6 +367,8 @@ func keepStep(has bool, cur, next chain.Assignment) string {
|
||||
return keepSkip // assigned since the bucket was read: its own write is not here yet
|
||||
case !has:
|
||||
return keepCreate
|
||||
case cur.State == chain.StatePending:
|
||||
return keepSkip // a wait for a build is not pulled in by a plan
|
||||
case !cur.Stands():
|
||||
if !cur.Assigned.IsZero() || !next.Assigned.IsZero() {
|
||||
return keepSkip // an assignment that ended is not pulled in again by a plan
|
||||
@@ -394,3 +379,27 @@ func keepStep(has bool, cur, next chain.Assignment) string {
|
||||
}
|
||||
return keepField
|
||||
}
|
||||
|
||||
// carriedPending is the wait a value that does not stand carries into the assignment made over it: a wait that was applying
|
||||
// stays applying, for the one making it settles it applied; one still waiting is made by hand, and says so. Nothing for a
|
||||
// value that held no wait, or whose key is not one that was pending.
|
||||
func carriedPending(was chain.Assignment, now time.Time) *chain.Pending {
|
||||
if was.Pending == nil || was.State != chain.StatePending {
|
||||
return nil
|
||||
}
|
||||
p := *was.Pending
|
||||
if p.State == chain.PendingWaiting {
|
||||
p.State, p.Note, p.Settled = chain.PendingApplied, "assigned by hand", &now
|
||||
}
|
||||
return &p
|
||||
}
|
||||
|
||||
// AssignmentValue is one key of the assignments bucket as it is held, standing or not: one read of an assignment.
|
||||
func (i *Inventory) AssignmentValue(ctx context.Context, node, module string) (chain.Assignment, bool, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return chain.Assignment{}, false, err
|
||||
}
|
||||
k, found, err := c.Assignments.Get(ctx, assignmentKey(node, module))
|
||||
return k.Value, found, err
|
||||
}
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -51,31 +49,24 @@ func TestAPassThatKeepsThePlansNeverRevivesAnEndedAssignment(t *testing.T) {
|
||||
ended := chain.Assignment{Node: "a", Module: "m", Assigned: at, Ended: at.Add(time.Minute)}
|
||||
pulledIn := chain.Assignment{Node: "a", Module: "p", RequiredBy: "m", Provisioning: plan}
|
||||
pulledInEnded := chain.Assignment{Node: "a", Module: "p", Ended: at}
|
||||
waiting := chain.Assignment{Node: "a", Module: "p", State: chain.StatePending,
|
||||
Pending: &chain.Pending{ID: "p-1", State: chain.PendingWaiting}}
|
||||
for name, tc := range map[string]struct {
|
||||
has bool
|
||||
cur, next chain.Assignment
|
||||
want string
|
||||
}{
|
||||
"the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip},
|
||||
"wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip},
|
||||
"a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate},
|
||||
"a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh},
|
||||
"an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip},
|
||||
"the plan is the same": {true, assigned, assigned, keepSkip},
|
||||
"the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField},
|
||||
"the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip},
|
||||
"wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip},
|
||||
"a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate},
|
||||
"a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh},
|
||||
"an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip},
|
||||
"a wait for a build, a plan for the module pulled in": {true, waiting, pulledIn, keepSkip},
|
||||
"the plan is the same": {true, assigned, assigned, keepSkip},
|
||||
"the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField},
|
||||
} {
|
||||
if got := keepStep(tc.has, tc.cur, tc.next); got != tc.want {
|
||||
t.Errorf("%s: %s, want %s", name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The store's mirror of the assignments is refused to its writer once the assignments are moved** (ADR 0408 §4, issue
|
||||
// 1093): asked at the moment of the write, so a tick that decided before the switch lays nothing after it.
|
||||
func TestTheStoresMirrorOfTheAssignmentsIsRefusedToItsWriterOnceTheyAreMoved(t *testing.T) {
|
||||
inv := &Inventory{moved: &movedKinds{kind: map[string]bool{KindAssignments: true}}}
|
||||
inv.UseChain(chain.InMemory(nil))
|
||||
if _, err := inv.AssignmentsMirrorToWrite(context.Background()); !errors.Is(err, ErrAssignmentsMoved) {
|
||||
t.Fatalf("the writer was given the store's rows: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,461 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// issue 1121: a build is under its module, so a lookup reads the keys it needs and no others.
|
||||
|
||||
type readCounting struct {
|
||||
chain.Backend
|
||||
gets, watches atomic.Int32
|
||||
}
|
||||
|
||||
func (r *readCounting) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
|
||||
r.gets.Add(1)
|
||||
return r.Backend.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (r *readCounting) Watch(ctx context.Context) (<-chan chain.Entry, error) {
|
||||
r.watches.Add(1)
|
||||
return r.Backend.Watch(ctx)
|
||||
}
|
||||
|
||||
func (r *readCounting) WatchFiltered(ctx context.Context, f string) (<-chan chain.Entry, error) {
|
||||
r.watches.Add(1)
|
||||
return r.Backend.WatchFiltered(ctx, f)
|
||||
}
|
||||
|
||||
func layoutChain(builds chain.Backend) (*Inventory, *chain.Chain) {
|
||||
c := chain.New(builds, chain.NewMemory(), chain.NewMemory(), func(context.Context) error { return nil }).
|
||||
WithManifests(builds)
|
||||
inv := &Inventory{state: &chainState{}}
|
||||
inv.UseChain(c)
|
||||
return inv, c
|
||||
}
|
||||
|
||||
var layoutAt = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func layoutBuild(module string, n int, commit, source string) Build {
|
||||
return Build{ID: fmt.Sprintf("build-%d", n), Repository: "novox/" + module, Module: module, Commit: commit,
|
||||
SourceFingerprint: source, ReuseKey: "v1:" + source, Made: []Artifact{{Name: module, Kind: "image", Reference: fmt.Sprintf("r/%s@sha256:%d", module, n)}},
|
||||
Manifest: []byte(fmt.Sprintf(`{"module":%q,"n":%d}`, module, n)), At: layoutAt.Add(time.Duration(n) * time.Minute)}
|
||||
}
|
||||
|
||||
func TestALookupOfOneModuleReadsThatModulesKeysOnly(t *testing.T) {
|
||||
mem := chain.NewMemory()
|
||||
inv, c := layoutChain(mem)
|
||||
ctx := context.Background()
|
||||
for n := 1; n <= 3; n++ {
|
||||
for _, m := range []string{"alpha", "beta"} {
|
||||
if err := inv.chainRecordBuild(ctx, c, layoutBuild(m, n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
got, err := inv.chainBuildList(ctx, c, "alpha", 10)
|
||||
if err != nil || len(got) != 3 {
|
||||
t.Fatalf("alpha's builds %+v: %v", got, err)
|
||||
}
|
||||
if want := map[string]int{"alpha.build.>": 1}; !reflect.DeepEqual(mem.Filtered(), want) {
|
||||
t.Fatalf("the list of one module watched %v; want only %v", mem.Filtered(), want)
|
||||
}
|
||||
if _, err := inv.chainBuildFingerprints(ctx, c, "alpha"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for f := range mem.Filtered() {
|
||||
if f != "alpha.build.>" && f != "alpha.commit.*" {
|
||||
t.Errorf("a lookup of alpha watched %q", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReuseBySourceFingerprintAndTheBuildOfACommitAreOneGet(t *testing.T) {
|
||||
counting := &readCounting{Backend: chain.NewMemory()}
|
||||
inv, c := layoutChain(counting)
|
||||
ctx := context.Background()
|
||||
for n := 1; n <= 20; n++ {
|
||||
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
counting.gets.Store(0)
|
||||
counting.watches.Store(0)
|
||||
id, err := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s7")
|
||||
if err != nil || id != "build-7" {
|
||||
t.Fatalf("source s7 is %q: %v", id, err)
|
||||
}
|
||||
if counting.gets.Load() != 2 || counting.watches.Load() != 0 {
|
||||
t.Fatalf("the reuse lookup made %d gets and %d watches; want 2 (the pointer, the record it names) and 0", counting.gets.Load(), counting.watches.Load())
|
||||
}
|
||||
counting.gets.Store(0)
|
||||
if id, err = inv.chainBuildOf(ctx, c, "alpha", "c19"); err != nil || id != "build-19" {
|
||||
t.Fatalf("commit c19 is %q: %v", id, err)
|
||||
}
|
||||
if counting.gets.Load() != 2 || counting.watches.Load() != 0 {
|
||||
t.Fatalf("the build of a commit made %d gets and %d watches; want 2 and 0", counting.gets.Load(), counting.watches.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildWrittenIsFoundByCommitSourceIDAndRequest(t *testing.T) {
|
||||
inv, c := layoutChain(chain.NewMemory())
|
||||
ctx := context.Background()
|
||||
ask := BuildRequest{ID: "build-5", Repository: "novox/alpha", Seat: "mesh-build-machine", At: layoutAt}
|
||||
if err := inv.chainRecordBuildRequest(ctx, c, ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, found, _ := inv.chainBuildByID(ctx, c, "build-5"); found {
|
||||
t.Fatal("a request nobody answered was read as a build")
|
||||
}
|
||||
b := layoutBuild("alpha", 5, "c5", "s5")
|
||||
if err := inv.chainRecordBuild(ctx, c, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, _ := inv.chainBuildOf(ctx, c, "alpha", "c5"); id != "build-5" {
|
||||
t.Errorf("by commit: %q", id)
|
||||
}
|
||||
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s5"); id != "build-5" {
|
||||
t.Errorf("by source: %q", id)
|
||||
}
|
||||
got, found, err := inv.chainBuildByID(ctx, c, "build-5")
|
||||
if err != nil || !found || got.Module != "alpha" {
|
||||
t.Fatalf("by id: %+v %v %v", got, found, err)
|
||||
}
|
||||
// The manifest is kept once, apart from the record, and read back whole.
|
||||
raw, err := inv.chainManifestOf(ctx, c, "build-5")
|
||||
if err != nil || !json.Valid(raw) || string(raw) != string(canonicalJSON(b.Manifest)) {
|
||||
t.Fatalf("manifest %q: %v", raw, err)
|
||||
}
|
||||
k, _, _ := c.Builds.Get(ctx, "alpha.build.build-5")
|
||||
if len(k.Value.Manifest) != 0 || k.Value.ManifestDigest == "" {
|
||||
t.Errorf("the record keeps the manifest itself: %+v", k.Value)
|
||||
}
|
||||
if k.Value.Request == nil {
|
||||
t.Error("the outcome lost the request it answers")
|
||||
}
|
||||
reqs, err := inv.chainRequests(ctx, c)
|
||||
if err != nil || len(reqs) != 1 || !reqs[0].Heard || reqs[0].Module != "alpha" {
|
||||
t.Fatalf("requests %+v: %v", reqs, err)
|
||||
}
|
||||
}
|
||||
|
||||
func legacyBuild(n int, module, commit string) chain.Build {
|
||||
b := toChainBuild(layoutBuild(module, n, commit, "s"+commit))
|
||||
return b
|
||||
}
|
||||
|
||||
func keysOf(t *testing.T, mem chain.Backend) []string {
|
||||
t.Helper()
|
||||
keys, err := mem.Keys(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// The layout of the bucket's keys is made once from the keys of before it: run again it changes nothing, the old keys stay,
|
||||
// and the pointers say the newest.
|
||||
func TestTheLayoutMigrationIsIdempotentAndKeepsTheOldKeys(t *testing.T) {
|
||||
mem := chain.NewMemory()
|
||||
inv, c := layoutChain(mem)
|
||||
ctx := context.Background()
|
||||
for n, m := range []string{"alpha", "alpha", "beta", "alpha"} {
|
||||
old := legacyBuild(n+1, m, fmt.Sprintf("c%d", n+1))
|
||||
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
failed := legacyBuild(9, "alpha", "c9")
|
||||
failed.Failed = "broke"
|
||||
failed.SourceFingerprint = ""
|
||||
if _, err := c.Builds.Create(ctx, failed.ID, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked := chain.Build{ID: "build-10", State: chain.BuildAsked, Repository: "novox/alpha", Asked: layoutAt,
|
||||
Request: &chain.Request{Repository: "novox/alpha", At: layoutAt}}
|
||||
if _, err := c.Builds.Create(ctx, asked.ID, asked); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := inv.layoutBuilds(ctx, c, nil)
|
||||
if err != nil || first.Written == 0 {
|
||||
t.Fatalf("first run %+v: %v", first, err)
|
||||
}
|
||||
keys := keysOf(t, mem)
|
||||
second, err := inv.layoutBuilds(ctx, c, nil)
|
||||
if err != nil || second.Written != 0 {
|
||||
t.Fatalf("second run %+v: %v", second, err)
|
||||
}
|
||||
if again := keysOf(t, mem); !reflect.DeepEqual(keys, again) {
|
||||
t.Fatalf("keys changed on the second run:\n%v\n%v", keys, again)
|
||||
}
|
||||
has := func(k string) bool { return containsKey(keys, k) }
|
||||
for _, k := range []string{"build-1", "build-9", "alpha.build.build-1", "alpha.build.build-4", "beta.build.build-3", "alpha.build.build-9", "req.build-10",
|
||||
"id.build-3", "alpha.newest", "alpha.worked", "alpha.held", "alpha.commit.c4", "alpha.source.v1-sc4", "beta.worked"} {
|
||||
if !has(k) {
|
||||
t.Errorf("no key %s in %v", k, keys)
|
||||
}
|
||||
}
|
||||
if e, _, _ := mem.Get(ctx, "alpha.newest"); string(e.Value) != "build-9" {
|
||||
t.Errorf("alpha.newest = %q; want the failed build-9", e.Value)
|
||||
}
|
||||
if e, _, _ := mem.Get(ctx, "alpha.worked"); string(e.Value) != "build-4" {
|
||||
t.Errorf("alpha.worked = %q; want build-4", e.Value)
|
||||
}
|
||||
if id, _ := inv.chainBuildOf(ctx, c, "alpha", "c4"); id != "build-4" {
|
||||
t.Errorf("by commit after the layout: %q", id)
|
||||
}
|
||||
// No record is read from an old key.
|
||||
list, err := inv.chainBuildList(ctx, c, "alpha", 10)
|
||||
if err != nil || len(list) != 4 {
|
||||
t.Fatalf("alpha's builds after the layout: %d %v", len(list), err)
|
||||
}
|
||||
}
|
||||
|
||||
func containsKey(s []string, k string) bool {
|
||||
for _, x := range s {
|
||||
if x == k {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The manifest is kept once under its digest in the same bucket, and the moved record keeps none.
|
||||
func TestAMovedRecordHoldsNoManifestAndItsManifestIsUnderItsDigest(t *testing.T) {
|
||||
mem := chain.NewMemory()
|
||||
inv, c := layoutChain(mem)
|
||||
ctx := context.Background()
|
||||
old := legacyBuild(1, "alpha", "c1")
|
||||
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, found, err := c.Builds.Get(ctx, "alpha.build.build-1")
|
||||
if err != nil || !found || len(k.Value.Manifest) != 0 || k.Value.ManifestDigest == "" {
|
||||
t.Fatalf("moved record %+v %v %v", k.Value, found, err)
|
||||
}
|
||||
if !strings.HasPrefix(k.Value.ManifestDigest, "manifest.sha256-") {
|
||||
t.Errorf("the digest %q is not a key under manifest.", k.Value.ManifestDigest)
|
||||
}
|
||||
e, found, _ := mem.Get(ctx, k.Value.ManifestDigest)
|
||||
if !found || string(e.Value) != string(canonicalJSON([]byte(old.Manifest))) {
|
||||
t.Errorf("the manifest under its digest is %q (found %v)", e.Value, found)
|
||||
}
|
||||
if got := chain.ManifestKey(e.Value); got != k.Value.ManifestDigest {
|
||||
t.Errorf("the digest does not name the bytes: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A pointer is a hint: a record written without its pointers is found by reading the module's records, and the pointers are
|
||||
// repaired; a pointer that names a record that is not what it was for is not followed.
|
||||
func TestAMissingOrWrongPointerFallsBackToTheModulesRecordsAndIsRepaired(t *testing.T) {
|
||||
mem := chain.NewMemory()
|
||||
inv, c := layoutChain(mem)
|
||||
ctx := context.Background()
|
||||
for n := 1; n <= 3; n++ {
|
||||
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A record whose pointers the writer never made (a crash between the record and its pointers).
|
||||
lost := toChainBuild(layoutBuild("alpha", 4, "c4", "s4"))
|
||||
if _, err := c.Builds.Create(ctx, "alpha.build.build-4", lost); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, err := inv.chainBuildOf(ctx, c, "alpha", "c4"); err != nil || id != "build-4" {
|
||||
t.Fatalf("the build of c4 is %q: %v", id, err)
|
||||
}
|
||||
if e, found, _ := mem.Get(ctx, "alpha.commit.c4"); !found || pointedID(e.Value) != "build-4" {
|
||||
t.Errorf("the commit pointer was not repaired: %q %v", e.Value, found)
|
||||
}
|
||||
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s4"); id != "build-4" {
|
||||
t.Errorf("the source pointer was not repaired: %q", id)
|
||||
}
|
||||
// A pointer to the wrong build is not followed.
|
||||
e, _, _ := mem.Get(ctx, "alpha.commit.c2")
|
||||
wrong, _ := json.Marshal(chain.CommitPointer{Build: "build-1"})
|
||||
if _, err := mem.Update(ctx, "alpha.commit.c2", wrong, e.Revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id, err := inv.chainBuildOf(ctx, c, "alpha", "c2"); err != nil || id != "build-2" {
|
||||
t.Fatalf("with a wrong pointer the build of c2 is %q: %v", id, err)
|
||||
}
|
||||
if e, _, _ := mem.Get(ctx, "alpha.commit.c2"); pointedID(e.Value) != "build-2" {
|
||||
t.Errorf("the wrong pointer stayed: %s", e.Value)
|
||||
}
|
||||
// A commit nothing built is none.
|
||||
if id, err := inv.chainBuildOf(ctx, c, "alpha", "nope"); err != nil || id != "" {
|
||||
t.Errorf("an unbuilt commit is %q: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
type touchRecording struct {
|
||||
chain.Backend
|
||||
mu sync.Mutex
|
||||
touched []string
|
||||
}
|
||||
|
||||
func (r *touchRecording) note(s string) { r.mu.Lock(); r.touched = append(r.touched, s); r.mu.Unlock() }
|
||||
func (r *touchRecording) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
|
||||
r.note("get " + key)
|
||||
return r.Backend.Get(ctx, key)
|
||||
}
|
||||
func (r *touchRecording) Watch(ctx context.Context) (<-chan chain.Entry, error) {
|
||||
r.note("watch >")
|
||||
return r.Backend.Watch(ctx)
|
||||
}
|
||||
func (r *touchRecording) WatchFiltered(ctx context.Context, f string) (<-chan chain.Entry, error) {
|
||||
r.note("watch " + f)
|
||||
return r.Backend.WatchFiltered(ctx, f)
|
||||
}
|
||||
func (r *touchRecording) Keys(ctx context.Context) ([]string, error) {
|
||||
r.note("keys")
|
||||
return r.Backend.Keys(ctx)
|
||||
}
|
||||
|
||||
// Once the layout is made, no read touches a key that is a build's id alone, or the bucket whole.
|
||||
func TestAfterTheLayoutNoReadTouchesABareBuildIDKey(t *testing.T) {
|
||||
rec := &touchRecording{Backend: chain.NewMemory()}
|
||||
inv, c := layoutChain(rec)
|
||||
ctx := context.Background()
|
||||
old := legacyBuild(1, "alpha", "c1")
|
||||
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 2, "c2", "s2")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec.mu.Lock()
|
||||
rec.touched = nil
|
||||
rec.mu.Unlock()
|
||||
_, _ = inv.chainBuildList(ctx, c, "alpha", 5)
|
||||
_, _ = inv.chainBuildList(ctx, c, "", 5)
|
||||
_, _, _ = inv.chainBuildByID(ctx, c, "build-1")
|
||||
_, _, _ = inv.chainBuildByID(ctx, c, "build-2")
|
||||
_, _ = inv.chainBuildOf(ctx, c, "alpha", "c1")
|
||||
_, _ = inv.chainBuildFingerprints(ctx, c, "alpha")
|
||||
_, _ = inv.chainFingerprints(ctx, c)
|
||||
_, _ = inv.chainManifestOf(ctx, c, "build-1")
|
||||
_, _, _, _ = inv.chainManifestBuild(ctx, c, "alpha", "c1")
|
||||
_, _ = chainBuilds(ctx, c)
|
||||
_, _ = inv.chainRequests(ctx, c)
|
||||
_, _ = inv.chainBuiltAgainst(ctx, c)
|
||||
rec.mu.Lock()
|
||||
defer rec.mu.Unlock()
|
||||
for _, what := range rec.touched {
|
||||
verb, key, _ := strings.Cut(what, " ")
|
||||
if what == "keys" || what == "watch >" || what == "watch *" || (verb == "get" && !strings.Contains(key, ".")) {
|
||||
t.Errorf("a read touched %q", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A pointer follows the newest build, not the last write: two builds overlap and the older, asked first, finishes last.
|
||||
func TestAnOlderBuildFinishingLastLeavesThePointersOnTheNewer(t *testing.T) {
|
||||
mem := chain.NewMemory()
|
||||
inv, c := layoutChain(mem)
|
||||
ctx := context.Background()
|
||||
older := layoutBuild("alpha", 1, "c1", "s1")
|
||||
older.Asked = layoutAt
|
||||
older.At = layoutAt.Add(20 * time.Minute) // finished last
|
||||
newer := layoutBuild("alpha", 2, "c1", "s2")
|
||||
newer.Asked = layoutAt.Add(5 * time.Minute)
|
||||
newer.At = layoutAt.Add(10 * time.Minute)
|
||||
for _, b := range []Build{newer, older} {
|
||||
if err := inv.chainRecordBuild(ctx, c, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"alpha.newest", "alpha.worked"} {
|
||||
if e, _, _ := mem.Get(ctx, key); string(e.Value) != "build-2" {
|
||||
t.Errorf("%s = %q; want the newer build-2", key, e.Value)
|
||||
}
|
||||
}
|
||||
if e, _, _ := mem.Get(ctx, "alpha.commit.c1"); pointedID(e.Value) != "build-2" {
|
||||
t.Errorf("the commit pointer = %s; want build-2", e.Value)
|
||||
}
|
||||
// The older build is still found by what only it has.
|
||||
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s1"); id != "build-1" {
|
||||
t.Errorf("the older build's own reuse key finds %q", id)
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing reads the chain on the bus before the builds are laid out and the marker written: the first read of a process that
|
||||
// may write lays them out itself (issue 1121: a refused read of the start made the controller exit), and the layout is done
|
||||
// once.
|
||||
func TestTheFirstReadLaysTheBuildsOutAndTheLayoutIsDoneOnce(t *testing.T) {
|
||||
mem := chain.NewMemory()
|
||||
inv, c := layoutChain(mem)
|
||||
inv.RequireLayout()
|
||||
ctx := context.Background()
|
||||
old := legacyBuild(1, "alpha", "c1")
|
||||
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.theChain(ctx); err != nil {
|
||||
t.Fatalf("the first read was refused, not laid out: %v", err)
|
||||
}
|
||||
if !containsKey(keysOf(t, mem), chain.BuildKey("alpha", old.ID)) {
|
||||
t.Fatalf("the first read did not lay %s out under its module: %v", old.ID, keysOf(t, mem))
|
||||
}
|
||||
if _, err := inv.theChain(ctx); err != nil {
|
||||
t.Fatalf("a read after the layout answered %v", err)
|
||||
}
|
||||
keys := keysOf(t, mem)
|
||||
if !containsKey(keys, chain.LayoutMarkerKey) {
|
||||
t.Fatalf("no marker among %v", keys)
|
||||
}
|
||||
if err := inv.LayBuildsOut(ctx, nil); err != nil || !reflect.DeepEqual(keys, keysOf(t, mem)) {
|
||||
t.Fatalf("the second layout changed the bucket: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A layout run again — the live start recorded the first in chain_moved, not in the marker — never sets a pointer back to an
|
||||
// older build, and does not read a manifest of a record already under its new key.
|
||||
func TestALayoutRunAgainLeavesThePointersOfTheBuildsMadeSince(t *testing.T) {
|
||||
counting := &readCounting{Backend: chain.NewMemory()}
|
||||
inv, c := layoutChain(counting)
|
||||
ctx := context.Background()
|
||||
for n := 1; n <= 2; n++ {
|
||||
old := legacyBuild(n, "alpha", fmt.Sprintf("c%d", n))
|
||||
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 3, "c3", "s3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counting.gets.Store(0)
|
||||
r, err := inv.layoutBuilds(ctx, c, nil)
|
||||
if err != nil || r.Written != 0 {
|
||||
t.Fatalf("the second layout %+v: %v", r, err)
|
||||
}
|
||||
for _, key := range []string{"alpha.newest", "alpha.worked", "alpha.held"} {
|
||||
if e, _, _ := counting.Get(ctx, key); string(e.Value) != "build-3" {
|
||||
t.Errorf("%s = %q after the second layout; want build-3", key, e.Value)
|
||||
}
|
||||
}
|
||||
if e, _, _ := counting.Get(ctx, "alpha.commit.c3"); pointedID(e.Value) != "build-3" {
|
||||
t.Errorf("commit c3 = %s", e.Value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/store"
|
||||
)
|
||||
|
||||
// **Migration 0105 refuses a store whose rows were not moved, and drops the tables of a moved one** (novox/hq ADR 0408 §8,
|
||||
// issue 1111). The store is built as the mesh holds it before this migration, with an assignment row in the table and no
|
||||
// record of the move: the migration raises, and nothing it did is kept. Then the move is recorded, a pending assignment
|
||||
// still open refuses it again, and once it has settled the tables go and build_gate stays.
|
||||
func TestMigration0105RefusesAStoreWhoseRowsWereNotMovedAndDropsTheTablesOfAMovedOne(t *testing.T) {
|
||||
inv, _ := ForTestBefore(t, 105)
|
||||
ctx := t.Context()
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "ui", Version: "1"}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.ExecForTest(t, `insert into assignment (node, module) select id, 'ui' from node where name = 'anchor'`)
|
||||
every, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Migration 105 alone: the ones after it (106, issue 1145) are not what this test judges.
|
||||
var all []store.Migration
|
||||
for _, m := range every {
|
||||
if m.Number <= 105 {
|
||||
all = append(all, m)
|
||||
}
|
||||
}
|
||||
tables := func() map[string]bool {
|
||||
t.Helper()
|
||||
out := map[string]bool{}
|
||||
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
||||
out[strings.Trim(row[0], `"`)] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
refused := func(why string) {
|
||||
t.Helper()
|
||||
_, err := inv.store.Migrate(ctx, all)
|
||||
if err == nil || !strings.Contains(err.Error(), why) {
|
||||
t.Fatalf("the migration was not refused for %q: %v", why, err)
|
||||
}
|
||||
if !tables()["assignment"] || !tables()["chain_moved"] {
|
||||
t.Fatalf("a refused migration kept a drop")
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from assignment`); rows[0][0] != "1" {
|
||||
t.Fatalf("a refused migration lost the assignment: %v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// No move recorded: the rows of assignments were never copied.
|
||||
refused("assignments")
|
||||
|
||||
// The move recorded for every kind, and a pending assignment still waiting for its build: no mover carries it.
|
||||
inv.ExecForTest(t, `insert into chain_moved (counts, largest) values ($1::jsonb, $1::jsonb)`,
|
||||
`{"builds": 0, "rollouts": 0, "assignments": 1}`)
|
||||
inv.ExecForTest(t, `insert into pending_assignment (node, module, build_id)
|
||||
select id, 'ui', 'b1' from node where name = 'anchor'`)
|
||||
refused("pending assignment")
|
||||
|
||||
// Expired and raised as a condition, and the condition neither answered nor cleared: still refused.
|
||||
inv.ExecForTest(t, `update pending_assignment set state = 'expired', raised_at = now()`)
|
||||
refused("pending assignment")
|
||||
|
||||
// Its condition cleared: the guard lets the migration through.
|
||||
inv.ExecForTest(t, `update pending_assignment set cleared_at = now()`)
|
||||
done, err := inv.store.Migrate(ctx, all)
|
||||
if err != nil || len(done) != 1 || done[0].Number != 105 {
|
||||
t.Fatalf("the migration did %v (%v)", done, err)
|
||||
}
|
||||
left := tables()
|
||||
for _, gone := range append(append([]string{}, chainTables...), "chain_moved") {
|
||||
if left[gone] {
|
||||
t.Errorf("the table %s is still in the store", gone)
|
||||
}
|
||||
}
|
||||
for _, stays := range []string{"build_gate", "assignment_generation", "node", "module"} {
|
||||
if !left[stays] {
|
||||
t.Errorf("the table %s is gone", stays)
|
||||
}
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from information_schema.columns where table_name = 'node' and column_name = 'sent_builds'`); rows[0][0] != "1" {
|
||||
t.Errorf("node.sent_builds, which stays as the source of the copy, is gone")
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
|
||||
t.Errorf("the function raise_assignment_generation is still in the store")
|
||||
}
|
||||
}
|
||||
|
||||
// A store that never held a row of the chain — a mesh raised after the move — passes the guard with no record of a move.
|
||||
func TestMigration0105PassesAStoreThatNeverHeldARow(t *testing.T) {
|
||||
inv, migrate := ForTestBefore(t, 105)
|
||||
migrate()
|
||||
if rows := inv.QueryForTest(t, `select count(*) from information_schema.tables where table_name in ('build', 'release_plan', 'assignment', 'chain_moved')`); rows[0][0] != "0" {
|
||||
t.Fatalf("tables of the chain remain: %v", rows)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// The first read of a chain that requires the layout lays the builds out itself when it may write, instead of refusing
|
||||
// the controller's own start (issue 1121, 2026-10-18: the controller exited every five seconds).
|
||||
func TestTheFirstReadOfAChainNotLaidOutLaysItOut(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
inv.RequireLayout()
|
||||
if _, found, _ := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); found {
|
||||
t.Fatal("a fresh chain holds the marker already; the test proves nothing")
|
||||
}
|
||||
if _, err := inv.theChain(ctx); err != nil {
|
||||
t.Fatalf("the first read was refused: %v", err)
|
||||
}
|
||||
if _, found, err := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); err != nil || !found {
|
||||
t.Fatalf("the read did not lay the builds out: found %v, %v", found, err)
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,11 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
@@ -39,8 +41,8 @@ func WithRollout(ctx context.Context, id string, modules map[string]RolloutModul
|
||||
// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the
|
||||
// commit. Empty when the assignments are not on the bus, and for a module not sent yet.
|
||||
func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
@@ -50,7 +52,7 @@ func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string
|
||||
out := map[string]chain.Ref{}
|
||||
for _, k := range all {
|
||||
a := k.Value
|
||||
if a.Node == node && !a.Assigned.IsZero() && a.Stands() && (a.Build != "" || a.Commit != "") {
|
||||
if a.Node == node && a.Stands() && (a.Build != "" || a.Commit != "") {
|
||||
out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit}
|
||||
}
|
||||
}
|
||||
@@ -59,8 +61,8 @@ func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string
|
||||
|
||||
// AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links.
|
||||
func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
@@ -77,14 +79,18 @@ func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Ass
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// recordRuns writes what a send put on a machine into the assignments, once the send is recorded: for each module it carried
|
||||
// a build of — by the commit — the build, what it published, the commit, the rollout and since when. **A module that runs the
|
||||
// commit already keeps what it names**, filled where it names nothing and never blanked, so a send that carries it again, or a
|
||||
// push by hand, changes nothing it knows. A module the machine is not assigned is no assignment to write.
|
||||
// recordRuns writes what a send put on a machine into the assignments, once the send is recorded, and it is how the mesh
|
||||
// knows what a machine was sent (SentBuilds). **A send that names its builds** writes, for each module it carried a build of
|
||||
// — by the commit — the build, what it published, the commit, the rollout and since when, on the module's key: filled where
|
||||
// the key names nothing, replaced when the commit moved, never blanked by a push by hand; a module nothing assigns, carried
|
||||
// because another requires it, gets a standing key marked pulled-in; and every other key of the machine loses what it named,
|
||||
// for the machine is not sent it any more. **A send that names none** (a declaration by hand) makes what the machine runs
|
||||
// unknown: every key of it says so (sent-unknown) and names no commit. **An error here fails the record of the send**, for
|
||||
// what a machine was sent is read from these keys and from nowhere else.
|
||||
func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error {
|
||||
c, on, err := i.ChainReads(ctx, KindAssignments)
|
||||
if err != nil || !on || len(builds) == 0 {
|
||||
return err
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("what the machine runs cannot be written: %w", err)
|
||||
}
|
||||
var node string
|
||||
if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) {
|
||||
@@ -93,24 +99,81 @@ func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[st
|
||||
return err
|
||||
}
|
||||
rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf)
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the assignments of %s cannot be read: %w", node, err)
|
||||
}
|
||||
var failed []error
|
||||
for _, k := range sortedValues(all) {
|
||||
a := k.Value
|
||||
if a.Node != node || !a.Stands() {
|
||||
continue
|
||||
}
|
||||
if builds[a.Module] != "" {
|
||||
continue // carried, and its commit is written below
|
||||
}
|
||||
if err := i.forgetRun(ctx, c, k, builds == nil, at); err != nil {
|
||||
failed = append(failed, fmt.Errorf("%s on %s: %w", a.Module, node, err))
|
||||
}
|
||||
}
|
||||
modules := make([]string, 0, len(builds))
|
||||
for m := range builds {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
var failed []error
|
||||
for _, m := range modules {
|
||||
commit := builds[m]
|
||||
if commit == "" {
|
||||
if builds[m] == "" {
|
||||
continue
|
||||
}
|
||||
if err := i.recordRun(ctx, c, node, m, commit, rollout, at); err != nil {
|
||||
if err := i.recordRun(ctx, c, node, m, builds[m], rollout, at); err != nil {
|
||||
failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(failed...)
|
||||
}
|
||||
|
||||
func sortedValues(all map[string]chain.Kept[chain.Assignment]) []chain.Kept[chain.Assignment] {
|
||||
out := make([]chain.Kept[chain.Assignment], 0, len(all))
|
||||
for key, k := range all {
|
||||
k.Key = key
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key })
|
||||
return out
|
||||
}
|
||||
|
||||
// forgetRun writes that a machine is no longer sent what a key named: its links are cleared, and for a send by hand the key
|
||||
// says what the machine runs is unknown.
|
||||
func (i *Inventory) forgetRun(ctx context.Context, c *chain.Chain, k chain.Kept[chain.Assignment], hand bool, at time.Time) error {
|
||||
for try := 0; try < casTries; try++ {
|
||||
next := k.Value
|
||||
next.Build, next.Artifact, next.Package, next.Commit, next.Rollout, next.Since = "", "", nil, "", "", time.Time{}
|
||||
next.SentUnknown = time.Time{}
|
||||
if hand {
|
||||
next.SentUnknown = at.UTC()
|
||||
}
|
||||
if next.Build == k.Value.Build && next.Commit == k.Value.Commit && next.Rollout == k.Value.Rollout &&
|
||||
next.Artifact == k.Value.Artifact && next.Package == k.Value.Package && next.Since.Equal(k.Value.Since) &&
|
||||
next.SentUnknown.Equal(k.Value.SentUnknown) {
|
||||
return nil
|
||||
}
|
||||
if _, err := c.Assignments.Update(ctx, k.Key, next, k.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
cur, found, gerr := c.Assignments.Get(ctx, k.Key)
|
||||
if gerr != nil {
|
||||
return gerr
|
||||
}
|
||||
if !found || !cur.Value.Stands() {
|
||||
return nil
|
||||
}
|
||||
k = chain.Kept[chain.Assignment]{Key: k.Key, Value: cur.Value, Revision: cur.Revision}
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the assignment kept moving under the write")
|
||||
}
|
||||
|
||||
func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error {
|
||||
key := assignmentKey(node, module)
|
||||
var build, rolloutID string
|
||||
@@ -124,11 +187,12 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found || cur.Value.Assigned.IsZero() || !cur.Value.Stands() {
|
||||
return nil // nothing assigned there to say what it runs
|
||||
if found && cur.Value.State != "" {
|
||||
return nil // a wait for a build: the module is not on the machine by it
|
||||
}
|
||||
same := cur.Value.Commit == commit
|
||||
if same && cur.Value.Build != "" && (rolloutID == "" || cur.Value.Rollout != "") {
|
||||
standing := found && cur.Value.Stands()
|
||||
same := standing && cur.Value.Commit == commit
|
||||
if same && cur.Value.Build != "" && cur.Value.SentUnknown.IsZero() && (rolloutID == "" || cur.Value.Rollout != "") {
|
||||
return nil // a send that carries what the machine runs already says nothing new
|
||||
}
|
||||
if !resolved {
|
||||
@@ -148,7 +212,13 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module,
|
||||
resolved = true
|
||||
}
|
||||
next := cur.Value
|
||||
if same {
|
||||
switch {
|
||||
case !standing:
|
||||
// Carried and assigned to nobody here: a standing key, no time assigned, marked pulled in. An ended one is
|
||||
// written over, the ended one staying in history.
|
||||
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Commit: commit, Build: build, Rollout: rolloutID,
|
||||
Since: at.UTC()}
|
||||
case same:
|
||||
// Already running it: fill what is missing, blank nothing.
|
||||
if next.Build == "" {
|
||||
next.Build = build
|
||||
@@ -159,18 +229,26 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module,
|
||||
if next.Since.IsZero() {
|
||||
next.Since = at.UTC()
|
||||
}
|
||||
} else {
|
||||
default:
|
||||
next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC()
|
||||
next.Artifact, next.Package = "", nil
|
||||
}
|
||||
next.SentUnknown = time.Time{}
|
||||
if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build {
|
||||
next.Artifact, next.Package = published.Artifact, published.Package
|
||||
}
|
||||
if next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout &&
|
||||
if standing && next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout &&
|
||||
next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) &&
|
||||
next.Since.Equal(cur.Value.Since) {
|
||||
next.Since.Equal(cur.Value.Since) && next.SentUnknown.Equal(cur.Value.SentUnknown) {
|
||||
return nil
|
||||
}
|
||||
if !found {
|
||||
if _, err := c.Assignments.Create(ctx, key, next); errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else {
|
||||
@@ -180,11 +258,105 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module,
|
||||
return fmt.Errorf("the assignment kept moving under the write")
|
||||
}
|
||||
|
||||
// saidRunsNot says a send's links could not be written, to the log of whoever sent: the send is away and is recorded, and
|
||||
// a record of what it put where that could not be written is said, never a reason to call the send failed.
|
||||
func saidRunsNot(node string, err error) {
|
||||
// CopySentBuilds copies what delivery 1 recorded in node.sent_builds onto the keys of the assignments, once, at the start of
|
||||
// the controller that holds the lease and has attached the chain (issue 1109, review round 1): delivery 1 wrote no commit onto
|
||||
// the key of a module an assignment pulled in, so without it those modules would be missing from SentBuilds although the
|
||||
// machine is known to have been sent. A key that names a commit is never overwritten, so it is idempotent; once a node's keys
|
||||
// are written its column is emptied, so a later restart cannot lay a stale commit over a key a later send cleared or ended.
|
||||
// It answers how many keys it wrote; an error is raised by the caller as a condition.
|
||||
func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s was sent its declaration, and the build, commit and rollout of each module "+
|
||||
"could not be written in its assignments: %v\n", node, err)
|
||||
return 0, err
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx, `select name, sent_builds from node where sent_builds is not null`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
type kept struct {
|
||||
name string
|
||||
sent map[string]string
|
||||
}
|
||||
var all []kept
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var raw []byte
|
||||
if err := rows.Scan(&name, &raw); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
sent := map[string]string{}
|
||||
if err := json.Unmarshal(raw, &sent); err != nil {
|
||||
rows.Close()
|
||||
return 0, fmt.Errorf("what %s was sent, in the store, cannot be read: %w", name, err)
|
||||
}
|
||||
all = append(all, kept{name, sent})
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, k := range all {
|
||||
if err := i.copySentBuildsOf(ctx, c, k.name, k.sent, now); err != nil {
|
||||
return 0, fmt.Errorf("what %s was sent cannot be copied onto its assignments: %w", k.name, err)
|
||||
}
|
||||
if _, err := i.store.Pool().Exec(ctx, `update node set sent_builds = null where name = $1`, k.name); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(all), nil
|
||||
}
|
||||
|
||||
// copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that
|
||||
// names no commit gets it (and the build, when one is known; no rollout), and where no key exists a standing pulled-in key is
|
||||
// made. A key with a commit, an ended key, a hand send's unknown and a wait for a build are left as they are.
|
||||
func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error {
|
||||
for _, module := range slices.Sorted(maps.Keys(sent)) {
|
||||
commit := sent[module]
|
||||
if commit == "" {
|
||||
continue
|
||||
}
|
||||
key := assignmentKey(node, module)
|
||||
build, err := i.chainBuildOf(ctx, c, module, commit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for try := 0; ; try++ {
|
||||
if try == casTries {
|
||||
return fmt.Errorf("the assignment %s kept moving under the write", key)
|
||||
}
|
||||
cur, found, err := c.Assignments.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found && (cur.Value.State != "" || !cur.Value.Ended.IsZero() || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) {
|
||||
break // a wait, an ended key (never revived), one that names a commit, or a hand send's unknown
|
||||
}
|
||||
next := cur.Value
|
||||
if !found {
|
||||
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at}
|
||||
}
|
||||
next.Commit, next.Build = commit, build
|
||||
if next.Since.IsZero() {
|
||||
next.Since = at
|
||||
}
|
||||
if !found {
|
||||
_, err = c.Assignments.Create(ctx, key, next)
|
||||
if errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
|
||||
if errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,663 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **The move** (novox/hq ADR 0408 §8, issue 981 note 7 Q2): the serving controller writes the rows of a kind into its
|
||||
// bucket at its start, holding the lease and after the buckets exist, and records the kind in chain_moved. Idempotent: a
|
||||
// key that exists is already moved and is never overwritten, so a bucket value written since is not laid over by the row
|
||||
// it came from. It counts the rows against the keys, logs the largest value, and **refuses** (a Refusal, which the
|
||||
// controller raises as an urgent condition) on a count that differs or a value over the bus's largest, with nothing
|
||||
// switched: a kind is recorded moved only once all of it is in the bucket.
|
||||
|
||||
// Refusal is a move that stopped without switching a kind, and why. It is permanent until the cause is put right; any
|
||||
// other error of a move (the bus away, a write not granted) is tried again.
|
||||
type Refusal struct {
|
||||
Kind, Why string
|
||||
}
|
||||
|
||||
func (r *Refusal) Error() string {
|
||||
return fmt.Sprintf("the move of %s to the bus is refused: %s", r.Kind, r.Why)
|
||||
}
|
||||
|
||||
// MoveReport is what a move of one kind did.
|
||||
type MoveReport struct {
|
||||
Kind string
|
||||
// Rows is how many records the tables held, Written how many keys this move made, and Present how many already
|
||||
// held a value (moved by an earlier start).
|
||||
Rows, Written, Present int
|
||||
// TableRows is how many rows of the tables those records came from: a request and its outcome are two rows and one
|
||||
// record.
|
||||
TableRows int
|
||||
// Largest is the biggest value, in bytes.
|
||||
Largest int
|
||||
// Already is a kind an earlier start moved: nothing was done.
|
||||
Already bool
|
||||
}
|
||||
|
||||
// mover is one kind of the chain: its rows, read from its tables, as the keys and values they become.
|
||||
type mover struct {
|
||||
kind string
|
||||
// rows reads the tables: each record's key and its value, whole, and how many rows of the tables they came from.
|
||||
rows func(ctx context.Context) ([]moveRow, int, error)
|
||||
// create writes one record under its key; chain.ErrExists when the key holds a value.
|
||||
create func(ctx context.Context, c *chain.Chain, row moveRow) error
|
||||
// keys are the keys the bucket holds.
|
||||
keys func(ctx context.Context, c *chain.Chain) ([]string, error)
|
||||
// count is how many rows the tables hold now, counted as rows reports them.
|
||||
count func(ctx context.Context, tx pgx.Tx) (int, error)
|
||||
// lock keeps the tables from taking a row while the switch is recorded.
|
||||
lock string
|
||||
// settle runs under that lock, with the rows read again: it makes the bucket hold what the rows say now. A row the old
|
||||
// controller changed after the first pass is in the bucket before the kind is recorded as moved, for before the switch
|
||||
// the table is the only truth.
|
||||
settle func(ctx context.Context, c *chain.Chain, rows []moveRow) error
|
||||
}
|
||||
|
||||
type moveRow struct {
|
||||
key string
|
||||
value any
|
||||
size int
|
||||
}
|
||||
|
||||
func (i *Inventory) movers() []mover {
|
||||
return []mover{{
|
||||
kind: KindBuilds,
|
||||
rows: func(ctx context.Context) ([]moveRow, int, error) {
|
||||
builds, err := i.allBuildRows(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
requests, err := i.allRequestRows(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
// A build request is the build's record in state asked, and the request stays beside the outcome that
|
||||
// answered it: one record under one id (issue 1091).
|
||||
byID := make(map[string]*chain.Build, len(builds))
|
||||
var order []string
|
||||
for _, b := range builds {
|
||||
v := toChainBuild(b)
|
||||
byID[b.ID] = &v
|
||||
order = append(order, b.ID)
|
||||
}
|
||||
for _, r := range requests {
|
||||
if v, answered := byID[r.ID]; answered {
|
||||
v.Request = toRequest(r)
|
||||
continue
|
||||
}
|
||||
v := askedRecord(r)
|
||||
if r.NotAsked != "" {
|
||||
// Never handed over: asked again once, after the move (askAgain).
|
||||
v.Request.Reask = chain.ReaskPending
|
||||
}
|
||||
byID[r.ID] = &v
|
||||
order = append(order, r.ID)
|
||||
}
|
||||
out := make([]moveRow, 0, len(order))
|
||||
for _, id := range order {
|
||||
body, err := json.Marshal(byID[id])
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, moveRow{key: chainBuildKey(id), value: *byID[id], size: len(body)})
|
||||
}
|
||||
return out, len(builds) + len(requests), nil
|
||||
},
|
||||
create: func(ctx context.Context, c *chain.Chain, row moveRow) error {
|
||||
_, err := c.Builds.Create(ctx, row.key, row.value.(chain.Build))
|
||||
return err
|
||||
},
|
||||
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Builds.Keys(ctx) },
|
||||
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
|
||||
var n int
|
||||
err := tx.QueryRow(ctx, `select (select count(*) from build) + (select count(*) from build_request)`).Scan(&n)
|
||||
return n, err
|
||||
},
|
||||
lock: `lock table build, build_request in share mode`,
|
||||
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
|
||||
return settleBucket(ctx, c.Builds, rows)
|
||||
},
|
||||
}, {
|
||||
kind: KindRollouts,
|
||||
rows: func(ctx context.Context) ([]moveRow, int, error) {
|
||||
plans, err := i.plans(ctx, `order by created`)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
merges, err := i.merges(ctx, ``)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
tablePlans := len(plans)
|
||||
if plans, err = holdMerges(plans, merges); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out := make([]moveRow, 0, len(plans))
|
||||
for _, p := range plans {
|
||||
body, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, moveRow{key: p.ID, value: p, size: len(body)})
|
||||
}
|
||||
return out, tablePlans + len(merges), nil
|
||||
},
|
||||
create: func(ctx context.Context, c *chain.Chain, row moveRow) error {
|
||||
_, err := chain.RolloutsOf[Plan](c).Create(ctx, row.key, row.value.(Plan))
|
||||
return err
|
||||
},
|
||||
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) {
|
||||
return chain.RolloutsOf[Plan](c).Keys(ctx)
|
||||
},
|
||||
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
|
||||
var n int
|
||||
err := tx.QueryRow(ctx, `select (select count(*) from release_plan) + (select count(*) from batched_merge)`).Scan(&n)
|
||||
return n, err
|
||||
},
|
||||
lock: `lock table release_plan, batched_merge in share mode`,
|
||||
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
|
||||
return settleBucket(ctx, chain.RolloutsOf[Plan](c), rows)
|
||||
},
|
||||
}, {
|
||||
kind: KindAssignments,
|
||||
rows: func(ctx context.Context) ([]moveRow, int, error) {
|
||||
assigned, err := i.assignedWithSends(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out := make([]moveRow, 0, len(assigned))
|
||||
for _, a := range assigned {
|
||||
body, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, moveRow{key: assignmentKey(a.Node, a.Module), value: a, size: len(body)})
|
||||
}
|
||||
return out, len(out), nil
|
||||
},
|
||||
create: createAssignment,
|
||||
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Assignments.Keys(ctx) },
|
||||
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
|
||||
var n int
|
||||
err := tx.QueryRow(ctx, `select count(*) from assignment`).Scan(&n)
|
||||
return n, err
|
||||
},
|
||||
lock: `lock table assignment in share mode`,
|
||||
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
|
||||
have := make(map[string]bool, len(rows))
|
||||
for _, r := range rows {
|
||||
have[r.key] = true
|
||||
if err := settleAssignment(ctx, c, r); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// An assignment the store no longer holds ended since the first pass: the bucket says so.
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for key, k := range all {
|
||||
if have[key] || k.Value.Assigned.IsZero() || !k.Value.Stands() {
|
||||
continue
|
||||
}
|
||||
k.Value.Ended = time.Now().UTC()
|
||||
if _, err := c.Assignments.Update(ctx, key, k.Value, k.Revision); err != nil && !errors.Is(err, chain.ErrMoved) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
// assignedWithSends is every assignment of the store as the chain keeps it: with the commit its machine was last sent for
|
||||
// the module (node.sent_builds), the build that is of it and what the build published, and since when (when the machine was
|
||||
// last sent). An assignment of a module the machine was never sent has none of those.
|
||||
func (i *Inventory) assignedWithSends(ctx context.Context) ([]chain.Assignment, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name, a.module, a.assigned, n.sent_at, n.sent_builds
|
||||
from assignment a join node n on n.id = a.node order by n.name, a.module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []chain.Assignment
|
||||
for rows.Next() {
|
||||
var a chain.Assignment
|
||||
var sentAt *time.Time
|
||||
var raw []byte
|
||||
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned, &sentAt, &raw); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
a.Assigned = a.Assigned.UTC()
|
||||
var sent map[string]string
|
||||
if len(raw) > 0 {
|
||||
_ = json.Unmarshal(raw, &sent)
|
||||
}
|
||||
if commit := sent[a.Module]; commit != "" && sentAt != nil {
|
||||
a.Commit, a.Since = commit, sentAt.UTC()
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The build of each commit, read once: the newest successful build of the module at it.
|
||||
type at struct{ module, commit string }
|
||||
var all []Build
|
||||
if _, moved, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if moved {
|
||||
if all, err = chainBuilds(ctx, i.attached()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if all, err = i.allBuildRows(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newest := map[at]Build{}
|
||||
for _, b := range all {
|
||||
k := at{b.Module, b.Commit}
|
||||
if b.Failed != "" || b.Module == "" {
|
||||
continue
|
||||
}
|
||||
if had, has := newest[k]; !has || b.At.After(had.At) || (b.At.Equal(had.At) && b.ID > had.ID) {
|
||||
newest[k] = b
|
||||
}
|
||||
}
|
||||
for k := range out {
|
||||
if out[k].Commit == "" {
|
||||
continue
|
||||
}
|
||||
b, found := newest[at{out[k].Module, out[k].Commit}]
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
out[k].Build = b.ID
|
||||
if published := publishedOf(b.Made); published != nil {
|
||||
out[k].Artifact, out[k].Package = published.Artifact, published.Package
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// holdMerges puts each batched merge into the record of the plan that answers it. A merge no plan answers — the plan_id
|
||||
// of a merge a failed walk carried is null, or names a plan the store never kept — is held, flagged waiting, by the
|
||||
// newest plan that lists it among its merges, else the record kept for the merges no plan answers: the record that held it
|
||||
// last.
|
||||
func holdMerges(plans []Plan, merges []BatchedMerge) ([]Plan, error) {
|
||||
waitingAt := -1
|
||||
at := make(map[string]int, len(plans))
|
||||
for k, p := range plans {
|
||||
at[p.ID] = k
|
||||
}
|
||||
newest := func(match func(Plan) bool) int {
|
||||
best := -1
|
||||
for k, p := range plans {
|
||||
if match(p) && (best < 0 || p.Created.After(plans[best].Created)) {
|
||||
best = k
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
for _, m := range merges {
|
||||
held := heldOf(m, m.Heard)
|
||||
holder, answered := at[m.Plan]
|
||||
if m.Plan == "" || !answered {
|
||||
held.Waiting = true
|
||||
holder = newest(func(p Plan) bool {
|
||||
if p.Delivery == nil {
|
||||
return false
|
||||
}
|
||||
for _, listed := range p.Delivery.Merges {
|
||||
if strings.EqualFold(listed.Repository, m.Repository) && listed.Commit == m.Commit {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
if holder < 0 {
|
||||
// Held by the record kept for the merges no plan answers, made with the others.
|
||||
if waitingAt < 0 {
|
||||
plans = append(plans, Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
|
||||
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
|
||||
waitingAt = len(plans) - 1
|
||||
}
|
||||
holder = waitingAt
|
||||
}
|
||||
}
|
||||
plans[holder].Held = append(plans[holder].Held, held)
|
||||
}
|
||||
return plans, nil
|
||||
}
|
||||
|
||||
// MoveChain moves every kind not moved yet, in order, and says what it did. It needs the chain attached and the lease
|
||||
// held (the chain's writes ask). It stops at the first kind that fails.
|
||||
func (i *Inventory) MoveChain(ctx context.Context, say func(format string, args ...any)) ([]MoveReport, error) {
|
||||
if say == nil {
|
||||
say = func(string, ...any) {}
|
||||
}
|
||||
if i.attached() == nil {
|
||||
return nil, ErrNoChain
|
||||
}
|
||||
var out []MoveReport
|
||||
for _, m := range i.movers() {
|
||||
r, err := i.moveKind(ctx, m, say)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// movePasses is how often a move reads the tables again when they took a row while it wrote: the controller before this
|
||||
// one may still have been writing as the lease changed hands.
|
||||
const movePasses = 5
|
||||
|
||||
func (i *Inventory) moveKind(ctx context.Context, m mover, say func(string, ...any)) (MoveReport, error) {
|
||||
report := MoveReport{Kind: m.kind}
|
||||
moved, err := i.ChainMoved(ctx)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if _, done := moved[m.kind]; done {
|
||||
report.Already = true
|
||||
return report, nil
|
||||
}
|
||||
for pass := 1; pass <= movePasses; pass++ {
|
||||
rows, tableRows, err := m.rows(ctx)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("the %s cannot be read from the tables: %w", m.kind, err)
|
||||
}
|
||||
report.Rows, report.TableRows, report.Largest = len(rows), tableRows, 0
|
||||
// Every value measured before any is written: a value over the bus's largest stops the move with nothing
|
||||
// written (ADR 0408 §6), never cut.
|
||||
for _, r := range rows {
|
||||
report.Largest = max(report.Largest, r.size)
|
||||
if r.size > broker.ChainValueBytes {
|
||||
return report, &Refusal{m.kind, fmt.Sprintf("the value of %s is %d bytes and the bus holds at most %d; "+
|
||||
"nothing is cut, so the record must be made smaller first", r.key, r.size, broker.ChainValueBytes)}
|
||||
}
|
||||
}
|
||||
report.Written, report.Present = 0, 0
|
||||
for _, r := range rows {
|
||||
switch err := m.create(ctx, i.attached(), r); {
|
||||
case err == nil:
|
||||
report.Written++
|
||||
case errors.Is(err, chain.ErrExists):
|
||||
report.Present++
|
||||
case errors.Is(err, chain.ErrBadKey):
|
||||
return report, &Refusal{m.kind, fmt.Sprintf("%s cannot be a key of the bus: %v", r.key, err)}
|
||||
default:
|
||||
return report, fmt.Errorf("%s %s is not written to the bus: %w", m.kind, r.key, err)
|
||||
}
|
||||
}
|
||||
// Rows against keys: every row's key is in the bucket.
|
||||
keys, err := m.keys(ctx, i.attached())
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("the keys of the %s cannot be read back: %w", m.kind, err)
|
||||
}
|
||||
have := make(map[string]bool, len(keys))
|
||||
for _, k := range keys {
|
||||
have[k] = true
|
||||
}
|
||||
for _, r := range rows {
|
||||
if !have[r.key] {
|
||||
return report, &Refusal{m.kind, fmt.Sprintf("the tables hold %d records and the bucket does not hold %s", len(rows), r.key)}
|
||||
}
|
||||
}
|
||||
settled, err := i.switchKind(ctx, m, report)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if settled {
|
||||
say("moved the %s to the bus: %d records, %d written now and %d already there, the largest %d bytes",
|
||||
m.kind, report.Rows, report.Written, report.Present, report.Largest)
|
||||
return report, nil
|
||||
}
|
||||
say("the %s took a row while they were moved, so they are read again (pass %d of %d)", m.kind, pass, movePasses)
|
||||
}
|
||||
return report, &Refusal{m.kind, fmt.Sprintf("the tables kept taking rows through %d passes; another controller is still writing them", movePasses)}
|
||||
}
|
||||
|
||||
// switchKind records the kind as moved, in one transaction that holds the tables against writes and finds their count
|
||||
// as the move read it: false when they took a row since.
|
||||
func (i *Inventory) switchKind(ctx context.Context, m mover, r MoveReport) (bool, error) {
|
||||
committed := false
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if _, err := tx.Exec(ctx, m.lock); err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := m.count(ctx, tx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if n != r.TableRows {
|
||||
return false, nil
|
||||
}
|
||||
// The store's mirror of the assignments writes nothing from here: it would lay values without the links over the keys
|
||||
// the settle writes and end the ones it ended.
|
||||
if m.kind == KindAssignments {
|
||||
i.moved.switching.Store(true)
|
||||
defer func() {
|
||||
if !committed {
|
||||
i.moved.switching.Store(false)
|
||||
}
|
||||
}()
|
||||
}
|
||||
// The rows read again under the lock: nothing writes them now, so what the bucket holds is made what they say.
|
||||
rows, tableRows, err := m.rows(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if tableRows != r.TableRows {
|
||||
return false, nil
|
||||
}
|
||||
if err := m.settle(ctx, i.attached(), rows); err != nil {
|
||||
return false, fmt.Errorf("the %s changed since they were first written, and the bucket cannot be brought to them: %w", m.kind, err)
|
||||
}
|
||||
counts, _ := json.Marshal(map[string]int{m.kind: r.Rows})
|
||||
largest, _ := json.Marshal(map[string]int{m.kind: r.Largest})
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into chain_moved (counts, largest) values ($1, $2)
|
||||
on conflict (only_one) do update
|
||||
set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest,
|
||||
moved_at = now()`, counts, largest); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
committed = true
|
||||
i.moved.mu.Lock()
|
||||
if i.moved.kind == nil {
|
||||
i.moved.kind = map[string]bool{}
|
||||
}
|
||||
i.moved.kind[m.kind] = true
|
||||
i.moved.mu.Unlock()
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// allBuildRows is every build of the table, whole.
|
||||
func (i *Inventory) allBuildRows(ctx context.Context) ([]Build, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select id, repository, ref, coalesce(module, ''), commit_hash, built_on, failed, made, source_path, manifest,
|
||||
built_against, built_contexts, build_sources, source_fingerprint, asked, at
|
||||
from build order by at, id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Build
|
||||
for rows.Next() {
|
||||
var b Build
|
||||
var made, manifest, against, read, sources []byte
|
||||
var asked *time.Time
|
||||
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path,
|
||||
&manifest, &against, &read, &sources, &b.SourceFingerprint, &asked, &b.At); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if asked != nil {
|
||||
b.Asked = *asked
|
||||
}
|
||||
b.Manifest = manifest
|
||||
for _, part := range []struct {
|
||||
raw []byte
|
||||
to any
|
||||
who string
|
||||
}{{made, &b.Made, "made"}, {against, &b.Against, "built_against"}, {read, &b.Read, "built_contexts"},
|
||||
{sources, &b.Sources, "build_sources"}} {
|
||||
if len(part.raw) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(part.raw, part.to); err != nil {
|
||||
return nil, fmt.Errorf("build %s: %s is not readable: %w", b.ID, part.who, err)
|
||||
}
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// allRequestRows is every build request of the table.
|
||||
func (i *Inventory) allRequestRows(ctx context.Context) ([]BuildRequest, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select id, repository, seat, source_path, ref, commit_hash, asked_for, coalesce(not_asked, ''),
|
||||
coalesce(outcome_unknown, ''), asked_at, at_terminal
|
||||
from build_request order by asked_at, id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []BuildRequest
|
||||
for rows.Next() {
|
||||
var a BuildRequest
|
||||
if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked,
|
||||
&a.OutcomeUnknown, &a.At, &a.AtTerminal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// createAssignment writes an assignment's row under its key; chain.ErrExists when the key holds a value that stays.
|
||||
func createAssignment(ctx context.Context, c *chain.Chain, row moveRow) error {
|
||||
want := row.value.(chain.Assignment)
|
||||
// The mirror of the store kept a value under this key until now, with the provisioner's plan beside it: the
|
||||
// plan stays, the record's links are laid in. A value that names its build already is a later word, and stays.
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Assignments.Get(ctx, row.key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
if _, err := c.Assignments.Create(ctx, row.key, want); !errors.Is(err, chain.ErrExists) {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if cur.Value.Build != "" || cur.Value.Commit != "" || !cur.Value.Since.IsZero() || (want.Build == "" && want.Commit == "") {
|
||||
return chain.ErrExists
|
||||
}
|
||||
merged := want
|
||||
merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning
|
||||
if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the assignment %s kept moving under the write", row.key)
|
||||
}
|
||||
|
||||
// settleBucket makes a bucket of whole values hold each row's value, by compare-and-set: a key that is absent is made, one
|
||||
// whose bytes differ from the row's is written over, and one that is the same is left.
|
||||
func settleBucket[T any](ctx context.Context, b chain.Bucket[T], rows []moveRow) error {
|
||||
for _, r := range rows {
|
||||
want, err := json.Marshal(r.value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
settled := false
|
||||
for try := 0; try < casTries && !settled; try++ {
|
||||
e, found, err := b.GetRaw(ctx, r.key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case !found:
|
||||
_, err = b.CreateRaw(ctx, r.key, want)
|
||||
if errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
}
|
||||
case bytes.Equal(e.Value, want):
|
||||
settled = true
|
||||
continue
|
||||
default:
|
||||
_, err = b.UpdateRaw(ctx, r.key, want, e.Revision)
|
||||
if errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
settled = true
|
||||
}
|
||||
if !settled {
|
||||
return fmt.Errorf("%s kept moving under the write", r.key)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// settleAssignment makes the value under an assignment's key what its row says, keeping the plan the mirror put beside it:
|
||||
// before the switch the store is the only truth of the links, so a commit it changed since the first pass is written over.
|
||||
func settleAssignment(ctx context.Context, c *chain.Chain, row moveRow) error {
|
||||
want := row.value.(chain.Assignment)
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Assignments.Get(ctx, row.key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
if _, err := c.Assignments.Create(ctx, row.key, want); errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
merged := want
|
||||
merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning
|
||||
a, _ := json.Marshal(merged)
|
||||
b, _ := json.Marshal(cur.Value)
|
||||
if bytes.Equal(a, b) {
|
||||
return nil
|
||||
}
|
||||
if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the assignment %s kept moving under the write", row.key)
|
||||
}
|
||||
@@ -1,949 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// A build row as the store held it before the move: written by SQL, the only hand-written state these tests have, for
|
||||
// it stands for what the mesh held before this release.
|
||||
func aBuildRow(t *testing.T, inv *Inventory, id, module, commit, failed string) {
|
||||
t.Helper()
|
||||
inv.ExecForTest(t,
|
||||
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest,
|
||||
built_against, built_contexts, asked, source_fingerprint, build_sources)
|
||||
values ($1::text, 'novox/' || $2::text, 'main', $2::text, $3::text, 'ace', $4::text,
|
||||
case when $4::text = ''
|
||||
then ('[{"name":"app","kind":"image","reference":"artifact-store://' || $2::text || '@sha256:' || $1::text || '"}]')::jsonb
|
||||
else '[]'::jsonb end,
|
||||
'modules/' || $2::text, ('{"module":"' || $2::text || '"}')::jsonb,
|
||||
'["artifact-store://base@sha256:aa"]'::jsonb, '[{"repository":"novox/other","ref":"main"}]'::jsonb,
|
||||
now() - interval '1 hour', 'fp-' || $1::text,
|
||||
('[{"paths":["modules/' || $2::text || '/module.json"]}]')::jsonb)`,
|
||||
id, module, commit, failed)
|
||||
}
|
||||
|
||||
// inTheTables is an inventory before the move: its builds and assignments are rows of the store, which a test of a data
|
||||
// migration of the store (that reads those rows) needs.
|
||||
func inTheTables(t *testing.T) *Inventory {
|
||||
t.Helper()
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
return inv
|
||||
}
|
||||
|
||||
// reportOf is the report of one kind among those a move made, which are one for each kind in the order they are moved.
|
||||
func reportOf(t *testing.T, reports []MoveReport, kind string) MoveReport {
|
||||
t.Helper()
|
||||
for _, r := range reports {
|
||||
if r.Kind == kind {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("no report of %s among %+v", kind, reports)
|
||||
return MoveReport{}
|
||||
}
|
||||
|
||||
// allAlready says every kind was moved by an earlier start: the second run writes nothing, and reports one for each kind.
|
||||
func allAlready(t *testing.T, reports []MoveReport) {
|
||||
t.Helper()
|
||||
if len(reports) != 3 {
|
||||
t.Fatalf("the move reported %d kinds, want 3: %+v", len(reports), reports)
|
||||
}
|
||||
for _, r := range reports {
|
||||
if !r.Already || r.Written != 0 {
|
||||
t.Fatalf("the second move of %s %+v", r.Kind, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func rowsInTheTable(t *testing.T, inv *Inventory, table string) string {
|
||||
t.Helper()
|
||||
rows := inv.QueryForTest(t, `select count(*) from `+table)
|
||||
return rows[0][0]
|
||||
}
|
||||
|
||||
// **The move copies every build row into the builds bucket, and a second run writes nothing** (ADR 0408 §8, issue
|
||||
// 1090): each row under its id, whole; the kind recorded in chain_moved; and from then on a build is read from the
|
||||
// bucket and written to it, the table taking no more.
|
||||
func TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
aBuildRow(t, inv, "build-2", "notes", "c2", "")
|
||||
aBuildRow(t, inv, "build-3", "wiki", "", "compile error")
|
||||
|
||||
// Before the move the tables are read, and the bucket is empty.
|
||||
if before, err := inv.Builds(ctx, "", 10); err != nil || len(before) != 3 {
|
||||
t.Fatalf("before the move the tables hold %d builds: %v", len(before), err)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("moved before the move: %v", moved)
|
||||
}
|
||||
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.Written != 3 || b.Largest == 0 {
|
||||
t.Fatalf("the move of the builds reported %+v", reports)
|
||||
}
|
||||
moved, err := inv.ChainMoved(ctx)
|
||||
if err != nil || moved[KindBuilds] != 3 {
|
||||
t.Fatalf("chain_moved says %v: %v", moved, err)
|
||||
}
|
||||
kept, found, err := inv.chain.Builds.Get(ctx, "build-2")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("build-2 is not in the bucket: %v %v", found, err)
|
||||
}
|
||||
if v := kept.Value; v.Module != "notes" || v.Commit != "c2" || v.Repository != "novox/notes" || v.Path != "modules/notes" ||
|
||||
v.SourceFingerprint != "fp-build-2" || len(v.Made) != 1 || len(v.Against) != 1 || len(v.Read) != 1 || len(v.Sources) != 1 ||
|
||||
v.Published == nil || v.Published.Artifact != "sha256:build-2" || v.Asked.IsZero() || v.At.IsZero() || len(v.Manifest) == 0 {
|
||||
t.Fatalf("build-2 was moved as %+v", v)
|
||||
}
|
||||
|
||||
// From now on the bucket is what is read and written.
|
||||
if err := inv.RecordBuild(ctx, aBuild("build-4", "notes", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "build"); n != "3" {
|
||||
t.Fatalf("the table took a build after the move: %s rows", n)
|
||||
}
|
||||
after, err := inv.Builds(ctx, "notes", 10)
|
||||
if err != nil || len(after) != 3 {
|
||||
t.Fatalf("notes has %d builds on the bus: %v", len(after), err)
|
||||
}
|
||||
|
||||
// A second run finds the kind moved and writes nothing: every key still has the one value it was made with.
|
||||
again, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allAlready(t, again)
|
||||
for _, id := range []string{"build-1", "build-2", "build-3", "build-4"} {
|
||||
if history, err := inv.chain.Builds.History(ctx, chainBuildKey(id)); err != nil || len(history) != 1 {
|
||||
t.Fatalf("%s has %d values: %v", id, len(history), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// failingToCreate is a bucket that fails to make one key while told to: the bus going away half way through a move.
|
||||
type failingToCreate struct {
|
||||
chain.Backend
|
||||
key string
|
||||
fail *atomic.Bool
|
||||
}
|
||||
|
||||
func (f failingToCreate) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if key == f.key && f.fail.Load() {
|
||||
return 0, errors.New("the bus went away")
|
||||
}
|
||||
return f.Backend.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
// **A controller restarted half way through the move finishes it: the bucket equals the table, with no key made twice and
|
||||
// no refusal** (ADR 0408 §8). Before a kind is recorded as moved nothing but the move writes its keys, and every write of
|
||||
// the kind goes to its table (each method of a kind asks ChainReads first and writes the store until it says moved), so the
|
||||
// table is the only truth: a key the first start made is the row it was, and the second start settles it to the row it is.
|
||||
func TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
aBuildRow(t, inv, "build-2", "notes", "c2", "")
|
||||
aBuildRow(t, inv, "build-3", "wiki", "c3", "")
|
||||
var away atomic.Bool
|
||||
away.Store(true)
|
||||
builds := chain.NewMemory()
|
||||
inv.UseChain(chain.New(failingToCreate{builds, "build-3", &away}, chain.NewMemory(), chain.NewMemory(),
|
||||
func(context.Context) error { return nil }))
|
||||
// The first start made two of the three keys and the bus went away.
|
||||
_, err := inv.MoveChain(ctx, nil)
|
||||
var refusal *Refusal
|
||||
if err == nil || errors.As(err, &refusal) {
|
||||
t.Fatalf("the first start answered %v: it should fail to be tried again, not be refused", err)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("a move that did not finish recorded %v", moved)
|
||||
}
|
||||
if keys, _ := builds.Keys(ctx); len(keys) != 2 {
|
||||
t.Fatalf("the first start made %v", keys)
|
||||
}
|
||||
// The controller is started again, the bus is back, and the move finishes.
|
||||
away.Store(false)
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the second start was refused or failed: %v", err)
|
||||
}
|
||||
if b := reportOf(t, reports, KindBuilds); b.Written != 1 || b.Present != 2 || b.Rows != 3 {
|
||||
t.Fatalf("the move reported %+v", b)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); moved[KindBuilds] != 3 {
|
||||
t.Fatalf("chain_moved says %v", moved)
|
||||
}
|
||||
// The bucket is the table: three keys, each made once and not again, each the row's own.
|
||||
for _, id := range []string{"build-1", "build-2", "build-3"} {
|
||||
history, err := builds.History(ctx, id)
|
||||
if err != nil || len(history) != 1 {
|
||||
t.Fatalf("%s has %d values: %v", id, len(history), err)
|
||||
}
|
||||
}
|
||||
rows, err := inv.allBuildRows(ctx)
|
||||
if err != nil || len(rows) != 3 {
|
||||
t.Fatalf("the table holds %d builds: %v", len(rows), err)
|
||||
}
|
||||
for _, row := range rows {
|
||||
got, found, err := inv.chain.Builds.Get(ctx, row.ID)
|
||||
want, _ := json.Marshal(toChainBuild(row))
|
||||
have, _ := json.Marshal(got.Value)
|
||||
if err != nil || !found || string(want) != string(have) {
|
||||
t.Fatalf("%s in the bucket is %s, the row says %s (%v)", row.ID, have, want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A value over the bus's largest message stops the move with nothing written and nothing switched** (ADR 0408 §6):
|
||||
// a Refusal, no key in the bucket, no kind recorded, and the tables still what is read.
|
||||
func TestTheMoveRefusesAValueOverTheLargestMessage(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
inv.ExecForTest(t, `insert into build (id, repository, module, commit_hash, manifest)
|
||||
values ('build-big', 'novox/wiki', 'wiki', 'c9', ('{"x":"' || repeat('a', 1100000) || '"}')::jsonb)`)
|
||||
_, err := inv.MoveChain(ctx, nil)
|
||||
var refusal *Refusal
|
||||
if !errors.As(err, &refusal) || refusal.Kind != KindBuilds {
|
||||
t.Fatalf("the move answered %v", err)
|
||||
}
|
||||
if keys, _ := inv.chain.Builds.Keys(ctx); len(keys) != 0 {
|
||||
t.Fatalf("the bucket holds %v after a refused move", keys)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("a refused move recorded %v", moved)
|
||||
}
|
||||
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 2 {
|
||||
t.Fatalf("the tables are no longer read: %d builds, %v", len(builds), err)
|
||||
}
|
||||
}
|
||||
|
||||
// losing is a bucket that says it wrote a key and kept nothing: the move's count of keys against rows must see it.
|
||||
type losing struct {
|
||||
chain.Backend
|
||||
lose string
|
||||
}
|
||||
|
||||
func (l losing) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if key == l.lose {
|
||||
return 1, nil
|
||||
}
|
||||
return l.Backend.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
// **The move refuses on a count that differs** (ADR 0408 §8): a row whose key is not in the bucket afterwards stops it,
|
||||
// nothing is recorded, and the tables are still read.
|
||||
func TestTheMoveRefusesOnACountThatDiffers(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
aBuildRow(t, inv, "build-2", "notes", "c2", "")
|
||||
inv.UseChain(chain.New(losing{chain.NewMemory(), "build-2"}, chain.NewMemory(), chain.NewMemory(),
|
||||
func(context.Context) error { return nil }))
|
||||
_, err := inv.MoveChain(ctx, nil)
|
||||
var refusal *Refusal
|
||||
if !errors.As(err, &refusal) {
|
||||
t.Fatalf("the move answered %v", err)
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("a refused move recorded %v", moved)
|
||||
}
|
||||
}
|
||||
|
||||
// **Without the lease the move writes nothing**, and fails the way the bus being away does — tried again, not refused:
|
||||
// the tables are still read.
|
||||
func TestTheMoveWithoutTheLeaseWritesNothing(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
mem := chain.NewMemory()
|
||||
inv.UseChain(chain.New(mem, chain.NewMemory(), chain.NewMemory(),
|
||||
func(context.Context) error { return errors.New("another controller holds the lease") }))
|
||||
_, err := inv.MoveChain(ctx, nil)
|
||||
var refusal *Refusal
|
||||
if err == nil || errors.As(err, &refusal) || !errors.Is(err, chain.ErrNotHeld) {
|
||||
t.Fatalf("the move answered %v", err)
|
||||
}
|
||||
if mem.Writes() != 0 {
|
||||
t.Fatalf("the bucket took %d writes without the lease", mem.Writes())
|
||||
}
|
||||
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
|
||||
t.Fatalf("recorded %v", moved)
|
||||
}
|
||||
}
|
||||
|
||||
// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its
|
||||
// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none.
|
||||
func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
first := aBuild("build-1", "notes", "")
|
||||
first.Commit = "c1"
|
||||
first.Against = []string{"artifact-store://base@sha256:aa"}
|
||||
first.ReuseKey = "rk-1"
|
||||
second := aBuild("build-2", "notes", "")
|
||||
second.Commit = "c2"
|
||||
failed := aBuild("build-3", "notes", "compile error")
|
||||
other := aBuild("build-4", "wiki", "")
|
||||
for _, b := range []Build{first, second, failed, other, first} {
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "build"); n != "0" {
|
||||
t.Fatalf("the table took %s builds", n)
|
||||
}
|
||||
kept, found, err := inv.chain.Builds.Get(ctx, "build-1")
|
||||
if err != nil || !found || kept.Value.ReuseKey != "rk-1" {
|
||||
t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err)
|
||||
}
|
||||
if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 1 {
|
||||
t.Fatalf("a build recorded twice has %d values", len(history))
|
||||
}
|
||||
list, err := inv.Builds(ctx, "notes", 10)
|
||||
if err != nil || len(list) != 3 {
|
||||
t.Fatalf("notes has %d builds: %v", len(list), err)
|
||||
}
|
||||
one, found, err := inv.BuildByID(ctx, "build-2")
|
||||
if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
|
||||
t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
|
||||
}
|
||||
if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
|
||||
t.Fatal("a build nobody recorded was found")
|
||||
}
|
||||
if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" {
|
||||
t.Fatalf("the build of notes at c1 is %q: %v", id, err)
|
||||
}
|
||||
held, err := inv.Held(ctx)
|
||||
if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" {
|
||||
t.Fatalf("held %v: %v", held, err)
|
||||
}
|
||||
against, err := inv.BuiltAgainst(ctx)
|
||||
if err != nil || len(against["notes"]) != 0 {
|
||||
// The newest successful build of notes, build-2, stood on nothing.
|
||||
t.Fatalf("built against %v: %v", against, err)
|
||||
}
|
||||
announced, err := inv.Announceable(ctx)
|
||||
if err != nil || len(announced) != 3 {
|
||||
t.Fatalf("announceable %d: %v", len(announced), err)
|
||||
}
|
||||
prints, err := inv.Fingerprints(ctx)
|
||||
if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
|
||||
t.Fatalf("fingerprints %v: %v", prints, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build request row as the store held it before the move.
|
||||
func aRequestRow(t *testing.T, inv *Inventory, id, notAsked string) {
|
||||
t.Helper()
|
||||
inv.ExecForTest(t,
|
||||
`insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at)
|
||||
values ($1::text, 'novox/notes', '', 'modules/notes', 'main', 'c1', 'merge', nullif($2::text, ''), now() - interval '1 hour')`,
|
||||
id, notAsked)
|
||||
}
|
||||
|
||||
// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue
|
||||
// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none
|
||||
// after the outcome is heard. The table of requests takes none.
|
||||
func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true}
|
||||
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
|
||||
t.Fatalf("a request kept twice: %v", err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "build_request"); n != "0" {
|
||||
t.Fatalf("the table took %s requests", n)
|
||||
}
|
||||
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 {
|
||||
t.Fatalf("a request nobody answered is a build: %+v %v", builds, err)
|
||||
}
|
||||
if _, found, _ := inv.BuildByID(ctx, "build-1"); found {
|
||||
t.Fatal("a request nobody answered has an outcome")
|
||||
}
|
||||
got, found, err := inv.BuildRequestByID(ctx, "build-1")
|
||||
if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() {
|
||||
t.Fatalf("the request reads back as %+v %v %v", got, found, err)
|
||||
}
|
||||
if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, err := inv.RequestsNamed(ctx, "notes")
|
||||
if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" {
|
||||
t.Fatalf("the request is %+v: %v", asked, err)
|
||||
}
|
||||
if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) {
|
||||
t.Fatalf("requested names %v: %v", names, err)
|
||||
}
|
||||
|
||||
// The outcome comes: the same key, now a build, with the request beside it.
|
||||
built := aBuild("build-1", "notes", "")
|
||||
if err := inv.RecordBuild(ctx, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" {
|
||||
t.Fatalf("the outcome reads as %+v %v %v", one, found, err)
|
||||
}
|
||||
answered, err := inv.RequestsNamed(ctx, "notes")
|
||||
if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" {
|
||||
t.Fatalf("the request answered is %+v: %v", answered, err)
|
||||
}
|
||||
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" {
|
||||
t.Fatalf("the request was lost with the outcome: %+v", again)
|
||||
}
|
||||
// Heard: the marks of an unanswered request change nothing.
|
||||
if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" {
|
||||
t.Fatalf("an answered request was marked not asked: %+v", again)
|
||||
}
|
||||
if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 3 {
|
||||
// asked, marked unknown, answered.
|
||||
t.Fatalf("the record has %d values", len(history))
|
||||
}
|
||||
// An outcome heard before its request is kept, and the request lands beside it.
|
||||
if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" {
|
||||
t.Fatalf("the request kept after its outcome reads %+v", late)
|
||||
}
|
||||
// A request older than the time they are kept is no longer listed.
|
||||
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old",
|
||||
At: time.Now().Add(-2 * KeptFor)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") {
|
||||
t.Fatalf("a request past its time is listed: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a
|
||||
// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it
|
||||
// is asked and so asked at most once.
|
||||
func TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
aRequestRow(t, inv, "build-1", "")
|
||||
aRequestRow(t, inv, "build-2", "")
|
||||
aRequestRow(t, inv, "not-asked-c1-modules-notes", "no seat holder")
|
||||
|
||||
if requests, err := inv.RequestsNamed(ctx, "notes"); err != nil || len(requests) != 3 {
|
||||
t.Fatalf("before the move the table holds %d requests: %v", len(requests), err)
|
||||
}
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.TableRows != 4 {
|
||||
t.Fatalf("the move reported %+v", b)
|
||||
}
|
||||
one, _, _ := inv.chain.Builds.Get(ctx, "build-1")
|
||||
if one.Value.State != "" || one.Value.Request == nil || one.Value.Request.For != "merge" || one.Value.Module != "notes" {
|
||||
t.Fatalf("an answered request moved as %+v", one.Value)
|
||||
}
|
||||
two, _, _ := inv.chain.Builds.Get(ctx, "build-2")
|
||||
if two.Value.State != chain.BuildAsked || two.Value.Request.Reask != "" {
|
||||
t.Fatalf("an unanswered request moved as %+v", two.Value)
|
||||
}
|
||||
never, _, _ := inv.chain.Builds.Get(ctx, "not-asked-c1-modules-notes")
|
||||
if never.Value.State != chain.BuildAsked || never.Value.Request.NotAsked != "no seat holder" || never.Value.Request.Reask != chain.ReaskPending {
|
||||
t.Fatalf("a request never handed over moved as %+v", never.Value)
|
||||
}
|
||||
requests, err := inv.RequestsNamed(ctx, "notes")
|
||||
if err != nil || len(requests) != 3 {
|
||||
t.Fatalf("after the move the bucket holds %d requests: %v", len(requests), err)
|
||||
}
|
||||
toAsk, err := inv.RequestsToAskAgain(ctx)
|
||||
if err != nil || len(toAsk) != 1 || toAsk[0].ID != "not-asked-c1-modules-notes" {
|
||||
t.Fatalf("to ask again: %+v %v", toAsk, err)
|
||||
}
|
||||
first, err := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
|
||||
second, err2 := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
|
||||
if err != nil || err2 != nil || !first || second {
|
||||
t.Fatalf("claimed %v then %v: %v %v", first, second, err, err2)
|
||||
}
|
||||
if left, _ := inv.RequestsToAskAgain(ctx); len(left) != 0 {
|
||||
t.Fatalf("a claimed request is still to be asked: %+v", left)
|
||||
}
|
||||
if err := inv.AskedAgainAs(ctx, toAsk[0].ID, "build-9"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if done, _, _ := inv.BuildRequestByID(ctx, toAsk[0].ID); done.NotAsked != "no seat holder" {
|
||||
t.Fatalf("the request lost its words: %+v", done)
|
||||
}
|
||||
}
|
||||
|
||||
// **The move carries every plan with the merges it answers, and the plans read the same from the bucket as from the
|
||||
// table** (ADR 0408 §8, ADR 0276, issue 1092): a batch with a merge, a walk that was built and rolled, a merge waiting
|
||||
// for a walk of its own, and a merge whose plan the store never kept — each in the record of the plan that held it last.
|
||||
func TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
batch := aBatch("plan-1", time.Now().Add(-3*time.Hour).UTC())
|
||||
walk := aBatch("plan-2", time.Now().Add(-2*time.Hour).UTC())
|
||||
walk.State, walk.Tier, walk.Tiers = PlanRolling, 1, [][]string{{"a"}, {"b"}}
|
||||
walk.Modules = map[string]*PlanModule{"a": {State: "built"}, "b": {}}
|
||||
walk.Delivery = &PlanDelivery{Awaits: "mesh-delivery", Merges: []PlanMerge{{Repository: "novox/mesh-catalog", Commit: "m-waiting"}}}
|
||||
for _, p := range []*Plan{batch, walk} {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
walk.Note = "second save"
|
||||
if err := inv.SavePlan(ctx, walk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []BatchedMerge{aMerge("m-batch", "plan-1"), aMerge("m-walk", "plan-2"), aMerge("m-dangling", "plan-never-kept")} {
|
||||
if added, err := inv.AddMerge(ctx, m); err != nil || !added {
|
||||
t.Fatalf("added %v: %v", added, err)
|
||||
}
|
||||
}
|
||||
waiting := aMerge("m-waiting", "plan-2")
|
||||
if _, err := inv.AddMerge(ctx, waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.AnswerMerge(ctx, waiting.Repository, waiting.Commit, "", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasPlans, err := inv.RecentPlans(ctx, 10)
|
||||
if err != nil || len(wasPlans) != 2 {
|
||||
t.Fatalf("the table holds %d plans: %v", len(wasPlans), err)
|
||||
}
|
||||
wasMerge, _, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")
|
||||
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var moved MoveReport
|
||||
for _, r := range reports {
|
||||
if r.Kind == KindRollouts {
|
||||
moved = r
|
||||
}
|
||||
}
|
||||
// Two plans, and the record of the merge whose plan the store never kept.
|
||||
if moved.Rows != 3 || moved.TableRows != 6 || moved.Written != 3 {
|
||||
t.Fatalf("the move of the rollouts reported %+v", moved)
|
||||
}
|
||||
nowPlans, err := inv.RecentPlans(ctx, 10)
|
||||
if err != nil || len(nowPlans) != 2 {
|
||||
t.Fatalf("the bucket holds %d plans: %v", len(nowPlans), err)
|
||||
}
|
||||
for k := range wasPlans {
|
||||
was, now := wasPlans[k], nowPlans[k]
|
||||
if was.ID != now.ID || was.State != now.State || was.Tier != now.Tier || was.Revision != now.Revision || was.Note != now.Note ||
|
||||
!was.Created.Equal(now.Created) || !reflect.DeepEqual(was.Tiers, now.Tiers) || len(was.Modules) != len(now.Modules) {
|
||||
t.Errorf("plan %s was %+v and is %+v", was.ID, was, now)
|
||||
}
|
||||
}
|
||||
if open, _ := inv.OpenPlans(ctx); len(open) != 1 || open[0].ID != "plan-2" {
|
||||
t.Fatalf("open plans %+v", open)
|
||||
}
|
||||
if batches, _ := inv.Batches(ctx); len(batches) != 1 || batches[0].ID != "plan-1" {
|
||||
t.Fatalf("batches %+v", batches)
|
||||
}
|
||||
if got, _ := inv.MergesOf(ctx, "plan-1"); len(got) != 1 || got[0].Commit != "m-batch" {
|
||||
t.Fatalf("the batch answers %+v", got)
|
||||
}
|
||||
nowMerge, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")
|
||||
if !known || nowMerge.Plan != "plan-2" || nowMerge.Branch != wasMerge.Branch || !nowMerge.Merged.Equal(wasMerge.Merged) ||
|
||||
!nowMerge.Heard.Equal(wasMerge.Heard) || !sameEvent(nowMerge.Event, wasMerge.Event) {
|
||||
t.Fatalf("a merge was %+v and is %+v", wasMerge, nowMerge)
|
||||
}
|
||||
alone, _ := inv.AloneMerges(ctx)
|
||||
if len(alone) != 1 || alone[0].Commit != "m-waiting" || alone[0].Plan != "" {
|
||||
t.Fatalf("waiting: %+v", alone)
|
||||
}
|
||||
if dangling, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-dangling"); !known || dangling.Plan != "" || dangling.Alone {
|
||||
t.Fatalf("a merge whose plan was never kept reads as %+v %v", dangling, known)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
|
||||
t.Fatalf("the table is %s plans", n)
|
||||
}
|
||||
// From now on a plan is saved to the bucket and the table takes none.
|
||||
later := aBatch("plan-3", time.Now().UTC())
|
||||
if err := inv.SavePlan(ctx, later); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
|
||||
t.Fatalf("the table took a plan after the move: %s", n)
|
||||
}
|
||||
again, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allAlready(t, again)
|
||||
}
|
||||
|
||||
// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
|
||||
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
|
||||
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
|
||||
// assignment goes with it.
|
||||
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres", "web")
|
||||
ctx := t.Context()
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := generationNow(t, inv)
|
||||
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
|
||||
t.Fatalf("assigned %v: %v", added, err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added {
|
||||
t.Fatalf("assigned again %v: %v", added, err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "assignment"); n != "0" {
|
||||
t.Fatalf("the table took %s assignments", n)
|
||||
}
|
||||
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
|
||||
if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
|
||||
t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
|
||||
}
|
||||
if generationNow(t, inv) <= start {
|
||||
t.Fatal("an assignment left the generation where it was")
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
|
||||
t.Fatalf("a module the mesh does not know: %v", err)
|
||||
}
|
||||
if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) {
|
||||
t.Fatalf("assigned on %s: %v %v", node, on, err)
|
||||
}
|
||||
if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) {
|
||||
t.Fatalf("running: %v %v", running, err)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
switch e.Manifest.Module {
|
||||
case "postgres":
|
||||
if !reflect.DeepEqual(e.On, []string{node}) {
|
||||
t.Errorf("postgres is on %v", e.On)
|
||||
}
|
||||
case "web":
|
||||
if len(e.On) != 0 {
|
||||
t.Errorf("web is on %v", e.On)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) {
|
||||
t.Fatalf("a module still assigned was forgotten: %v", err)
|
||||
}
|
||||
// A seat is held by an assignment, or by nothing.
|
||||
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil {
|
||||
t.Fatal("a seat was handed to a module that is not assigned")
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil {
|
||||
t.Fatalf("a seat handed to an assignment: %v", err)
|
||||
}
|
||||
|
||||
before := generationNow(t, inv)
|
||||
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if generationNow(t, inv) <= before {
|
||||
t.Fatal("an unassignment left the generation where it was")
|
||||
}
|
||||
// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
|
||||
gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
|
||||
if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
|
||||
t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err)
|
||||
}
|
||||
if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() ||
|
||||
history[1].Value.Stands() {
|
||||
t.Fatalf("the earlier values of the key: %+v", history)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
|
||||
t.Fatalf("an ended assignment is still assigned: %v", on)
|
||||
}
|
||||
if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 {
|
||||
t.Fatalf("an ended assignment is running: %v", running)
|
||||
}
|
||||
if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 {
|
||||
t.Fatalf("the seat the assignment held is held by %+v: %v", held, err)
|
||||
}
|
||||
if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" {
|
||||
t.Fatalf("an unassignment of nothing: %v", err)
|
||||
}
|
||||
if err := inv.ForgetModule(ctx, "postgres"); err != nil {
|
||||
t.Fatalf("a module nothing is assigned could not be forgotten: %v", err)
|
||||
}
|
||||
// Made again after it ended: a fresh value over the ended one, which stays in history.
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
|
||||
t.Fatal(added, err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || added {
|
||||
t.Fatal(added, err)
|
||||
}
|
||||
if err := inv.Unassign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
|
||||
t.Fatalf("assigned again after it ended: %v %v", added, err)
|
||||
}
|
||||
again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
|
||||
if !again.Value.Stands() || !again.Value.Ended.IsZero() {
|
||||
t.Fatalf("the assignment made again is %+v", again.Value)
|
||||
}
|
||||
if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() {
|
||||
t.Fatalf("the values of the key: %+v", history)
|
||||
}
|
||||
}
|
||||
|
||||
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
|
||||
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
|
||||
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
|
||||
// ends an assignment.
|
||||
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "dunst", "web")
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"dunst", "web"} {
|
||||
if _, err := inv.Assign(ctx, node, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A build and a commit put in the value by another write.
|
||||
cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
|
||||
cur.Value.Build, cur.Value.Commit = "build-1", "c1"
|
||||
if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _ := inv.NodeByName(ctx, node)
|
||||
digest := "sha256:" + strings.Repeat("d", 64)
|
||||
plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
|
||||
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
|
||||
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
|
||||
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
written, removed, err := inv.KeepAssignmentPlans(ctx)
|
||||
if err != nil || written != 2 || removed != 0 {
|
||||
t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
|
||||
}
|
||||
dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
|
||||
if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
|
||||
t.Fatalf("dunst's value is %+v", dunst.Value)
|
||||
}
|
||||
if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
|
||||
t.Fatalf("web has a plan: %+v", web.Value)
|
||||
}
|
||||
notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify")
|
||||
if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil {
|
||||
t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
|
||||
t.Fatalf("a pulled-in module is assigned: %v", on)
|
||||
}
|
||||
if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 {
|
||||
t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err)
|
||||
}
|
||||
// Sent again without the pulled-in module: its value ends, the assignments stay.
|
||||
only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
|
||||
t.Fatalf("ended %d: %v", removed, err)
|
||||
}
|
||||
if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() {
|
||||
t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
|
||||
t.Fatalf("assigned: %v", on)
|
||||
}
|
||||
}
|
||||
|
||||
// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
|
||||
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
|
||||
// already is left as it is.
|
||||
func TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
for _, name := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.AddNode(ctx, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"notes", "wiki"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "notes"}, {"anchor", "wiki"}, {"laptop", "notes"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
aBuildRow(t, inv, "build-notes", "notes", "c-notes", "")
|
||||
anchor, _ := inv.NodeByName(ctx, "anchor")
|
||||
if err := inv.RecordSent(ctx, anchor.ID, "sent-1", map[string]string{"notes": "c-notes"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The mirror of the store put a value under one key, with a plan.
|
||||
if _, err := inv.chain.Assignments.Create(ctx, "anchor.wiki", chain.Assignment{Node: "anchor", Module: "wiki",
|
||||
Assigned: time.Now().UTC(), Provisioning: json.RawMessage(`{"plan":1}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reports, err := inv.MoveChain(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var moved MoveReport
|
||||
for _, r := range reports {
|
||||
if r.Kind == KindAssignments {
|
||||
moved = r
|
||||
}
|
||||
}
|
||||
if moved.Rows != 3 || moved.Written+moved.Present != 3 {
|
||||
t.Fatalf("the move of the assignments reported %+v", moved)
|
||||
}
|
||||
notes, _, _ := inv.chain.Assignments.Get(ctx, "anchor.notes")
|
||||
if notes.Value.Commit != "c-notes" || notes.Value.Build != "build-notes" || notes.Value.Since.IsZero() || notes.Value.Assigned.IsZero() {
|
||||
t.Fatalf("anchor.notes moved as %+v", notes.Value)
|
||||
}
|
||||
if notes.Value.Artifact != "sha256:build-notes" {
|
||||
t.Fatalf("what the build published: %+v", notes.Value)
|
||||
}
|
||||
laptop, _, _ := inv.chain.Assignments.Get(ctx, "laptop.notes")
|
||||
if laptop.Value.Commit != "" || laptop.Value.Build != "" {
|
||||
t.Fatalf("a machine never sent has a build: %+v", laptop.Value)
|
||||
}
|
||||
wiki, _, _ := inv.chain.Assignments.Get(ctx, "anchor.wiki")
|
||||
if string(wiki.Value.Provisioning) != `{"plan":1}` {
|
||||
t.Fatalf("the plan the mirror put in the value was lost: %+v", wiki.Value)
|
||||
}
|
||||
// From now on the bucket is what is written.
|
||||
if _, err := inv.Assign(ctx, "laptop", "wiki"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := rowsInTheTable(t, inv, "assignment"); n != "3" {
|
||||
t.Fatalf("the table took an assignment after the move: %s", n)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, "laptop"); !reflect.DeepEqual(on, []string{"notes", "wiki"}) {
|
||||
t.Fatalf("assigned on laptop: %v", on)
|
||||
}
|
||||
}
|
||||
|
||||
// hooked runs a function the first time a key is made in a bucket: the old controller writing the store between the move's
|
||||
// first pass and its switch.
|
||||
type hooked struct {
|
||||
chain.Backend
|
||||
once *sync.Once
|
||||
then func()
|
||||
}
|
||||
|
||||
func (h hooked) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
rev, err := h.Backend.Create(ctx, key, value)
|
||||
h.once.Do(h.then)
|
||||
return rev, err
|
||||
}
|
||||
|
||||
// **A row the old controller changes after the move's first pass is in the bucket when the kind is recorded as moved**
|
||||
// (ADR 0408 §8, issue 981 note 7 Q2): before the switch the store is the only truth, so under the lock the move reads the
|
||||
// rows again and writes over every key that differs — a plan's state and a build's failure — and ends an assignment the
|
||||
// store dropped. The count alone would have kept the older value and refused the next save of the plan.
|
||||
func TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved(t *testing.T) {
|
||||
inv, finish := ForTestBefore(t, 105)
|
||||
finish()
|
||||
ctx := context.Background()
|
||||
aBuildRow(t, inv, "build-1", "notes", "c1", "")
|
||||
for _, n := range []string{"anchor"} {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"notes", "wiki"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"notes", "wiki"} {
|
||||
if _, err := inv.Assign(ctx, "anchor", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p := aBatch("plan-1", time.Now().Add(-time.Hour).UTC())
|
||||
p.State, p.Tiers = PlanBuilding, [][]string{{"notes"}}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Between a kind's first pass and its switch the old controller saves the plan again, fails the build and unassigns wiki:
|
||||
// each is done when the first key of its kind is made, so the first pass has read the rows as they were.
|
||||
after := func(sql string) hooked {
|
||||
return hooked{chain.NewMemory(), &sync.Once{}, func() { inv.ExecForTest(t, sql) }}
|
||||
}
|
||||
inv.UseChain(chain.New(
|
||||
after(`update build set failed = 'it broke later' where id = 'build-1'`),
|
||||
after(`update release_plan set state = 'rolling', revision = revision + 1, note = 'the old controller was here' where id = 'plan-1'`),
|
||||
after(`delete from assignment where module = 'wiki'`),
|
||||
func(context.Context) error { return nil }))
|
||||
if _, err := inv.MoveChain(ctx, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now, err := inv.PlanByID(ctx, "plan-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if now.State != PlanRolling || now.Note != "the old controller was here" || now.Revision != 2 {
|
||||
t.Fatalf("the bucket holds a plan from before the change: %s %q revision %d", now.State, now.Note, now.Revision)
|
||||
}
|
||||
if b, found, _ := inv.BuildByID(ctx, "build-1"); !found || b.Failed != "it broke later" {
|
||||
t.Fatalf("the bucket holds the build as it was: %+v", b)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, "anchor"); !slices.Equal(on, []string{"notes"}) {
|
||||
t.Fatalf("anchor is assigned %v, though the store dropped wiki", on)
|
||||
}
|
||||
// The plan can be saved at the revision the store had.
|
||||
now.Note = "saved after the move"
|
||||
if err := inv.SavePlan(ctx, &now); err != nil {
|
||||
t.Fatalf("the next save of the plan is refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// sameEvent says two encodings of the forge's announcement are one: the store keeps it as jsonb, which spells it with its own
|
||||
// white space, and whoever reads it (the cut, mesh-delivery) decodes it, so the white space is no part of what it says.
|
||||
func sameEvent(a, b json.RawMessage) bool {
|
||||
var x, y bytes.Buffer
|
||||
return json.Compact(&x, a) == nil && json.Compact(&y, b) == nil && x.String() == y.String()
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The move of the chain has a switch, made by the last migration and written by no migration** (ADR 0408 §8, issue
|
||||
// 1089): migration 0104 makes the one-row table chain_moved, and does not drop a table of the chain, which a later
|
||||
// delivery does once the move is read back.
|
||||
func TestMigration0104MakesTheSwitchOfTheChainsMoveAndNothingElse(t *testing.T) {
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, m := range migrations {
|
||||
if m.Number != 104 {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
sql := strings.ToLower(m.SQL)
|
||||
if !strings.Contains(sql, "create table chain_moved") || !strings.Contains(sql, "check (only_one)") {
|
||||
t.Errorf("migration 0104 does not make the one-row chain_moved:\n%s", m.SQL)
|
||||
}
|
||||
for _, line := range strings.Split(sql, "\n") {
|
||||
if strings.HasPrefix(strings.TrimSpace(line), "--") {
|
||||
continue
|
||||
}
|
||||
for _, verb := range []string{"drop table", "insert ", "delete ", "update "} {
|
||||
if strings.Contains(line, verb) {
|
||||
t.Errorf("migration 0104 does more than make the switch: %q", line)
|
||||
}
|
||||
}
|
||||
// The one other change: a seat's holder is no longer a foreign key to the assignment (issue 1093); the tables of
|
||||
// the chain are dropped by a later migration (issue 1096).
|
||||
if strings.Contains(line, "alter ") && !strings.Contains(line, "alter table seat_holding drop constraint") {
|
||||
t.Errorf("migration 0104 alters more than the seat holdings' foreign key: %q", line)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("no migration 0104")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **A pending assignment is the value of the key of the assignment it waits for** (ADR 0261, ADR 0408, issue 1094): state
|
||||
// pending, the wait in its `pending` object under an id that is a string, no table, and one open wait per machine and module.
|
||||
func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web")
|
||||
ctx := t.Context()
|
||||
p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-1", Repository: "novox/mesh-catalog",
|
||||
Path: "/modules/sensors/"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := fmt.Sprintf("p-%d", p.Since.UnixNano()); p.ID != want || p.State != PendingWaiting || p.Path != "modules/sensors" {
|
||||
t.Fatalf("recorded %+v, want id %s", p, want)
|
||||
}
|
||||
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.sensors")
|
||||
if err != nil || !found || kept.Value.State != chain.StatePending || kept.Value.Stands() || !kept.Value.Assigned.IsZero() ||
|
||||
kept.Value.Pending == nil || kept.Value.Pending.ID != p.ID || kept.Value.Pending.Build != "build-1" {
|
||||
t.Fatalf("the key holds %+v (found %v, %v)", kept.Value, found, err)
|
||||
}
|
||||
rows, err := inv.Pending(ctx, time.Time{})
|
||||
if err != nil || len(rows) != 1 || rows[0].ID != p.ID || rows[0].Node != "anchor" || rows[0].Build != "build-1" {
|
||||
t.Fatalf("waiting: %+v %v", rows, err)
|
||||
}
|
||||
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}); !errors.Is(err, ErrAlreadyPending) {
|
||||
t.Fatalf("a second open wait: %v", err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "build-3"}); !errors.Is(err, ErrOnTheMachine) {
|
||||
t.Fatalf("a wait for a module that is on the machine: %v", err)
|
||||
}
|
||||
// Written over once it ended: a settled wait is replaced by a new one under a new id.
|
||||
if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil || !ok {
|
||||
t.Fatalf("settled %v %v", ok, err)
|
||||
}
|
||||
next, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"})
|
||||
if err != nil || next.ID == p.ID {
|
||||
t.Fatalf("a wait after one ended: %+v %v", next, err)
|
||||
}
|
||||
// The earlier wait ended unmade and nobody answered its condition, so the new wait carries it (issue 1108): both are on record.
|
||||
if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); !known[p.ID] || !known[next.ID] {
|
||||
t.Fatalf("known %v: the key holds the newer one and carries the unanswered earlier one", known)
|
||||
}
|
||||
// The first wait, its settling and the new wait: three values, the earlier wait still in the history.
|
||||
if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 3 {
|
||||
t.Fatalf("the earlier wait is in the key's history: %d values", len(history))
|
||||
}
|
||||
}
|
||||
|
||||
// **A pending value is skipped by what reads assignments** (issue 1094): the machine is assigned nothing by it, the module
|
||||
// runs nowhere, and a module only waiting for a build is not forgotten as assigned.
|
||||
func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web")
|
||||
ctx := t.Context()
|
||||
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
|
||||
t.Fatalf("assigned %v", on)
|
||||
}
|
||||
if running, _ := inv.Running(ctx, "web"); len(running) != 0 {
|
||||
t.Fatalf("running %v", running)
|
||||
}
|
||||
if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 {
|
||||
t.Fatalf("runs %v", runs)
|
||||
}
|
||||
entries, _ := inv.Catalogued(ctx)
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == "web" && len(e.On) != 0 {
|
||||
t.Fatalf("web is on %v", e.On)
|
||||
}
|
||||
}
|
||||
if err := inv.ForgetModule(ctx, "web"); errors.Is(err, ErrStillAssigned) {
|
||||
t.Fatalf("a module only waiting is assigned: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Two claims of one wait make one, and the other sees false** (issue 1094): by compare-and-set on the key's revision.
|
||||
func TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var won atomic.Int32
|
||||
var wg sync.WaitGroup
|
||||
for k := 0; k < 8; k++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if ok, err := inv.ClaimPending(ctx, p.ID); err == nil && ok {
|
||||
won.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if won.Load() != 1 {
|
||||
t.Fatalf("%d claims won", won.Load())
|
||||
}
|
||||
if again, err := inv.ClaimPending(ctx, p.ID); err != nil || again {
|
||||
t.Fatalf("a claim of a claimed wait: %v %v", again, err)
|
||||
}
|
||||
if rows, _ := inv.Stuck(ctx, time.Now().Add(time.Minute)); len(rows) != 1 || rows[0].State != PendingApplying || rows[0].Settled == nil {
|
||||
t.Fatalf("stuck %+v", rows)
|
||||
}
|
||||
if err := inv.ReleaseClaim(ctx, p.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows, _ := inv.Pending(ctx, time.Time{}); len(rows) != 1 || rows[0].Settled != nil {
|
||||
t.Fatalf("released %+v", rows)
|
||||
}
|
||||
// Only a wait that waits is withdrawn or expired, only a claim is applied.
|
||||
if ok, _ := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingApplied, "x"); ok {
|
||||
t.Fatal("a wait was applied")
|
||||
}
|
||||
if ok, _ := inv.SettlePending(ctx, p.ID, PendingApplying, PendingRefused, "x"); ok {
|
||||
t.Fatal("a wait that is not claimed was refused as a claim")
|
||||
}
|
||||
if ok, err := inv.RepointPending(ctx, p.ID, "b-2", "novox/other", "x/"); err != nil || !ok {
|
||||
t.Fatalf("repoint %v %v", ok, err)
|
||||
}
|
||||
if rows, _ := inv.Pending(ctx, time.Time{}); rows[0].Build != "b-2" || rows[0].Path != "x" {
|
||||
t.Fatalf("repointed %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// **Assigning over a wait carries the wait into the value that stands** (issue 1094): the claim stays applying for the one
|
||||
// making it to settle applied, and one still waiting is made by hand, and says so.
|
||||
func TestAssignOverAPendingKeepsItsRecordAsApplied(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web", "cache")
|
||||
ctx := t.Context()
|
||||
claimed, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"})
|
||||
if ok, err := inv.ClaimPending(ctx, claimed.ID); err != nil || !ok {
|
||||
t.Fatal(ok, err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
|
||||
t.Fatalf("assigned %v %v", added, err)
|
||||
}
|
||||
standing, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
|
||||
if !standing.Value.Stands() || standing.Value.Assigned.IsZero() || standing.Value.Pending == nil ||
|
||||
standing.Value.Pending.State != PendingApplying {
|
||||
t.Fatalf("the value that stands: %+v", standing.Value)
|
||||
}
|
||||
if ok, err := inv.SettlePending(ctx, claimed.ID, PendingApplying, PendingApplied, "made"); err != nil || !ok {
|
||||
t.Fatalf("settled %v %v", ok, err)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); len(on) != 1 || on[0] != "web" {
|
||||
t.Fatalf("assigned %v", on)
|
||||
}
|
||||
rows, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
|
||||
if len(rows) != 1 || rows[0].State != PendingApplied || rows[0].Note != "made" {
|
||||
t.Fatalf("what became of the wait: %+v", rows)
|
||||
}
|
||||
|
||||
waiting, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "cache", Build: "b-2"})
|
||||
if _, err := inv.Assign(ctx, node, "cache"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := inv.PendingFor(ctx, node, "cache")
|
||||
if len(got) != 1 || got[0].ID != waiting.ID || got[0].State != PendingApplied || !strings.Contains(got[0].Note, "by hand") {
|
||||
t.Fatalf("a wait assigned by hand: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Marks are stamped once, and a wait of a machine the store dropped is no wait** (issue 1094).
|
||||
func TestAPendingIsMarkedOnceAndGoesWithItsMachine(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
p, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"})
|
||||
if _, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows, _ := inv.ToRaise(ctx); len(rows) != 1 {
|
||||
t.Fatalf("to raise %+v", rows)
|
||||
}
|
||||
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
|
||||
if !first[0].Raised.Equal(*second[0].Raised) {
|
||||
t.Fatalf("the first stamp did not stand: %v then %v", first[0].Raised, second[0].Raised)
|
||||
}
|
||||
if err := inv.MarkPending(ctx, p.ID, "frobnicated"); err == nil {
|
||||
t.Fatal("a mark nobody knows")
|
||||
}
|
||||
if rows, _ := inv.ToRaise(ctx); len(rows) != 0 {
|
||||
t.Fatalf("to raise %+v", rows)
|
||||
}
|
||||
if rows, _ := inv.ToClear(ctx); len(rows) != 1 {
|
||||
t.Fatalf("to clear %+v", rows)
|
||||
}
|
||||
if _, err := inv.RemoveNodeForTest(ctx, node); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows, _ := inv.Pending(ctx, time.Unix(0, 0)); len(rows) != 0 {
|
||||
t.Fatalf("a wait outlived its machine: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// **A wait replaced by a newer one never leaves the operator without the condition it raised, or was to raise** (issue
|
||||
// 1108): the key carries the replaced wait while its condition is unanswered, still known, still to clear, and a newer
|
||||
// wait takes only the unanswered ones forward.
|
||||
func TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record := func(build string) PendingAssignment {
|
||||
p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: build})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
end := func(p PendingAssignment) {
|
||||
if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom "+p.Build); err != nil || !ok {
|
||||
t.Fatal(ok, err)
|
||||
}
|
||||
}
|
||||
first := record("b-1")
|
||||
end(first)
|
||||
if err := inv.MarkPending(ctx, first.ID, "raised"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := record("b-2")
|
||||
if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID}); !known[first.ID] || !known[second.ID] {
|
||||
t.Fatalf("known %v: the replaced wait's condition is unanswered", known)
|
||||
}
|
||||
if clear, _ := inv.ToClear(ctx); len(clear) != 1 || clear[0].ID != first.ID {
|
||||
t.Fatalf("to clear %+v: the first's condition is still to clear", clear)
|
||||
}
|
||||
rows, _ := inv.PendingFor(ctx, node, "sensors")
|
||||
if len(rows) != 2 || rows[0].ID != second.ID || rows[1].ID != first.ID {
|
||||
t.Fatalf("the key says %+v, newest first", rows)
|
||||
}
|
||||
// The second ends unmade before the first was answered: both are to be said, the first's condition not cleared by it.
|
||||
end(second)
|
||||
if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID {
|
||||
t.Fatalf("to raise %+v", raise)
|
||||
}
|
||||
// A third replaces the second, which was not raised yet: it is carried, with the first (still unanswered).
|
||||
if err := inv.MarkPending(ctx, first.ID, "acknowledged"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
third := record("b-3")
|
||||
if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID {
|
||||
t.Fatalf("the second was never raised and is to be: %+v", raise)
|
||||
}
|
||||
if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID, third.ID}); known[first.ID] || !known[second.ID] || !known[third.ID] {
|
||||
t.Fatalf("known %v: the first was answered (taken back) and is let go, the second is not", known)
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,14 @@ func (i *Inventory) chainRecordBuildRequest(ctx context.Context, c *chain.Chain,
|
||||
if a.At.IsZero() {
|
||||
a.At = time.Now().UTC()
|
||||
}
|
||||
key := chainBuildKey(a.ID)
|
||||
key := chain.RequestKey(a.ID)
|
||||
// An outcome under a module that came first has no request beside it: the request is kept on it, and its own record says
|
||||
// it was answered (issue 1121).
|
||||
if ptr, found, err := c.Builds.GetRaw(ctx, chain.IDKey(a.ID)); err != nil {
|
||||
return err
|
||||
} else if found && string(ptr.Value) != chain.RequestPrefix {
|
||||
return i.keepAnsweredRequest(ctx, c, string(ptr.Value), a)
|
||||
}
|
||||
for try := 0; try < casTries; try++ {
|
||||
_, err := c.Builds.Create(ctx, key, askedRecord(a))
|
||||
if err == nil {
|
||||
@@ -65,8 +72,35 @@ func (i *Inventory) chainRecordBuildRequest(ctx context.Context, c *chain.Chain,
|
||||
return fmt.Errorf("build request %s is not kept on the bus: it kept moving under the write", a.ID)
|
||||
}
|
||||
|
||||
// keepAnsweredRequest puts a request on the outcome that answered it before the request was kept, and keeps the request's
|
||||
// own record as answered.
|
||||
func (i *Inventory) keepAnsweredRequest(ctx context.Context, c *chain.Chain, module string, a BuildRequest) error {
|
||||
outKey := chain.BuildKey(module, a.ID)
|
||||
for try := 0; try < casTries; try++ {
|
||||
out, found, err := c.Builds.Get(ctx, outKey)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
if out.Value.Request == nil {
|
||||
out.Value.Request = toRequest(a)
|
||||
if _, err := c.Builds.Update(ctx, outKey, out.Value, out.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("build request %s is not kept on the bus: %w", a.ID, err)
|
||||
}
|
||||
}
|
||||
rec := askedRecord(a)
|
||||
rec.State, rec.Module, rec.Failed, rec.At = chain.BuildAnswered, module, out.Value.Failed, out.Value.At
|
||||
if _, err := c.Builds.Create(ctx, chain.RequestKey(a.ID), rec); err != nil && !errors.Is(err, chain.ErrExists) {
|
||||
return fmt.Errorf("build request %s is not kept on the bus: %w", a.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("build request %s is not kept: it kept moving under the write", a.ID)
|
||||
}
|
||||
|
||||
func (i *Inventory) chainBuildRequestByID(ctx context.Context, c *chain.Chain, id string) (BuildRequest, bool, error) {
|
||||
k, found, err := c.Builds.Get(ctx, chainBuildKey(id))
|
||||
k, found, err := c.Builds.Get(ctx, chain.RequestKey(id))
|
||||
if err != nil || !found || k.Value.Request == nil {
|
||||
return BuildRequest{}, false, err
|
||||
}
|
||||
@@ -75,7 +109,7 @@ func (i *Inventory) chainBuildRequestByID(ctx context.Context, c *chain.Chain, i
|
||||
|
||||
// chainMarkRequest changes a request that has no outcome yet, by compare-and-set; one answered, or none kept, is left.
|
||||
func (i *Inventory) chainMarkRequest(ctx context.Context, c *chain.Chain, id string, mark func(*chain.Request)) error {
|
||||
key := chainBuildKey(id)
|
||||
key := chain.RequestKey(id)
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Builds.Get(ctx, key)
|
||||
if err != nil || !found || cur.Value.State != chain.BuildAsked || cur.Value.Request == nil {
|
||||
@@ -94,13 +128,14 @@ func (i *Inventory) chainMarkRequest(ctx context.Context, c *chain.Chain, id str
|
||||
// chainRequests is every request kept, newest first, each with its outcome where its build was heard. Requests older than
|
||||
// KeptFor are not listed: the bucket keeps them, and no reader needs them.
|
||||
func (i *Inventory) chainRequests(ctx context.Context, c *chain.Chain) ([]RequestOutcome, error) {
|
||||
all, err := c.Builds.All(ctx)
|
||||
all, err := requestRecords(ctx, c)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the builds cannot be read from the bus: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
oldest := time.Now().Add(-KeptFor)
|
||||
var out []RequestOutcome
|
||||
for id, k := range all {
|
||||
for _, k := range all {
|
||||
id := strings.TrimPrefix(k.Key, chain.RequestPrefix+".")
|
||||
r := k.Value.Request
|
||||
if r == nil || !r.At.After(oldest) {
|
||||
continue
|
||||
@@ -120,19 +155,25 @@ func (i *Inventory) chainRequests(ctx context.Context, c *chain.Chain) ([]Reques
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// requestRecords are the records under `req.`: the requests, and the builds that knew no module.
|
||||
func requestRecords(ctx context.Context, c *chain.Chain) ([]chain.Kept[chain.Build], error) {
|
||||
return buildsIn(ctx, c, chain.RequestPrefix+".*")
|
||||
}
|
||||
|
||||
// RequestsToAskAgain are the requests the move found never handed over (issue 981 note 7 Q5), not yet asked again: each is
|
||||
// asked once, and never twice.
|
||||
func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindBuilds)
|
||||
if err != nil || !on {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.Builds.All(ctx)
|
||||
all, err := requestRecords(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BuildRequest
|
||||
for id, k := range all {
|
||||
for _, k := range all {
|
||||
id := strings.TrimPrefix(k.Key, chain.RequestPrefix+".")
|
||||
if k.Value.State == chain.BuildAsked && k.Value.Request != nil && k.Value.Request.Reask == chain.ReaskPending {
|
||||
out = append(out, fromRequest(id, *k.Value.Request))
|
||||
}
|
||||
@@ -144,16 +185,16 @@ func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, err
|
||||
// ClaimToAskAgain takes a request to ask again, by compare-and-set: true for the one process that took it. **Claimed
|
||||
// before it is asked and settled after**, so a crash between loses the ask rather than making it twice.
|
||||
func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindBuilds)
|
||||
if err != nil || !on {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
cur, found, err := c.Builds.Get(ctx, chainBuildKey(id))
|
||||
cur, found, err := c.Builds.Get(ctx, chain.RequestKey(id))
|
||||
if err != nil || !found || cur.Value.Request == nil || cur.Value.Request.Reask != chain.ReaskPending {
|
||||
return false, err
|
||||
}
|
||||
cur.Value.Request.Reask = chain.ReaskClaimed
|
||||
if _, err := c.Builds.Update(ctx, chainBuildKey(id), cur.Value, cur.Revision); err != nil {
|
||||
if _, err := c.Builds.Update(ctx, chain.RequestKey(id), cur.Value, cur.Revision); err != nil {
|
||||
if errors.Is(err, chain.ErrMoved) {
|
||||
return false, nil
|
||||
}
|
||||
@@ -164,8 +205,8 @@ func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error
|
||||
|
||||
// AskedAgainAs records the build id a claimed request was asked as.
|
||||
func (i *Inventory) AskedAgainAs(ctx context.Context, id, asked string) error {
|
||||
c, on, err := i.ChainReads(ctx, KindBuilds)
|
||||
if err != nil || !on {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.Reask = asked })
|
||||
|
||||
@@ -139,17 +139,17 @@ func (v *rolloutView) list() []viewEntry {
|
||||
|
||||
// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole.
|
||||
func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) {
|
||||
i.moved.mu.Lock()
|
||||
v := i.moved.view
|
||||
i.state.mu.Lock()
|
||||
v := i.state.view
|
||||
if v == nil || v.chain != c || v.dead.Load() {
|
||||
if v != nil {
|
||||
v.stop()
|
||||
}
|
||||
v = newRolloutView(c)
|
||||
i.moved.view = v
|
||||
i.state.view = v
|
||||
v.start()
|
||||
}
|
||||
i.moved.mu.Unlock()
|
||||
i.state.mu.Unlock()
|
||||
select {
|
||||
case <-v.ready:
|
||||
case <-ctx.Done():
|
||||
@@ -165,11 +165,11 @@ func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutVi
|
||||
|
||||
// stopViews ends the view's watch: the inventory is closing.
|
||||
func (i *Inventory) stopViews() {
|
||||
i.moved.mu.Lock()
|
||||
defer i.moved.mu.Unlock()
|
||||
if i.moved.view != nil {
|
||||
i.moved.view.stop()
|
||||
i.moved.view = nil
|
||||
i.state.mu.Lock()
|
||||
defer i.state.mu.Unlock()
|
||||
if i.state.view != nil {
|
||||
i.state.view.stop()
|
||||
i.state.view = nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -492,12 +492,12 @@ func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m Batched
|
||||
return false, err
|
||||
}
|
||||
if !found {
|
||||
i.moved.mu.Lock()
|
||||
if i.moved.pending == nil {
|
||||
i.moved.pending = map[string]pendingMerge{}
|
||||
i.state.mu.Lock()
|
||||
if i.state.pending == nil {
|
||||
i.state.pending = map[string]pendingMerge{}
|
||||
}
|
||||
i.moved.pending[key] = pendingMerge{plan: m.Plan, m: held}
|
||||
i.moved.mu.Unlock()
|
||||
i.state.pending[key] = pendingMerge{plan: m.Plan, m: held}
|
||||
i.state.mu.Unlock()
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -712,12 +712,12 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
|
||||
held, has := copies[key]
|
||||
if !has {
|
||||
// A merge carried for a plan not saved yet goes where it is moved.
|
||||
i.moved.mu.Lock()
|
||||
if pm, pending := i.moved.pending[key]; pending && m.Plan != "" {
|
||||
i.state.mu.Lock()
|
||||
if pm, pending := i.state.pending[key]; pending && m.Plan != "" {
|
||||
pm.plan, pm.m.Alone = m.Plan, m.Alone
|
||||
i.moved.pending[key] = pm
|
||||
i.state.pending[key] = pm
|
||||
}
|
||||
i.moved.mu.Unlock()
|
||||
i.state.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
w := winner(held)
|
||||
@@ -803,14 +803,14 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
|
||||
if e := edits[p.ID]; e != nil {
|
||||
held = e.apply(held)
|
||||
}
|
||||
i.moved.mu.Lock()
|
||||
for key, pm := range i.moved.pending {
|
||||
i.state.mu.Lock()
|
||||
for key, pm := range i.state.pending {
|
||||
if pm.plan == p.ID {
|
||||
held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held)
|
||||
_ = key
|
||||
}
|
||||
}
|
||||
i.moved.mu.Unlock()
|
||||
i.state.mu.Unlock()
|
||||
record.Held = held
|
||||
entered, duration := tierLeftOf(old, *p, time.Now())
|
||||
record.TierEntered, record.Updated = entered, time.Now().UTC()
|
||||
@@ -859,13 +859,13 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
|
||||
}
|
||||
}
|
||||
|
||||
i.moved.mu.Lock()
|
||||
for key, pm := range i.moved.pending {
|
||||
i.state.mu.Lock()
|
||||
for key, pm := range i.state.pending {
|
||||
if _, saved := done[pm.plan]; saved {
|
||||
delete(i.moved.pending, key)
|
||||
delete(i.state.pending, key)
|
||||
}
|
||||
}
|
||||
i.moved.mu.Unlock()
|
||||
i.state.mu.Unlock()
|
||||
for _, p := range plans {
|
||||
w := done[p.ID]
|
||||
p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held
|
||||
@@ -908,8 +908,8 @@ func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uin
|
||||
// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired
|
||||
// at the next cut (ADR 0408, issue 1092). It says how many it repaired.
|
||||
func (i *Inventory) RepairMerges(ctx context.Context) (int, error) {
|
||||
c, on, err := i.ChainReads(ctx, KindRollouts)
|
||||
if err != nil || !on {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) {
|
||||
t.Helper()
|
||||
c := chain.InMemory(nil)
|
||||
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv.UseChain(c)
|
||||
t.Cleanup(inv.stopViews)
|
||||
return inv, c
|
||||
@@ -117,10 +117,10 @@ func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) {
|
||||
}
|
||||
return errors.New("the lease is not held")
|
||||
})
|
||||
other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
|
||||
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
|
||||
other.UseChain(away)
|
||||
t.Cleanup(other.stopViews)
|
||||
other.moved.pending = inv.moved.pending
|
||||
other.state.pending = inv.state.pending
|
||||
p := aBatch("plan-new", plansAt)
|
||||
if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) {
|
||||
t.Fatalf("the save answered %v", err)
|
||||
@@ -158,7 +158,7 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test
|
||||
var fail atomic.Bool
|
||||
rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail}
|
||||
c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })
|
||||
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv.UseChain(c)
|
||||
t.Cleanup(inv.stopViews)
|
||||
ctx := context.Background()
|
||||
@@ -190,9 +190,6 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test
|
||||
if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 {
|
||||
t.Fatalf("the merge is held by %d records, want the two of a stopped move", n)
|
||||
}
|
||||
inv.moved.mu.Lock()
|
||||
inv.moved.kind = map[string]bool{KindRollouts: true}
|
||||
inv.moved.mu.Unlock()
|
||||
repaired := 0
|
||||
for k, copies := range mergeHolders(v.list()) {
|
||||
if len(copies) < 2 {
|
||||
@@ -310,7 +307,7 @@ func (c *countingBackend) Keys(ctx context.Context) ([]string, error) {
|
||||
func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
|
||||
counting := &countingBackend{Backend: chain.NewMemory()}
|
||||
c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil })
|
||||
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv.UseChain(c)
|
||||
t.Cleanup(inv.stopViews)
|
||||
ctx := context.Background()
|
||||
@@ -337,7 +334,7 @@ func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
|
||||
t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load())
|
||||
}
|
||||
// Another process writes; the view follows.
|
||||
other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
|
||||
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
|
||||
other.UseChain(c)
|
||||
t.Cleanup(other.stopViews)
|
||||
late := aBatch("plan-4", plansAt.Add(time.Hour))
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **What delivery 1 recorded in node.sent_builds is copied onto the keys of the assignments, once** (issue 1109, review
|
||||
// round 1): a module pulled in by an assignment had no commit on its key, so it would fall out of SentBuilds. The copy
|
||||
// creates a standing pulled-in key where none stands, fills a standing key that names no commit (commit, and the build
|
||||
// when one is known, no rollout), and overwrites nothing: a key with a commit, a hand send's unknown, a wait for a build.
|
||||
func TestWhatTheColumnRecordedIsCopiedOntoTheKeysAndOverwritesNothing(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := t.Context()
|
||||
at := time.Date(2026, 10, 18, 4, 0, 0, 0, time.UTC)
|
||||
if err := inv.chainRecordBuild(ctx, c, Build{ID: "b-wg", Module: "mesh-wireguard", Commit: "c-wg"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := func(module string, a chain.Assignment) {
|
||||
a.Node, a.Module = "laptop", module
|
||||
if _, err := c.Assignments.Create(ctx, assignmentKey("laptop", module), a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put("notes", chain.Assignment{Assigned: at}) // assigned, no commit yet
|
||||
put("web", chain.Assignment{Assigned: at, Commit: "c-web-new", Build: "b1"}) // already has one
|
||||
put("hand", chain.Assignment{Assigned: at, SentUnknown: at}) // a hand send since
|
||||
put("waits", chain.Assignment{Assigned: at, State: chain.StatePending}) // a wait for a build
|
||||
put("ended", chain.Assignment{Assigned: at, Ended: at}) // unassigned
|
||||
sent := map[string]string{"notes": "c-notes", "web": "c-web-old", "mesh-wireguard": "c-wg", "hand": "c-hand",
|
||||
"waits": "c-waits", "ended": "c-ended"}
|
||||
|
||||
for pass := 0; pass < 2; pass++ { // and the second pass changes nothing
|
||||
if err := inv.copySentBuildsOf(ctx, c, "laptop", sent, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
get := func(module string) chain.Assignment {
|
||||
t.Helper()
|
||||
k, found, err := c.Assignments.Get(ctx, assignmentKey("laptop", module))
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s: %v %v", module, found, err)
|
||||
}
|
||||
return k.Value
|
||||
}
|
||||
if a := get("notes"); a.Commit != "c-notes" || a.Rollout != "" || !a.Stands() || a.PulledIn {
|
||||
t.Errorf("notes: %+v", a)
|
||||
}
|
||||
if a := get("web"); a.Commit != "c-web-new" || a.Build != "b1" {
|
||||
t.Errorf("a key with a commit was overwritten: %+v", a)
|
||||
}
|
||||
if a := get("hand"); a.Commit != "" || a.SentUnknown.IsZero() {
|
||||
t.Errorf("a hand send's unknown was overwritten: %+v", a)
|
||||
}
|
||||
if a := get("waits"); a.Commit != "" || a.State != chain.StatePending {
|
||||
t.Errorf("a wait was overwritten: %+v", a)
|
||||
}
|
||||
if a := get("mesh-wireguard"); a.Commit != "c-wg" || a.Build != "b-wg" || !a.PulledIn || !a.Stands() || a.Rollout != "" {
|
||||
t.Errorf("the pulled-in module has no standing key of its own: %+v", a)
|
||||
}
|
||||
if a := get("ended"); a.Commit != "" || a.Stands() {
|
||||
t.Errorf("an ended key was revived or given a commit: %+v", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The copy reads the column of the store, writes the keys, and empties the column it has copied: SentBuilds then answers the
|
||||
// pulled-in module, and a second copy finds nothing to do.
|
||||
func TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.ExecForTest(t, `update node set sent_builds = '{"mesh-wireguard": "c-wg"}'::jsonb, sent_at = now() where name = 'laptop'`)
|
||||
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 1 {
|
||||
t.Fatalf("the copy did %d nodes (%v)", n, err)
|
||||
}
|
||||
got, known, err := inv.SentBuilds(ctx, "laptop")
|
||||
if err != nil || !known || got["mesh-wireguard"] != "c-wg" {
|
||||
t.Fatalf("SentBuilds answers %v, known %v (%v)", got, known, err)
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" {
|
||||
t.Fatalf("the column was not emptied: %v", rows)
|
||||
}
|
||||
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 {
|
||||
t.Fatalf("a second copy did %d nodes (%v)", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine delivery 2 has sent has its column emptied by the send itself, so a copy after it leaves a key the send cleared
|
||||
// without a commit.
|
||||
func TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.AddNode(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.ExecForTest(t, `update node set sent_builds = '{"notes": "c-old"}'::jsonb where name = 'laptop'`)
|
||||
if err := inv.RecordSentUnder(ctx, record.ID, "d1", map[string]string{}, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" {
|
||||
t.Fatalf("a send left the column: %v", rows)
|
||||
}
|
||||
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 {
|
||||
t.Fatalf("the copy did %d nodes (%v)", n, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **What a machine was sent is read from the keys of its assignments that name a commit** (ADR 0408, issue 1109): an assigned
|
||||
// module's, and a module only pulled in, whose key is a standing value marked pulled-in and never a state.
|
||||
func TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web", "cache")
|
||||
ctx := t.Context()
|
||||
record, _ := inv.NodeByName(ctx, node)
|
||||
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, node); err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"web": "c1", "cache": "c2"}) {
|
||||
t.Fatalf("sent %v %v %v", builds, known, err)
|
||||
}
|
||||
pulled, found, _ := inv.AssignmentValue(ctx, node, "cache")
|
||||
if !found || !pulled.Stands() || pulled.State != "" || !pulled.PulledIn || !pulled.Assigned.IsZero() || pulled.Commit != "c2" {
|
||||
t.Fatalf("the module only carried has the key %+v", pulled)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"web"}) {
|
||||
t.Fatalf("a module only pulled in is assigned: %v", on)
|
||||
}
|
||||
// Assigned later, it keeps what it was sent and is pulled in no longer.
|
||||
if _, err := inv.Assign(ctx, node, "cache"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
made, _, _ := inv.AssignmentValue(ctx, node, "cache")
|
||||
if made.PulledIn || made.Assigned.IsZero() || made.Commit != "c2" {
|
||||
t.Fatalf("assigned over a pulled-in key: %+v", made)
|
||||
}
|
||||
}
|
||||
|
||||
// **A send replaces the whole map of the node, and a hand send makes it unknown until the next send that names its builds**
|
||||
// (issue 1109): a module no longer carried loses its commit, the node reads unknown when any key says so.
|
||||
func TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web", "cache")
|
||||
ctx := t.Context()
|
||||
record, _ := inv.NodeByName(ctx, node)
|
||||
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "d2", map[string]string{"web": "c3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c3"}) {
|
||||
t.Fatalf("the second send did not replace the map: %v %v", builds, known)
|
||||
}
|
||||
if old, _, _ := inv.AssignmentValue(ctx, node, "cache"); old.Commit != "" || old.Build != "" {
|
||||
t.Fatalf("a module no longer carried keeps %+v", old)
|
||||
}
|
||||
// By hand: unknown, with a time and no commit anywhere.
|
||||
if err := inv.RecordSent(ctx, record.ID, "d3", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, _ := inv.SentBuilds(ctx, node); known || builds != nil {
|
||||
t.Fatalf("a hand send reads as %v %v", builds, known)
|
||||
}
|
||||
web, _, _ := inv.AssignmentValue(ctx, node, "web")
|
||||
if web.SentUnknown.IsZero() || web.Commit != "" || !web.Stands() {
|
||||
t.Fatalf("web after a hand send: %+v", web)
|
||||
}
|
||||
if sent, _ := inv.Outstanding(ctx, node); sent != "d3" {
|
||||
t.Fatalf("the digest of the hand send: %q", sent)
|
||||
}
|
||||
// A send that names its builds makes it known again.
|
||||
if err := inv.RecordSent(ctx, record.ID, "d4", map[string]string{"web": "c4"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c4"}) {
|
||||
t.Fatalf("after a send that names its builds: %v %v", builds, known)
|
||||
}
|
||||
if again, _, _ := inv.AssignmentValue(ctx, node, "web"); !again.SentUnknown.IsZero() {
|
||||
t.Fatalf("web still says unknown: %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
// failingToWrite is an assignments bucket that fails every write.
|
||||
type failingToWrite struct{ chain.Backend }
|
||||
|
||||
func (f failingToWrite) Create(context.Context, string, []byte) (uint64, error) {
|
||||
return 0, errors.New("the bus went away")
|
||||
}
|
||||
|
||||
func (f failingToWrite) Update(context.Context, string, []byte, uint64) (uint64, error) {
|
||||
return 0, errors.New("the bus went away")
|
||||
}
|
||||
|
||||
// **A links write that fails fails the record of the send, loudly** (issue 1109): what a machine was sent is read from the
|
||||
// keys, so a send recorded without them would read as one that never happened.
|
||||
func TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web")
|
||||
ctx := t.Context()
|
||||
record, _ := inv.NodeByName(ctx, node)
|
||||
inv.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), failingToWrite{chain.NewMemory()},
|
||||
func(context.Context) error { return nil }))
|
||||
err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1"})
|
||||
if err == nil {
|
||||
t.Fatal("a send whose links could not be written was recorded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The tables of the chain, and the column, that the controller's store no longer holds (novox/hq ADR 0408 §8, issue 1096).
|
||||
var chainTables = []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment"}
|
||||
|
||||
// sqlStatement is what makes a literal a statement of the store: it opens with a verb of SQL.
|
||||
var sqlStatement = regexp.MustCompile(`(?is)^\s*(select|insert|update|delete|alter|drop|create|with|truncate|from|join|where|and|set|values)\b|\b(select\s.+\sfrom|insert\s+into|delete\s+from)\b`)
|
||||
|
||||
// **No code names a table of the chain** (acceptance 12, issue 1096), found by reading the sources: a string literal of any
|
||||
// non-test file of internal/ or cmd/ that names one after from, into, update, join, delete from, references or table —
|
||||
// build_gate and port_assignment are other words —. This is the static half of the proof; the
|
||||
// integration test reads the catalog.
|
||||
func TestNoCodeNamesATableOfTheChain(t *testing.T) {
|
||||
names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b`)
|
||||
var found []string
|
||||
for _, root := range []string{"../../internal", "../../cmd"} {
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && d.Name() == "vendor" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ast.Inspect(file, func(n ast.Node) bool {
|
||||
lit, ok := n.(*ast.BasicLit)
|
||||
if !ok || lit.Kind != token.STRING {
|
||||
return true
|
||||
}
|
||||
text, err := strconv.Unquote(lit.Value)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
if !sqlStatement.MatchString(text) {
|
||||
return true // prose that happens to say "from assignment" is no statement
|
||||
}
|
||||
if m := names.FindString(text); m != "" {
|
||||
found = append(found, filepath.ToSlash(path)+": "+strings.Join(strings.Fields(m), " "))
|
||||
}
|
||||
return true
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(found) > 0 {
|
||||
t.Fatalf("code still names a table of the chain, which the store no longer holds:\n %s", strings.Join(found, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// The scan sees what it says it sees: planted, a string naming a table is found, and a comment or a word beside is not.
|
||||
func TestTheSourceScanFindsATablePlanted(t *testing.T) {
|
||||
names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b`)
|
||||
for text, want := range map[string]bool{
|
||||
"select id from build where id = $1": true,
|
||||
"insert into assignment (node, module)": true,
|
||||
"select 1 from build_gate": false,
|
||||
"delete from port_assignment": false,
|
||||
"select a from build_request a join build b": true,
|
||||
"an assignment from build to build": true,
|
||||
"the assignment generation": false,
|
||||
"select count(*) from pending_assignment": true,
|
||||
} {
|
||||
if got := names.MatchString(text); got != want {
|
||||
t.Errorf("%q: %v, want %v", text, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Migration 105 drops the tables of the chain, behind its guard** (issue 1096; it was the last until issue 1145 added 106): it names each table, the column and
|
||||
// the function, drops no build_gate, and raises before it drops anything.
|
||||
func TestMigration105DropsTheTablesOfTheChain(t *testing.T) {
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var last struct{ Number int }
|
||||
for _, m := range migrations {
|
||||
if m.Number == 105 {
|
||||
last.Number = m.Number
|
||||
}
|
||||
}
|
||||
if last.Number != 105 {
|
||||
t.Fatal("there is no migration 105")
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("migrations", migrationFile(t, last.Number)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var code []string
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if !strings.HasPrefix(strings.TrimSpace(line), "--") {
|
||||
code = append(code, line)
|
||||
}
|
||||
}
|
||||
sql := strings.ToLower(strings.Join(code, "\n"))
|
||||
for _, table := range append(append([]string{}, chainTables...), "chain_moved") {
|
||||
if !strings.Contains(sql, "drop table "+table+";") {
|
||||
t.Errorf("migration %d does not drop %s", last.Number, table)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"drop function raise_assignment_generation()"} {
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Errorf("migration %d lacks %q", last.Number, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "sent_builds") {
|
||||
t.Errorf("migration %d touches node.sent_builds, which stays as the source of the copy (issue 1112 drops it)", last.Number)
|
||||
}
|
||||
if strings.Contains(sql, "build_gate") || strings.Contains(sql, "assignment_generation;") {
|
||||
t.Errorf("migration %d touches a table that stays", last.Number)
|
||||
}
|
||||
if raise, drop := strings.Index(sql, "raise exception"), strings.Index(sql, "drop table"); raise < 0 || drop < 0 || raise > drop {
|
||||
t.Errorf("migration %d does not raise before it drops (%d, %d)", last.Number, raise, drop)
|
||||
}
|
||||
for _, kind := range []string{"'builds'", "'rollouts'", "'assignments'", "pending_assignment", "chain_moved"} {
|
||||
if !strings.Contains(sql[:strings.Index(sql, "drop table")], kind) {
|
||||
t.Errorf("the guard of migration %d does not look at %s", last.Number, kind)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func migrationFile(t *testing.T, number int) string {
|
||||
t.Helper()
|
||||
files, err := filepath.Glob(filepath.Join("migrations", strconv.Itoa(number/1000%10)+strconv.Itoa(number/100%10)+strconv.Itoa(number/10%10)+strconv.Itoa(number%10)+"-*.sql"))
|
||||
if err != nil || len(files) != 1 {
|
||||
t.Fatalf("the file of migration %d: %v %v", number, files, err)
|
||||
}
|
||||
return filepath.Base(files[0])
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its
|
||||
// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none.
|
||||
func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
first := aBuild("build-1", "notes", "")
|
||||
first.Commit = "c1"
|
||||
first.Against = []string{"artifact-store://base@sha256:aa"}
|
||||
first.ReuseKey = "rk-1"
|
||||
second := aBuild("build-2", "notes", "")
|
||||
second.Commit = "c2"
|
||||
failed := aBuild("build-3", "notes", "compile error")
|
||||
other := aBuild("build-4", "wiki", "")
|
||||
for _, b := range []Build{first, second, failed, other, first} {
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
kept, found, err := inv.chain.Builds.Get(ctx, chain.BuildKey("notes", "build-1"))
|
||||
if err != nil || !found || kept.Value.ReuseKey != "rk-1" {
|
||||
t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err)
|
||||
}
|
||||
if history, _ := inv.chain.Builds.History(ctx, chain.BuildKey("notes", "build-1")); len(history) != 1 {
|
||||
t.Fatalf("a build recorded twice has %d values", len(history))
|
||||
}
|
||||
list, err := inv.Builds(ctx, "notes", 10)
|
||||
if err != nil || len(list) != 3 {
|
||||
t.Fatalf("notes has %d builds: %v", len(list), err)
|
||||
}
|
||||
one, found, err := inv.BuildByID(ctx, "build-2")
|
||||
if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
|
||||
t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
|
||||
}
|
||||
if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
|
||||
t.Fatal("a build nobody recorded was found")
|
||||
}
|
||||
if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" {
|
||||
t.Fatalf("the build of notes at c1 is %q: %v", id, err)
|
||||
}
|
||||
held, err := inv.Held(ctx)
|
||||
if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" {
|
||||
t.Fatalf("held %v: %v", held, err)
|
||||
}
|
||||
against, err := inv.BuiltAgainst(ctx)
|
||||
if err != nil || len(against["notes"]) != 0 {
|
||||
// The newest successful build of notes, build-2, stood on nothing.
|
||||
t.Fatalf("built against %v: %v", against, err)
|
||||
}
|
||||
announced, err := inv.Announceable(ctx)
|
||||
if err != nil || len(announced) != 3 {
|
||||
t.Fatalf("announceable %d: %v", len(announced), err)
|
||||
}
|
||||
prints, err := inv.Fingerprints(ctx)
|
||||
if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
|
||||
t.Fatalf("fingerprints %v: %v", prints, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build request row as the store held it before the move.
|
||||
|
||||
// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue
|
||||
// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none
|
||||
// after the outcome is heard. The table of requests takes none.
|
||||
func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true}
|
||||
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
|
||||
t.Fatalf("a request kept twice: %v", err)
|
||||
}
|
||||
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 {
|
||||
t.Fatalf("a request nobody answered is a build: %+v %v", builds, err)
|
||||
}
|
||||
if _, found, _ := inv.BuildByID(ctx, "build-1"); found {
|
||||
t.Fatal("a request nobody answered has an outcome")
|
||||
}
|
||||
got, found, err := inv.BuildRequestByID(ctx, "build-1")
|
||||
if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() {
|
||||
t.Fatalf("the request reads back as %+v %v %v", got, found, err)
|
||||
}
|
||||
if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, err := inv.RequestsNamed(ctx, "notes")
|
||||
if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" {
|
||||
t.Fatalf("the request is %+v: %v", asked, err)
|
||||
}
|
||||
if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) {
|
||||
t.Fatalf("requested names %v: %v", names, err)
|
||||
}
|
||||
|
||||
// The outcome comes: the same key, now a build, with the request beside it.
|
||||
built := aBuild("build-1", "notes", "")
|
||||
if err := inv.RecordBuild(ctx, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" {
|
||||
t.Fatalf("the outcome reads as %+v %v %v", one, found, err)
|
||||
}
|
||||
answered, err := inv.RequestsNamed(ctx, "notes")
|
||||
if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" {
|
||||
t.Fatalf("the request answered is %+v: %v", answered, err)
|
||||
}
|
||||
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" {
|
||||
t.Fatalf("the request was lost with the outcome: %+v", again)
|
||||
}
|
||||
// Heard: the marks of an unanswered request change nothing.
|
||||
if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" {
|
||||
t.Fatalf("an answered request was marked not asked: %+v", again)
|
||||
}
|
||||
if history, _ := inv.chain.Builds.History(ctx, chain.RequestKey("build-1")); len(history) != 3 {
|
||||
// asked, marked unknown, answered.
|
||||
t.Fatalf("the record has %d values", len(history))
|
||||
}
|
||||
// An outcome heard before its request is kept, and the request lands beside it.
|
||||
if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" {
|
||||
t.Fatalf("the request kept after its outcome reads %+v", late)
|
||||
}
|
||||
// A request older than the time they are kept is no longer listed.
|
||||
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old",
|
||||
At: time.Now().Add(-2 * KeptFor)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") {
|
||||
t.Fatalf("a request past its time is listed: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a
|
||||
// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it
|
||||
// is asked and so asked at most once.
|
||||
|
||||
// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
|
||||
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
|
||||
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
|
||||
// assignment goes with it.
|
||||
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres", "web")
|
||||
ctx := t.Context()
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := generationNow(t, inv)
|
||||
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
|
||||
t.Fatalf("assigned %v: %v", added, err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added {
|
||||
t.Fatalf("assigned again %v: %v", added, err)
|
||||
}
|
||||
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
|
||||
if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
|
||||
t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
|
||||
}
|
||||
if generationNow(t, inv) <= start {
|
||||
t.Fatal("an assignment left the generation where it was")
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
|
||||
t.Fatalf("a module the mesh does not know: %v", err)
|
||||
}
|
||||
if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) {
|
||||
t.Fatalf("assigned on %s: %v %v", node, on, err)
|
||||
}
|
||||
if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) {
|
||||
t.Fatalf("running: %v %v", running, err)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
switch e.Manifest.Module {
|
||||
case "postgres":
|
||||
if !reflect.DeepEqual(e.On, []string{node}) {
|
||||
t.Errorf("postgres is on %v", e.On)
|
||||
}
|
||||
case "web":
|
||||
if len(e.On) != 0 {
|
||||
t.Errorf("web is on %v", e.On)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) {
|
||||
t.Fatalf("a module still assigned was forgotten: %v", err)
|
||||
}
|
||||
// A seat is held by an assignment, or by nothing.
|
||||
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil {
|
||||
t.Fatal("a seat was handed to a module that is not assigned")
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil {
|
||||
t.Fatalf("a seat handed to an assignment: %v", err)
|
||||
}
|
||||
|
||||
before := generationNow(t, inv)
|
||||
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if generationNow(t, inv) <= before {
|
||||
t.Fatal("an unassignment left the generation where it was")
|
||||
}
|
||||
// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
|
||||
gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
|
||||
if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
|
||||
t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err)
|
||||
}
|
||||
if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() ||
|
||||
history[1].Value.Stands() {
|
||||
t.Fatalf("the earlier values of the key: %+v", history)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
|
||||
t.Fatalf("an ended assignment is still assigned: %v", on)
|
||||
}
|
||||
if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 {
|
||||
t.Fatalf("an ended assignment is running: %v", running)
|
||||
}
|
||||
if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 {
|
||||
t.Fatalf("the seat the assignment held is held by %+v: %v", held, err)
|
||||
}
|
||||
if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" {
|
||||
t.Fatalf("an unassignment of nothing: %v", err)
|
||||
}
|
||||
if err := inv.ForgetModule(ctx, "postgres"); err != nil {
|
||||
t.Fatalf("a module nothing is assigned could not be forgotten: %v", err)
|
||||
}
|
||||
// Made again after it ended: a fresh value over the ended one, which stays in history.
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
|
||||
t.Fatal(added, err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || added {
|
||||
t.Fatal(added, err)
|
||||
}
|
||||
if err := inv.Unassign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
|
||||
t.Fatalf("assigned again after it ended: %v %v", added, err)
|
||||
}
|
||||
again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
|
||||
if !again.Value.Stands() || !again.Value.Ended.IsZero() {
|
||||
t.Fatalf("the assignment made again is %+v", again.Value)
|
||||
}
|
||||
if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() {
|
||||
t.Fatalf("the values of the key: %+v", history)
|
||||
}
|
||||
}
|
||||
|
||||
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
|
||||
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
|
||||
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
|
||||
// ends an assignment.
|
||||
|
||||
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
|
||||
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
|
||||
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
|
||||
// ends an assignment.
|
||||
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "dunst", "web")
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"dunst", "web"} {
|
||||
if _, err := inv.Assign(ctx, node, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, _ := inv.NodeByName(ctx, node)
|
||||
digest := "sha256:" + strings.Repeat("d", 64)
|
||||
plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
|
||||
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
|
||||
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
|
||||
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A build and a commit put in the value by another write, after the send: a send that carries no builds (a hand send, as this
|
||||
// one) forgets what each assignment of its machine runs (issue 1109), so they are written once it is recorded.
|
||||
cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
|
||||
cur.Value.Build, cur.Value.Commit = "build-1", "c1"
|
||||
if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
written, removed, err := inv.KeepAssignmentPlans(ctx)
|
||||
if err != nil || written != 2 || removed != 0 {
|
||||
t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
|
||||
}
|
||||
dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
|
||||
if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
|
||||
t.Fatalf("dunst's value is %+v", dunst.Value)
|
||||
}
|
||||
if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
|
||||
t.Fatalf("web has a plan: %+v", web.Value)
|
||||
}
|
||||
notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify")
|
||||
if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil {
|
||||
t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
|
||||
t.Fatalf("a pulled-in module is assigned: %v", on)
|
||||
}
|
||||
if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 {
|
||||
t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err)
|
||||
}
|
||||
// Sent again without the pulled-in module: its value ends, the assignments stay.
|
||||
only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
|
||||
t.Fatalf("ended %d: %v", removed, err)
|
||||
}
|
||||
if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() {
|
||||
t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
|
||||
t.Fatalf("assigned: %v", on)
|
||||
}
|
||||
}
|
||||
|
||||
// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
|
||||
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
|
||||
// already is left as it is.
|
||||
@@ -2,7 +2,6 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -327,84 +326,34 @@ type recentBuild struct {
|
||||
|
||||
// recentBuilds is, of each module the mesh still holds, its KeptBuilds newest successful builds' artifacts and bases.
|
||||
func (i *Inventory) recentBuilds(ctx context.Context) ([]recentBuild, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
builds, err := i.chainRecentMade(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]recentBuild, 0, len(builds))
|
||||
for _, b := range builds {
|
||||
out = append(out, recentBuild{made: b.Made, against: b.Against})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select made, built_against from (
|
||||
select b.made, b.built_against,
|
||||
row_number() over (partition by b.module order by b.at desc, b.id desc) as back
|
||||
from build b
|
||||
join module m on m.name = b.module
|
||||
where b.failed = '' and b.module is not null and b.module <> ''
|
||||
) ranked where back <= $1`, KeptBuilds)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []recentBuild
|
||||
for rows.Next() {
|
||||
var raw, against []byte
|
||||
if err := rows.Scan(&raw, &against); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r recentBuild
|
||||
if err := json.Unmarshal(raw, &r.made); err != nil {
|
||||
r.made = nil
|
||||
}
|
||||
if err := json.Unmarshal(against, &r.against); err != nil {
|
||||
r.against = nil
|
||||
}
|
||||
out = append(out, r)
|
||||
builds, err := i.chainRecentMade(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, rows.Err()
|
||||
out := make([]recentBuild, 0, len(builds))
|
||||
for _, b := range builds {
|
||||
out = append(out, recentBuild{made: b.Made, against: b.Against})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// everyReferenceMade is every artifact reference any successful build recorded, oldest build first.
|
||||
func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
|
||||
var made [][]Artifact
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
builds, err := i.chainEveryReferenceMade(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, b := range builds {
|
||||
made = append(made, b.Made)
|
||||
}
|
||||
} else {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select made from build where failed = '' and module is not null and module <> ''
|
||||
order by at asc, id asc`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var raw []byte
|
||||
if err := rows.Scan(&raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a []Artifact
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
continue
|
||||
}
|
||||
made = append(made, a)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
builds, err := i.chainEveryReferenceMade(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var made [][]Artifact
|
||||
for _, b := range builds {
|
||||
made = append(made, b.Made)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
|
||||
@@ -2,11 +2,7 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
|
||||
@@ -148,41 +144,3 @@ func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or
|
||||
// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier.
|
||||
func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) {
|
||||
var oldTier int
|
||||
var oldState string
|
||||
var since time.Time
|
||||
err = tx.QueryRow(ctx,
|
||||
`select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`,
|
||||
p.ID).Scan(&oldTier, &oldState, &since)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return now, nil // a new plan enters its first tier now
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if oldState == PlanAssembling || oldState == PlanQueued {
|
||||
return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276)
|
||||
}
|
||||
wasOpen := oldState == PlanBuilding || oldState == PlanRolling
|
||||
if !wasOpen || (oldTier == p.Tier && p.Open()) {
|
||||
return since, nil // still in the tier, or already ended
|
||||
}
|
||||
var modules []string
|
||||
if oldTier >= 0 && oldTier < len(p.Tiers) {
|
||||
modules = p.Tiers[oldTier]
|
||||
}
|
||||
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State)
|
||||
if p.Open() {
|
||||
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier)
|
||||
}
|
||||
_, err = tx.Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`,
|
||||
DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since,
|
||||
now.Sub(since).Milliseconds(), detail)
|
||||
return now, err
|
||||
}
|
||||
|
||||
@@ -65,16 +65,15 @@ func ForTest(t *testing.T) *Inventory {
|
||||
if _, err := inv.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The chain is on the bus in a mesh; in a test it is in memory, with every kind moved, as a mesh raised after this
|
||||
// release has them (ADR 0408, issue 1089). The tests of the move itself use ForTestBefore.
|
||||
// The chain is on the bus in a mesh; in a test it is in memory (ADR 0408).
|
||||
c := chain.InMemory(nil)
|
||||
// A fresh test chain has nothing to lay out: it is laid out already, as the controller's start leaves a mesh (issue 1121).
|
||||
if _, err := c.Builds.CreateRaw(t.Context(), chain.LayoutMarkerKey, []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.UseChain(c)
|
||||
testChains.Store(os.Getenv(store.Variable(Name)), c)
|
||||
t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) })
|
||||
if _, err := inv.store.Pool().Exec(t.Context(),
|
||||
`insert into chain_moved (counts, largest) values ($1, $1)`, `{"builds": 0, "rollouts": 0, "assignments": 0}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return inv
|
||||
}
|
||||
|
||||
@@ -132,8 +131,12 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) {
|
||||
if err := inv.Ready(t.Context(), 20*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The chain is in memory and nothing is moved: the tests of a move seed the tables and move them.
|
||||
// The chain is in memory, for what a test of the store before a migration reads of it.
|
||||
c := chain.InMemory(nil)
|
||||
// A fresh test chain has nothing to lay out: it is laid out already, as the controller's start leaves a mesh (issue 1121).
|
||||
if _, err := c.Builds.CreateRaw(t.Context(), chain.LayoutMarkerKey, []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.UseChain(c)
|
||||
testChains.Store(os.Getenv(store.Variable(Name)), c)
|
||||
t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) })
|
||||
@@ -183,3 +186,21 @@ func (i *Inventory) QueryForTest(t *testing.T, sql string, args ...any) [][]stri
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PlanValuesForTest runs the pass that keeps the provisioner's plans in the values of the assignments (KeepAssignmentPlans)
|
||||
// and answers the values that stand, by key: what a reader of the bucket reads.
|
||||
func (i *Inventory) PlanValuesForTest(t *testing.T) map[string][]byte {
|
||||
t.Helper()
|
||||
if _, _, err := i.KeepAssignmentPlans(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := i.KeptAssignments(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := make(map[string][]byte, len(kept))
|
||||
for key, v := range kept {
|
||||
out[key] = v.Value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
+12
-86
@@ -2,8 +2,6 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -241,65 +239,20 @@ func (i *Inventory) RestoreModule(ctx context.Context, b Build) error {
|
||||
// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told
|
||||
// from one that does (the bus's planned step, ADR 0236).
|
||||
func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainBuildFingerprints(ctx, c, module)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select commit_hash, made, coalesce(manifest::text, '') from build
|
||||
where module = $1 and failed = '' and commit_hash <> '' order by at desc limit 50`, module)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]string{}
|
||||
for rows.Next() {
|
||||
var commit, manifest string
|
||||
var made []byte
|
||||
if err := rows.Scan(&commit, &made, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, seen := out[commit]; seen {
|
||||
continue
|
||||
}
|
||||
sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...))
|
||||
out[commit] = hex.EncodeToString(sum[:])
|
||||
}
|
||||
return out, rows.Err()
|
||||
return i.chainBuildFingerprints(ctx, c, module)
|
||||
}
|
||||
|
||||
// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint.
|
||||
func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainFingerprints(ctx, c)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select module, commit_hash, made, coalesce(manifest::text, '') from build
|
||||
where module is not null and failed = '' and commit_hash <> '' order by at desc`)
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]map[string]string{}
|
||||
for rows.Next() {
|
||||
var module, commit, manifest string
|
||||
var made []byte
|
||||
if err := rows.Scan(&module, &commit, &made, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string]string{}
|
||||
}
|
||||
if _, seen := out[module][commit]; seen {
|
||||
continue
|
||||
}
|
||||
sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...))
|
||||
out[module][commit] = hex.EncodeToString(sum[:])
|
||||
}
|
||||
return out, rows.Err()
|
||||
return i.chainFingerprints(ctx, c)
|
||||
}
|
||||
|
||||
// PassedCommits is, per module, the commits a build of which passed its gate on some machine.
|
||||
@@ -326,19 +279,11 @@ func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bo
|
||||
// BuildOf is the id of the newest successful build of a module from a commit; empty when none is
|
||||
// recorded (a manifest handed over by hand).
|
||||
func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
} else if on {
|
||||
return i.chainBuildOf(ctx, c, module, commit)
|
||||
}
|
||||
var id string
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id from build where module = $1 and commit_hash = $2 and failed = '' order by at desc limit 1`,
|
||||
module, commit).Scan(&id)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
return id, err
|
||||
return i.chainBuildOf(ctx, c, module, commit)
|
||||
}
|
||||
|
||||
// ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the
|
||||
@@ -372,38 +317,19 @@ func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (cata
|
||||
|
||||
// manifestOf is the manifest a build was recorded with; nil when it kept none.
|
||||
func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return i.chainManifestOf(ctx, c, id)
|
||||
}
|
||||
var manifest []byte
|
||||
if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, id).Scan(&manifest); err != nil {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return manifest, nil
|
||||
return i.chainManifestOf(ctx, c, id)
|
||||
}
|
||||
|
||||
// newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest:
|
||||
// its id and manifest; false when there is none.
|
||||
func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) {
|
||||
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
|
||||
return "", nil, false, err
|
||||
} else if on {
|
||||
return i.chainManifestBuild(ctx, c, module, commit)
|
||||
}
|
||||
var id string
|
||||
var raw []byte
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, manifest from build
|
||||
where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null'
|
||||
and (commit_hash = $2 or starts_with(commit_hash, $2))
|
||||
order by at desc limit 1`, module, commit).Scan(&id, &raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil, false, nil
|
||||
}
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return "", nil, false, err
|
||||
}
|
||||
return id, raw, true, nil
|
||||
return i.chainManifestBuild(ctx, c, module, commit)
|
||||
}
|
||||
|
||||
@@ -93,28 +93,19 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s
|
||||
if declared == "" {
|
||||
return nil, nil
|
||||
}
|
||||
// Only a module assigned there: from the bucket when the assignments are on the bus, else the store's join.
|
||||
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
|
||||
// Only a module assigned there, from the assignments bucket.
|
||||
assignedHere := map[string]bool{}
|
||||
modules, err := i.assignedModules(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
assignedHere := map[string]bool{}
|
||||
join := `join assignment a on a.node = s.node and a.module = s.module`
|
||||
if onTheBus {
|
||||
join = ``
|
||||
modules, err := i.assignedModules(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, m := range modules {
|
||||
assignedHere[m] = true
|
||||
}
|
||||
for _, m := range modules {
|
||||
assignedHere[m] = true
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select s.node, s.module, s.name, s.replace_after_start
|
||||
from module_secret s
|
||||
join node n on n.id = s.node
|
||||
`+join+`
|
||||
where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start
|
||||
and s.origin = 'accepted' and s.replace_after_start is not null
|
||||
and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module))
|
||||
@@ -134,7 +125,7 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
if onTheBus && !assignedHere[d.module] {
|
||||
if !assignedHere[d.module] {
|
||||
continue
|
||||
}
|
||||
dues = append(dues, d)
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
-- The tables of builds, rollouts and assignments leave the store (novox/hq ADR 0408 §8, design 47 milestone 5, issue 1096).
|
||||
--
|
||||
-- Delivery 1 wrote their rows into the bus's buckets at the controller's start and recorded each kind it had counted in
|
||||
-- chain_moved; from then on nothing in the controller reads or writes the tables. This drops them. **It is one-way**: a
|
||||
-- controller older than this schema is refused by the put-back gate, and the rows are in the buckets only.
|
||||
--
|
||||
-- **The guard comes first, in the same transaction as the drops**, because a migration runs when the store is opened, before
|
||||
-- the controller of this delivery serves and before any connection to the bus: it can prove the move from the store alone.
|
||||
-- A kind that holds rows and is not named in chain_moved was never moved, so dropping its tables would lose it; a pending
|
||||
-- assignment still waiting, being made, or ended unmade with its condition unanswered has no mover (it is a value of the
|
||||
-- assignments bucket from this delivery on), so the table must hold none. A store that never held a row of a kind — a new
|
||||
-- mesh — passes. A migration that raises leaves nothing applied, `prepare` fails with the reason, and the running controller
|
||||
-- keeps its tables.
|
||||
do $$
|
||||
declare
|
||||
kind text;
|
||||
holds boolean;
|
||||
missing text[] := '{}';
|
||||
begin
|
||||
foreach kind in array array['builds', 'rollouts', 'assignments'] loop
|
||||
holds := case kind
|
||||
when 'builds' then exists (select 1 from build) or exists (select 1 from build_request)
|
||||
when 'rollouts' then exists (select 1 from release_plan) or exists (select 1 from batched_merge)
|
||||
when 'assignments' then exists (select 1 from assignment)
|
||||
end;
|
||||
if holds and not coalesce((select counts ? kind from chain_moved), false) then
|
||||
missing := missing || kind;
|
||||
end if;
|
||||
end loop;
|
||||
if array_length(missing, 1) > 0 then
|
||||
raise exception 'the controller''s store still holds rows of % that chain_moved does not name as moved to the bus; '
|
||||
'run the controller of the previous delivery and read chain_moved back first', array_to_string(missing, ', ');
|
||||
end if;
|
||||
if exists (select 1 from pending_assignment
|
||||
where state in ('waiting', 'applying') or (raised_at is not null and acknowledged_at is null and cleared_at is null)) then
|
||||
raise exception 'a pending assignment is still in the table (waiting, being made, or ended with its condition '
|
||||
'unanswered); let it settle, then deliver again';
|
||||
end if;
|
||||
end
|
||||
$$;
|
||||
|
||||
drop table build_request;
|
||||
drop table build;
|
||||
drop table batched_merge;
|
||||
drop table release_plan;
|
||||
drop table pending_assignment;
|
||||
-- Its triggers go with it; the function they called goes here. The counter table stays: the controller raises it.
|
||||
drop table assignment;
|
||||
drop function raise_assignment_generation();
|
||||
-- node.sent_builds STAYS, only as the source of the one-time copy onto the assignments' keys (issue 1109); dropping it is
|
||||
-- issue 1112. build_gate stays (issue 1097).
|
||||
drop table chain_moved;
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Whether the machine's node-engine said it reads a module on each resource of a declaration and the build of each
|
||||
-- module (its reports' `reads_modules`, novox/hq issue 1145): until it has, it is sent neither, because an older
|
||||
-- node-engine refuses a key it does not know, whole. As reads_generation (migration 0094).
|
||||
alter table node add column reads_modules boolean not null default false;
|
||||
@@ -5,7 +5,6 @@ package inventory
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -148,40 +147,6 @@ func TestAPersonRecordsACopyAndTheFirstRecordStands(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The migration records the copies the build records already name: only a module's own on-<argument>
|
||||
// repository, and only once, by the earliest build naming it.
|
||||
func TestTheCopiesTheRecordsAlreadyNameAreRecorded(t *testing.T) {
|
||||
inv := inTheTables(t)
|
||||
ctx := context.Background()
|
||||
copyRef := mirrorRef("route-proxy/on-go_base", 1)
|
||||
notACopy := ref("go-base", "build", 2)
|
||||
builtOn(t, inv, "b1", "route-proxy", 1, nil, copyRef, notACopy)
|
||||
builtOn(t, inv, "b2", "route-proxy", 2, nil, copyRef)
|
||||
// The rows a mesh had before the table: written by builds that did not say what they copied.
|
||||
if _, err := inv.store.Pool().Exec(ctx, `truncate artifact_mirrored`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql, err := os.ReadFile("migrations/0081-a-mirrored-base-is-recorded.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, backfill, _ := strings.Cut(string(sql), "insert into artifact_mirrored")
|
||||
if _, err := inv.store.Pool().Exec(ctx, "insert into artifact_mirrored"+backfill); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reference, build string
|
||||
var n int
|
||||
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from artifact_mirrored`).Scan(&n); err != nil || n != 1 {
|
||||
t.Fatalf("%d copies recorded (%v); want the one", n, err)
|
||||
}
|
||||
if err := inv.store.Pool().QueryRow(ctx, `select reference, build from artifact_mirrored`).Scan(&reference, &build); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reference != copyRef || build != "b1" {
|
||||
t.Fatalf("recorded %s by %s", reference, build)
|
||||
}
|
||||
}
|
||||
|
||||
// A copy let go of and copied again by a later build is held again: its record is back, and the sweep
|
||||
// decides about it as before.
|
||||
func TestACopyLetGoOfAndCopiedAgainIsHeldAgain(t *testing.T) {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
@@ -70,39 +69,8 @@ type DeclaredMirrored struct {
|
||||
var DeclaredMirroredFields = []string{"kind", "default", "why", "type", "description", "enum", "minimum", "maximum",
|
||||
"pattern", "trusted", "predefined", "required", "runtime"}
|
||||
|
||||
// ErrAssignmentsMoved is a write of the store's mirror of the assignments, refused because the assignments are keys of the bus.
|
||||
var ErrAssignmentsMoved = errors.New("the assignments are keys of the bus, and the store's rows are no longer what they say")
|
||||
|
||||
// AssignmentsMirrorToWrite is AssignmentsMirror for the mirror's writer, which lays what it returns over the bucket and
|
||||
// ends the keys it does not return: **refused once the assignments are moved** (ADR 0408 §4), asked at the moment of the
|
||||
// write, so a tick that decided before the switch cannot lay mirror-shaped values over the links after it.
|
||||
func (i *Inventory) AssignmentsMirrorToWrite(ctx context.Context) (map[string][]byte, error) {
|
||||
if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
|
||||
return nil, err
|
||||
} else if on {
|
||||
return nil, ErrAssignmentsMoved
|
||||
}
|
||||
return i.AssignmentsMirror(ctx)
|
||||
}
|
||||
|
||||
// AssignmentsMirror is every assignment's mirrored value, keyed `<node>.<module>`, with the provisioner's plan of
|
||||
// each assignment of a module handed to it, and a value of its own for each module handed to it on a node that
|
||||
// nothing assigned there, only pulled in by another (issue 765).
|
||||
func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, error) {
|
||||
var plans map[string]Provisioning
|
||||
if i.provisioningPlans != nil {
|
||||
var err error
|
||||
if plans, err = i.provisioningPlans(ctx); err != nil {
|
||||
return nil, fmt.Errorf("composing the provisioner's plans: %w", err)
|
||||
}
|
||||
}
|
||||
assignments, err := i.assignmentsMirrored(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return AssignmentValues(assignments, plans)
|
||||
}
|
||||
|
||||
// assignmentsMirrored is every standing assignment as the provisioner's plans are matched to it (SentProvisioningPlans,
|
||||
// KeepAssignmentPlans).
|
||||
func (i *Inventory) assignmentsMirrored(ctx context.Context) ([]AssignmentMirrored, error) {
|
||||
rows, err := i.assignments(ctx)
|
||||
if err != nil {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user