Archived
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8adb7f1a05 | ||
|
|
249d97d1c8 | ||
|
|
7d63d2e68c | ||
|
|
fe00fec52c | ||
|
|
b74268fb08 |
@@ -415,16 +415,22 @@ func recordAsked(ctx context.Context, r inventory.BuildRequest) {
|
||||
recordBuildRequest(ctx, open.inventory, r)
|
||||
}
|
||||
|
||||
// markNotAsked says a kept build request was not handed over, or not waited for.
|
||||
func markNotAsked(ctx context.Context, id string, why error) {
|
||||
// markWaitFailed says what became of a kept build request whose waited ask failed: never handed over, so
|
||||
// nothing runs; or handed over and no longer waited for, so asked with its outcome unknown — still read as in
|
||||
// flight until its outcome or its bound (novox/hq issue 325).
|
||||
func markWaitFailed(ctx context.Context, id string, why error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s could not be marked as not asked: %v\n", id, err)
|
||||
fmt.Fprintf(os.Stderr, "build %s could not be marked: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
defer open.Close()
|
||||
if err := open.inventory.MarkNotAsked(ctx, id, why.Error()); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s could not be marked as not asked: %v\n", id, err)
|
||||
mark := open.inventory.MarkOutcomeUnknown
|
||||
if errors.Is(why, link.ErrNotHandedOver) {
|
||||
mark = open.inventory.MarkNotAsked
|
||||
}
|
||||
if err := mark(ctx, id, why.Error()); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "build %s could not be marked: %v\n", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -518,15 +524,15 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// Waited for: kept while it is waited for, since that can take minutes, and marked when the hand-over or
|
||||
// the wait failed — an outcome heard later is still the last word.
|
||||
// Waited for: kept while it is waited for, since that can take minutes, and marked when the hand-over
|
||||
// failed (not asked) or the wait did (asked, outcome unknown) — an outcome heard later is the last word.
|
||||
if keep {
|
||||
recordAsked(ctx, asked)
|
||||
}
|
||||
result, err := ask.Submit(ctx, request, wait)
|
||||
if err != nil {
|
||||
if keep {
|
||||
markNotAsked(ctx, request.ID, err)
|
||||
markWaitFailed(ctx, request.ID, err)
|
||||
}
|
||||
return request.ID, err
|
||||
}
|
||||
|
||||
@@ -281,8 +281,11 @@ func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string {
|
||||
return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where,
|
||||
short(r.Commit), firstLine(r.NotAsked))
|
||||
case r.NotAsked != "":
|
||||
return fmt.Sprintf("%s; build %s, asked by %s, was not handed over or not waited for: %s", where, r.ID,
|
||||
return fmt.Sprintf("%s; build %s, asked by %s, was not handed over: %s", where, r.ID,
|
||||
askerOr(r.For), firstLine(r.NotAsked))
|
||||
case r.OutcomeUnknown != "":
|
||||
return fmt.Sprintf("%s; build %s was asked at %s, its asker stopped waiting (%s), and no outcome has "+
|
||||
"been heard in %s", where, r.ID, clock(r.At), firstLine(r.OutcomeUnknown), now.Sub(r.At).Round(time.Minute))
|
||||
default:
|
||||
return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID,
|
||||
clock(r.At), now.Sub(r.At).Round(time.Minute))
|
||||
@@ -545,7 +548,7 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
||||
switch {
|
||||
case r == nil && now.Sub(p.Since) > buildRequestBound:
|
||||
why = fmt.Sprintf("build %s is no longer on record", p.Build)
|
||||
case r != nil && !stillComing(*r, now) && (r.Heard || r.NotAsked != ""):
|
||||
case r != nil && !stillComing(*r, now) && (r.Heard || r.NotAsked != "" || r.OutcomeUnknown != ""):
|
||||
why = whyNotBuilt(*r, now)
|
||||
case r != nil && !stillComing(*r, now):
|
||||
why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound)
|
||||
@@ -576,9 +579,10 @@ func containsString(xs []string, x string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// pendingObservation is the condition for a pending assignment that was not made.
|
||||
// pendingObservation is the condition for a pending assignment that was not made: one per pending
|
||||
// assignment, its row's id the last part of its id, so one row's condition is never another's.
|
||||
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: p.Node + "." + p.Module,
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID),
|
||||
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
|
||||
Resolver: conditions.ResolverOperator, Source: "pending assignments",
|
||||
Summary: p.Note}
|
||||
@@ -628,7 +632,7 @@ func raisePendingEnded(ctx context.Context, inv *inventory.Inventory) []string {
|
||||
said = append(said, fmt.Sprintf("the condition for %s on %s was cleared and not recorded: %v", p.Module, p.Node, err))
|
||||
}
|
||||
}
|
||||
return said
|
||||
return append(said, clearOrphaned(ctx, keeper, inv)...)
|
||||
}
|
||||
|
||||
// pendingAnswered says whether a pending assignment that was not made has been answered since, and how.
|
||||
@@ -647,14 +651,60 @@ func pendingAnswered(ctx context.Context, inv *inventory.Inventory, p inventory.
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
// A newer pending assignment answers it only while it is open or once it was made: one that itself
|
||||
// ended unmade is its own condition, and answers nothing.
|
||||
for _, r := range rows {
|
||||
if r.ID != p.ID && r.Since.After(p.Since) {
|
||||
if r.ID != p.ID && r.Since.After(p.Since) && (r.Open() || r.State == inventory.PendingApplied) {
|
||||
return "assigned again, pending on another build", true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// clearOrphaned clears every open condition of an assignment not made whose pending assignment is no longer
|
||||
// on record: its machine was removed, which takes its pending assignments with it.
|
||||
func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventory.Inventory) []string {
|
||||
open, err := keeper.Open(ctx)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)}
|
||||
}
|
||||
byID := map[int64]string{}
|
||||
var ids []int64
|
||||
for _, c := range open {
|
||||
if c.Kind != kindPendingEnded {
|
||||
continue
|
||||
}
|
||||
// `<scope>.<node>.<module>.<row>.<kind>`: the row is the part before the kind.
|
||||
parts := strings.Split(c.Key, ".")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
var id int64
|
||||
if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil {
|
||||
continue
|
||||
}
|
||||
byID[id] = c.Key
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
known, err := inv.PendingKnown(ctx, ids)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("the pending assignments could not be read to clear orphaned conditions: %v", err)}
|
||||
}
|
||||
var said []string
|
||||
for id, key := range byID {
|
||||
if known[id] {
|
||||
continue
|
||||
}
|
||||
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil {
|
||||
said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err))
|
||||
}
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// withdrawPending is `unassign` of a module only pending on a machine, under the machine's hold: a waiting
|
||||
// pending assignment is withdrawn; one being made now is refused, never overridden; one that expired or was
|
||||
// refused is taken back, which answers its condition. False when there is none.
|
||||
@@ -663,6 +713,7 @@ func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
var takenBack []string
|
||||
for _, p := range rows {
|
||||
switch p.State {
|
||||
case inventory.PendingApplying:
|
||||
@@ -687,9 +738,12 @@ func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module
|
||||
if err := inv.MarkPending(ctx, p.ID, "acknowledged"); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return fmt.Sprintf("the pending assignment of %s to %s, which %s, is taken back; its condition clears",
|
||||
module, node, p.State), true, nil
|
||||
takenBack = append(takenBack, fmt.Sprintf("build %s (%s)", p.Build, p.State))
|
||||
}
|
||||
}
|
||||
if len(takenBack) > 0 {
|
||||
return fmt.Sprintf("the pending assignment(s) of %s to %s that were not made are taken back: %s; their "+
|
||||
"conditions clear", module, node, strings.Join(takenBack, ", ")), true, nil
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -15,6 +16,39 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// failedBuild settles the merge's build of modules/sensors as failed.
|
||||
func failedBuild(t *testing.T, open *stores, id string) {
|
||||
t.Helper()
|
||||
if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa",
|
||||
"", "boom")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool {
|
||||
t.Helper()
|
||||
_, found, err := conditionsFrom.Get(t.Context(), pendingObservation(p).Key())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
|
||||
t.Helper()
|
||||
rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.ID == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("pending assignment %d is not on record", id)
|
||||
return inventory.PendingAssignment{}
|
||||
}
|
||||
|
||||
// novox/hq issue 325: an assignment of a module the catalogue does not hold says which case it is in — a
|
||||
// build in flight (kept pending), known and not built (said, with build), or unknown (refused, with the
|
||||
// closest names) — and a pending assignment is made when its build registers the module, or ends with why.
|
||||
@@ -370,7 +404,7 @@ func TestAFailedAskIsNotABuildInFlight(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = assign(ctx, open, "laptop", "gauges")
|
||||
if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over or not waited for") {
|
||||
if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") {
|
||||
t.Fatalf("a build not handed over: %v", err)
|
||||
}
|
||||
// Heard first, the outcome stands: marking it afterwards changes nothing.
|
||||
@@ -602,7 +636,8 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
|
||||
t.Fatalf("a module registered meanwhile: %+v", p)
|
||||
}
|
||||
|
||||
// Ended rows are deleted after KeptFor (review point 7); open ones never.
|
||||
// Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is
|
||||
// still open (second review, bug B).
|
||||
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -612,10 +647,13 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if !r.Open() {
|
||||
t.Fatalf("an ended pending assignment outlived %s: %+v (%v)", inventory.KeptFor, r, said)
|
||||
if !r.Open() && (r.Raised == nil || r.Cleared != nil) {
|
||||
t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") {
|
||||
t.Fatalf("the applied row was not deleted: %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The seat's assign passes build on; unassign takes none.
|
||||
@@ -641,3 +679,186 @@ func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) {
|
||||
t.Fatalf("headline %q", w.Headline)
|
||||
}
|
||||
}
|
||||
|
||||
// **One row's condition is never another's** (second review, bug A). The sequence: the first pending
|
||||
// assignment's build fails and the tick raises it; the person assigns again with build "true", a second
|
||||
// pending assignment; its build fails too before the next tick. That tick raises the second, and must not
|
||||
// clear it by judging the first answered by the second: each row has its own condition, and a newer row that
|
||||
// itself ended unmade answers nothing.
|
||||
func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := 0
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
asked++
|
||||
return fmt.Sprintf("b-%d", asked), nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failedBuild(t, open, "b-1")
|
||||
settlePending(ctx, open, time.Now())
|
||||
first := pendingOf(t, open)[0]
|
||||
if !conditionOpen(t, first) {
|
||||
t.Fatal("the first assignment not made raised nothing")
|
||||
}
|
||||
|
||||
if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failedBuild(t, open, "b-2")
|
||||
settlePending(ctx, open, time.Now())
|
||||
|
||||
var second inventory.PendingAssignment
|
||||
for _, p := range pendingOf(t, open) {
|
||||
if p.Build == "b-2" {
|
||||
second = p
|
||||
}
|
||||
}
|
||||
if second.State != inventory.PendingExpired {
|
||||
t.Fatalf("the second: %+v", second)
|
||||
}
|
||||
if !conditionOpen(t, second) {
|
||||
t.Fatal("the second assignment's condition was cleared in the tick that raised it")
|
||||
}
|
||||
if !conditionOpen(t, rowOf(t, open, first.ID)) {
|
||||
t.Fatal("the first was judged answered by a second that itself was not made")
|
||||
}
|
||||
// unassign takes both back, and the next tick clears both.
|
||||
if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") ||
|
||||
!strings.Contains(said, "b-2") {
|
||||
t.Fatalf("unassign: %q %v", said, err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
if conditionOpen(t, first) || conditionOpen(t, second) {
|
||||
t.Fatal("a condition stayed open after both were taken back")
|
||||
}
|
||||
}
|
||||
|
||||
// **A raised row outlives the pruning until its condition clears** (second review, bug B); and a machine's
|
||||
// removal, which takes its rows with it, clears their conditions at the next tick.
|
||||
func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failedBuild(t, open, "b-modules/sensors")
|
||||
settlePending(ctx, open, time.Now())
|
||||
row := pendingOf(t, open)[0]
|
||||
|
||||
settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||
if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) {
|
||||
t.Fatal("a raised row was pruned, or its condition closed, while the condition was open")
|
||||
}
|
||||
if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||
if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] {
|
||||
t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err)
|
||||
}
|
||||
|
||||
// On anchor, then anchor removed: the row goes, and its condition with it at the next tick.
|
||||
if _, err := assign(ctx, open, "anchor", "sensors"); err == nil {
|
||||
t.Fatal("known and not built was not refused")
|
||||
}
|
||||
q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
q.Node = "anchor"
|
||||
if !conditionOpen(t, q) {
|
||||
t.Fatal("not raised")
|
||||
}
|
||||
if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
if conditionOpen(t, q) {
|
||||
t.Fatal("a removed machine's condition stayed open")
|
||||
}
|
||||
}
|
||||
|
||||
// **A claim left by a controller that stopped** (second review): the tick settles it from what the mesh
|
||||
// holds — back to waiting when the module is not assigned there, applied when it is — and leaves a fresh
|
||||
// claim alone.
|
||||
func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
claimed := func(node string) inventory.PendingAssignment {
|
||||
p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok {
|
||||
t.Fatalf("claim: %v %v", ok, err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
notAssigned, assigned := claimed("laptop"), claimed("anchor")
|
||||
register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"})
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
settlePending(ctx, open, time.Now())
|
||||
if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying {
|
||||
t.Fatal("a fresh claim was settled")
|
||||
}
|
||||
settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
|
||||
if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied {
|
||||
t.Fatalf("a stale claim of an assigned module: %+v", got)
|
||||
}
|
||||
// Released to waiting, and in the same tick made, since sensors is registered now.
|
||||
if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied ||
|
||||
!slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatalf("a stale claim of a module not assigned: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A waited build whose asker stopped waiting is asked, outcome unknown** (second review): it may still run,
|
||||
// so it reads as in flight until its outcome or its bound, never as not asked; a build never handed over
|
||||
// is not asked.
|
||||
func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, id := range []string{"build-timeout", "build-nothandedover"} {
|
||||
dir := "modules/" + strings.TrimPrefix(id, "build-")
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: dir, Ref: "main", For: "build"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
markWaitFailed(ctx, "build-timeout", errors.New("no build machine answered within 10m0s"))
|
||||
markWaitFailed(ctx, "build-nothandedover", fmt.Errorf("%w: cannot submit a build: nats: timeout", link.ErrNotHandedOver))
|
||||
|
||||
said, err := assign(ctx, open, "laptop", "timeout")
|
||||
if err != nil || !strings.Contains(said, "being built") || !strings.Contains(said, "kept as pending") {
|
||||
t.Fatalf("a build no longer waited for: %q %v", said, err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "nothandedover"); err == nil || !strings.Contains(err.Error(), "was not handed over") {
|
||||
t.Fatalf("a build never handed over: %v", err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now().Add(buildRequestBound+time.Minute))
|
||||
rows, _ := inv.PendingFor(ctx, "laptop", "timeout")
|
||||
if len(rows) != 1 || rows[0].State != inventory.PendingExpired || !strings.Contains(rows[0].Note, "its asker stopped waiting") {
|
||||
t.Fatalf("past the bound: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,7 +108,7 @@ func MembershipFor(node string, d Declared, where Placements) Membership {
|
||||
m.Seats = append(m.Seats, SeatServed{Seat: s.Name, Verb: verb, Subject: seatToolSubject(s, verb, node)})
|
||||
}
|
||||
}
|
||||
m.State = stateIssuedFor(d)
|
||||
m.State = stateIssuedFor(d, node)
|
||||
if len(d.Invokes) > 0 {
|
||||
m.Reaches = map[string][]string{}
|
||||
for _, t := range d.Invokes {
|
||||
|
||||
@@ -111,6 +111,10 @@ type Principal struct {
|
||||
// instances and read by whoever declares it.
|
||||
State []string
|
||||
Reads []string
|
||||
// PerMachine is the local names of its state keyed by machine, reaching the key named for Node and
|
||||
// no other; KeyedReads the keys of others' state it reads one key at a time (novox/hq ADR 0260).
|
||||
PerMachine []string
|
||||
KeyedReads []KeyedRead
|
||||
|
||||
// SnapshotsTheBus is the bus's own module, the one holding mesh-broker (novox/hq ADR 0235). Its
|
||||
// whole authority is BusSnapshotGrants: it copies the streams for the night's backup and nothing
|
||||
@@ -591,7 +595,8 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
|
||||
// 5. Its state, and the state of others it reads (novox/hq ADR 0201): every one read and
|
||||
// watched, its own written too.
|
||||
pub = append(pub, stateGrants(p.Module, p.State, p.Reads)...)
|
||||
pub = append(pub, stateGrants(stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State,
|
||||
PerMachine: p.PerMachine, Reads: p.Reads, KeyedReads: p.KeyedReads})...)
|
||||
|
||||
case KindNodeTools:
|
||||
// **One process serves what every module on the machine would have served for itself**
|
||||
@@ -667,7 +672,8 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// does not write another's bucket through it is the runtime's to keep, from the membership
|
||||
// each assignment is issued, as it keeps each module's events under that module's own name.
|
||||
for _, d := range p.Carries {
|
||||
pub = append(pub, stateGrants(d.Module, stateNames(d.State), d.Reads)...)
|
||||
pub = append(pub, stateGrants(stateAccess{Module: d.Module, Node: p.Node, Keeps: stateNames(d.State),
|
||||
PerMachine: perMachineNames(d.State), Reads: d.Reads, KeyedReads: d.KeyedReads})...)
|
||||
}
|
||||
sub = unique(sub)
|
||||
pub = unique(pub)
|
||||
|
||||
+89
-14
@@ -32,6 +32,9 @@ type Bucket struct {
|
||||
History int
|
||||
// TTLSeconds is how long a value lives; zero is until replaced or deleted.
|
||||
TTLSeconds int
|
||||
// PerMachine says each machine's instance writes only the key named for its machine (novox/hq ADR
|
||||
// 0260), and is granted that key and no other.
|
||||
PerMachine bool
|
||||
}
|
||||
|
||||
// BucketName is the bucket a module's state lives in: the module and the local name joined by an
|
||||
@@ -72,52 +75,124 @@ func bucketOfRead(read string) (string, bool) {
|
||||
// read; without CONSUMER.CREATE no key is listed and nothing is watched; without CONSUMER.DELETE a
|
||||
// watch cannot be stopped and lingers on the server. A write outside these is refused by the server
|
||||
// — and reaches the writer as a timeout, not a refusal, which is why the runtime refuses first.
|
||||
func stateGrants(module string, keeps []string, reads []string) []string {
|
||||
//
|
||||
// **One key, where the state is per machine** (novox/hq ADR 0260). The owner's instance on a machine
|
||||
// writes and reads the key named for that machine, and a holder granted a read for its machine reads
|
||||
// that key: a direct get of the key's own subject, and a consumer whose filter is that subject — the
|
||||
// server's create-with-filter form, which a client uses for a watch of one key and which the server
|
||||
// holds to the filter it names. What cannot be narrowed by key: binding (STREAM.INFO), deleting a
|
||||
// consumer (named at random by the client) and flow control; none of them reads or writes a value.
|
||||
func stateGrants(a stateAccess) []string {
|
||||
var out []string
|
||||
read := func(bucket string) {
|
||||
read := func(bucket, key string) {
|
||||
stream := "KV_" + bucket
|
||||
out = append(out, "$JS.API.STREAM.INFO."+stream)
|
||||
if key == "" {
|
||||
out = append(out,
|
||||
"$JS.API.DIRECT.GET."+stream+".>",
|
||||
"$JS.API.CONSUMER.CREATE."+stream+".>")
|
||||
} else {
|
||||
out = append(out,
|
||||
"$JS.API.DIRECT.GET."+stream+".$KV."+bucket+"."+key,
|
||||
"$JS.API.CONSUMER.CREATE."+stream+".*.$KV."+bucket+"."+key)
|
||||
}
|
||||
out = append(out,
|
||||
"$JS.API.STREAM.INFO."+stream,
|
||||
"$JS.API.DIRECT.GET."+stream+".>",
|
||||
"$JS.API.CONSUMER.CREATE."+stream+".>",
|
||||
"$JS.API.CONSUMER.DELETE."+stream+".>",
|
||||
"$JS.FC."+stream+".>")
|
||||
}
|
||||
for _, name := range keeps {
|
||||
machineKey := ""
|
||||
if safeSubject.MatchString(a.Node) {
|
||||
machineKey = a.Node
|
||||
}
|
||||
perMachine := map[string]bool{}
|
||||
for _, n := range a.PerMachine {
|
||||
perMachine[n] = true
|
||||
}
|
||||
for _, name := range a.Keeps {
|
||||
if !safeSubject.MatchString(name) {
|
||||
continue
|
||||
}
|
||||
bucket := BucketName(module, name)
|
||||
read(bucket)
|
||||
bucket := BucketName(a.Module, name)
|
||||
if perMachine[name] {
|
||||
if machineKey == "" {
|
||||
continue // a machine with no usable name reaches none of it
|
||||
}
|
||||
read(bucket, machineKey)
|
||||
out = append(out, "$KV."+bucket+"."+machineKey)
|
||||
continue
|
||||
}
|
||||
read(bucket, "")
|
||||
out = append(out, "$KV."+bucket+".>")
|
||||
}
|
||||
for _, r := range reads {
|
||||
for _, r := range a.Reads {
|
||||
if bucket, ok := bucketOfRead(r); ok {
|
||||
read(bucket)
|
||||
read(bucket, "")
|
||||
}
|
||||
}
|
||||
for _, k := range a.KeyedReads {
|
||||
if bucket, ok := bucketOfRead(k.Read); ok && safeSubject.MatchString(k.Key) {
|
||||
read(bucket, k.Key)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// KeyedRead is one key of another module's state a principal reads, and no other key of it.
|
||||
type KeyedRead struct {
|
||||
Read string `json:"read"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// stateAccess is what one principal may do with state: the module's own buckets (some per machine),
|
||||
// what it reads whole, and what it reads for its machine's key alone.
|
||||
type stateAccess struct {
|
||||
Module, Node string
|
||||
Keeps, PerMachine []string
|
||||
Reads []string
|
||||
KeyedReads []KeyedRead
|
||||
}
|
||||
|
||||
// perMachineNames is the local names of a module's buckets keyed by machine.
|
||||
func perMachineNames(buckets []Bucket) []string {
|
||||
var out []string
|
||||
for _, b := range buckets {
|
||||
if b.PerMachine {
|
||||
out = append(out, b.Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// StateIssued is one bucket an assignment may reach, by the name its module uses for it: its own
|
||||
// state by the local name, another's as `<module>.<name>` (novox/hq ADR 0201).
|
||||
// state by the local name, another's as `<module>.<name>` (novox/hq ADR 0201). Key, when set, is the
|
||||
// one key it may reach (novox/hq ADR 0260): this machine's.
|
||||
type StateIssued struct {
|
||||
Name string `json:"name"`
|
||||
Bucket string `json:"bucket"`
|
||||
Writes bool `json:"writes,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// stateIssuedFor is every bucket a module's code may reach, as its membership lists them.
|
||||
func stateIssuedFor(d Declared) []StateIssued {
|
||||
// stateIssuedFor is every bucket a module's code may reach on a machine, as its membership lists them.
|
||||
func stateIssuedFor(d Declared, node string) []StateIssued {
|
||||
var out []StateIssued
|
||||
for _, b := range d.State {
|
||||
out = append(out, StateIssued{Name: b.Name, Bucket: BucketName(d.Module, b.Name), Writes: true})
|
||||
issued := StateIssued{Name: b.Name, Bucket: BucketName(d.Module, b.Name), Writes: true}
|
||||
if b.PerMachine {
|
||||
issued.Key = node
|
||||
}
|
||||
out = append(out, issued)
|
||||
}
|
||||
for _, r := range d.Reads {
|
||||
if bucket, ok := bucketOfRead(r); ok {
|
||||
out = append(out, StateIssued{Name: r, Bucket: bucket})
|
||||
}
|
||||
}
|
||||
for _, k := range d.KeyedReads {
|
||||
if bucket, ok := bucketOfRead(k.Read); ok {
|
||||
out = append(out, StateIssued{Name: k.Read, Bucket: bucket, Key: k.Key})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -78,7 +78,8 @@ func TestAModuleWithNoStateReachesNoBucket(t *testing.T) {
|
||||
|
||||
// A read that names no bucket grants nothing rather than something that happens to parse.
|
||||
func TestAReadThatNamesNoBucketGrantsNothing(t *testing.T) {
|
||||
if got := stateGrants("a", nil, []string{"nodot", "x.", ".y", "a.b>"}); len(got) != 0 {
|
||||
if got := stateGrants(stateAccess{Module: "a", Node: "one", Reads: []string{"nodot", "x.", ".y", "a.b>"},
|
||||
KeyedReads: []KeyedRead{{Read: "nodot", Key: "one"}, {Read: "a.b", Key: "x>"}}}); len(got) != 0 {
|
||||
t.Fatalf("granted %v for reads that name no bucket", got)
|
||||
}
|
||||
}
|
||||
@@ -178,3 +179,58 @@ func TestABucketIsAssertedOverAHeldConnection(t *testing.T) {
|
||||
t.Fatalf("the bucket is not there: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **State keyed by machine reaches its machine's key alone** (novox/hq ADR 0260): the owner's instance on
|
||||
// a machine writes and reads that key, and a bar granted a key reads that key — a direct get of its
|
||||
// subject and a consumer filtered to it — and no other key of the bucket.
|
||||
func TestPerMachineStateAndAKeyedReadReachOneKey(t *testing.T) {
|
||||
owner, err := PermissionsFor(Principal{Kind: KindModule, Node: "laptop", Module: "power", PasswordHash: "x",
|
||||
State: []string{"draw"}, PerMachine: []string{"draw"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, owner.Publish, "$KV.power_draw.laptop")
|
||||
has(t, owner.Publish, "$JS.API.DIRECT.GET.KV_power_draw.$KV.power_draw.laptop")
|
||||
has(t, owner.Publish, "$JS.API.CONSUMER.CREATE.KV_power_draw.*.$KV.power_draw.laptop")
|
||||
hasNot(t, owner.Publish, "$KV.power_draw.>")
|
||||
hasNot(t, owner.Publish, "$JS.API.DIRECT.GET.KV_power_draw.>")
|
||||
hasNot(t, owner.Publish, "$JS.API.CONSUMER.CREATE.KV_power_draw.>")
|
||||
|
||||
bar, err := PermissionsFor(Principal{Kind: KindModule, Node: "laptop", Module: "bar", PasswordHash: "x",
|
||||
KeyedReads: []KeyedRead{{Read: "power.draw", Key: "laptop"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, bar.Publish, "$JS.API.STREAM.INFO.KV_power_draw")
|
||||
has(t, bar.Publish, "$JS.API.DIRECT.GET.KV_power_draw.$KV.power_draw.laptop")
|
||||
has(t, bar.Publish, "$JS.API.CONSUMER.CREATE.KV_power_draw.*.$KV.power_draw.laptop")
|
||||
for _, s := range bar.Publish {
|
||||
if strings.HasPrefix(s, "$KV.") || strings.HasSuffix(s, "KV_power_draw.>") && !strings.Contains(s, "DELETE") && !strings.HasPrefix(s, "$JS.FC.") {
|
||||
t.Fatalf("a keyed read was granted %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
// And the machine's runtime, which carries both, is granted the union: still the one key.
|
||||
runtime, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "laptop", Module: RuntimeModule, PasswordHash: "x",
|
||||
Carries: []Declared{
|
||||
{Module: "power", State: []Bucket{{Module: "power", Name: "draw", PerMachine: true}}},
|
||||
{Module: "bar", KeyedReads: []KeyedRead{{Read: "power.draw", Key: "laptop"}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, runtime.Publish, "$KV.power_draw.laptop")
|
||||
hasNot(t, runtime.Publish, "$KV.power_draw.>")
|
||||
hasNot(t, runtime.Publish, "$JS.API.DIRECT.GET.KV_power_draw.>")
|
||||
}
|
||||
|
||||
func TestAMembershipNamesTheOneKeyOfKeyedState(t *testing.T) {
|
||||
m := MembershipFor("laptop", Declared{Module: "bar", KeyedReads: []KeyedRead{{Read: "power.draw", Key: "laptop"}}}, Placements{})
|
||||
if len(m.State) != 1 || m.State[0] != (StateIssued{Name: "power.draw", Bucket: "power_draw", Key: "laptop"}) {
|
||||
t.Fatalf("%+v", m.State)
|
||||
}
|
||||
o := MembershipFor("laptop", Declared{Module: "power", State: []Bucket{{Module: "power", Name: "draw", PerMachine: true}}}, Placements{})
|
||||
if len(o.State) != 1 || o.State[0] != (StateIssued{Name: "draw", Bucket: "power_draw", Writes: true, Key: "laptop"}) {
|
||||
t.Fatalf("%+v", o.State)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,9 @@ type Declared struct {
|
||||
State []Bucket
|
||||
// Reads are other modules' state it reads, each `<module>.<name>` (novox/hq ADR 0201).
|
||||
Reads []string
|
||||
// KeyedReads are keys of other modules' state it reads, each one key alone: what a seat's holder is
|
||||
// granted for the pieces the modules beside it offer (novox/hq ADR 0260).
|
||||
KeyedReads []KeyedRead
|
||||
// NoAccount says the module declares no own secret named broker, so no account could ever be
|
||||
// delivered to it and nothing can connect as it (novox/hq issue 195). Said in the negative so a
|
||||
// record that does not say is composed as it always was.
|
||||
@@ -113,6 +116,7 @@ func Users(r Records) ([]Principal, error) {
|
||||
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
|
||||
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
|
||||
State: stateNames(d.State), Reads: d.Reads, SnapshotsTheBus: d.SnapshotsTheBus,
|
||||
PerMachine: perMachineNames(d.State), KeyedReads: d.KeyedReads,
|
||||
})
|
||||
}
|
||||
if runtimeHere {
|
||||
|
||||
@@ -136,16 +136,18 @@ var barShows = map[string]map[string]barOption{
|
||||
"state": {"state": {kind: "text", required: true}, "key": {kind: "text"}},
|
||||
}
|
||||
|
||||
// barBlockReads is the state a block shows, if any.
|
||||
func barBlockReads(data map[string]any) []string {
|
||||
// barBlockReads is the state and key a block shows, if any; an empty key is the machine's own.
|
||||
func barBlockReads(data map[string]any) []StateRead {
|
||||
if data["shows"] != "state" {
|
||||
return nil
|
||||
}
|
||||
options, _ := data["options"].(map[string]any)
|
||||
if s, ok := options["state"].(string); ok && s != "" {
|
||||
return []string{s}
|
||||
s, ok := options["state"].(string)
|
||||
if !ok || s == "" {
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
key, _ := options["key"].(string)
|
||||
return []StateRead{{Read: s, Key: key}}
|
||||
}
|
||||
|
||||
type barOption struct {
|
||||
|
||||
@@ -38,10 +38,10 @@ type Shape struct {
|
||||
// Examples are pieces every holder's template must render, one for each variant the shape has,
|
||||
// so a holder that cannot render one is refused at registration and not found on a machine.
|
||||
Examples []map[string]any
|
||||
// Reads is the state on the bus a piece shows, as `<module>.<name>`, or nothing (novox/hq ADR
|
||||
// 0255): a contributor offers only state it keeps itself, and the holder on the same machine is
|
||||
// granted to read it, so the holder's manifest names no contributor.
|
||||
Reads func(data map[string]any) []string `json:"-"`
|
||||
// Reads is the state on the bus a piece shows, each a state and the one key of it, or nothing
|
||||
// (novox/hq ADR 0260): a contributor offers only state it keeps itself, and the holder on the same
|
||||
// machine is granted to read that key, so the holder's manifest names no contributor.
|
||||
Reads func(data map[string]any) []StateRead `json:"-"`
|
||||
}
|
||||
|
||||
// PlaceField is one field a holder places by, and the values it takes.
|
||||
@@ -120,25 +120,38 @@ func ownStateProblems(m Manifest, i int, r Receivable, c SeatContribution) []str
|
||||
keeps[s.Name] = true
|
||||
}
|
||||
var problems []string
|
||||
for _, read := range r.Shape.Reads(c.Data) {
|
||||
module, local, err := ReadState(read)
|
||||
for _, sr := range r.Shape.Reads(c.Data) {
|
||||
module, local, err := ReadState(sr.Read)
|
||||
switch {
|
||||
case err != nil:
|
||||
problems = append(problems, fmt.Sprintf("%s's contribution %d shows state %v", m.Module, i+1, err))
|
||||
case module != m.Module || !keeps[local]:
|
||||
problems = append(problems, fmt.Sprintf("%s's contribution %d shows the state %s, which %s does not keep; a "+
|
||||
"module offers its own state only (novox/hq ADR 0255)", m.Module, i+1, read, m.Module))
|
||||
"module offers its own state only (novox/hq ADR 0260)", m.Module, i+1, sr.Read, m.Module))
|
||||
}
|
||||
if sr.Key != "" && !stateKeyName.MatchString(sr.Key) {
|
||||
problems = append(problems, fmt.Sprintf("%s's contribution %d shows the key %q; a key is lower-case letters, "+
|
||||
"digits and hyphens", m.Module, i+1, sr.Key))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// StateRead is one key of another module's state that a holder reads: Read is `<module>.<name>`,
|
||||
// and Key the one key, empty for the holder's own machine until ReadsGranted names it.
|
||||
type StateRead struct {
|
||||
Read, Key string
|
||||
}
|
||||
|
||||
// stateKeyName is a key a piece may name: a machine's name is one.
|
||||
var stateKeyName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
// ReadsGranted is the state a holder reads because modules on its machine offer it in pieces of a
|
||||
// kind its seat receives as data (novox/hq ADR 0255), sorted and once each. The bus grants the holder
|
||||
// these beside its own `reads`: the holder names no contributor, and reads only what is offered to it
|
||||
// on the machine it runs on.
|
||||
func ReadsGranted(holder Manifest, onMachine []Manifest) []string {
|
||||
seen := map[string]bool{}
|
||||
// kind its seat receives as data (novox/hq ADR 0260), each with the one key the piece shows — the
|
||||
// machine's own when it names none — sorted and once each. The bus grants the holder these keys and
|
||||
// no others: the holder names no contributor, and reads only what is offered to it where it runs.
|
||||
func ReadsGranted(holder Manifest, onMachine []Manifest, machine string) []StateRead {
|
||||
seen := map[StateRead]bool{}
|
||||
for _, claim := range holder.Claims {
|
||||
s, known := SeatNamed(claim.Name)
|
||||
if !known {
|
||||
@@ -153,16 +166,30 @@ func ReadsGranted(holder Manifest, onMachine []Manifest) []string {
|
||||
if cs, ok := SeatNamed(c.Seat); !ok || cs.Name != s.Name || c.Kind != r.Kind {
|
||||
continue
|
||||
}
|
||||
for _, read := range r.Shape.Reads(c.Data) {
|
||||
if module, _, err := ReadState(read); err == nil && module == m.Module && module != holder.Module {
|
||||
seen[read] = true
|
||||
for _, sr := range r.Shape.Reads(c.Data) {
|
||||
if sr.Key == "" {
|
||||
sr.Key = machine
|
||||
}
|
||||
if module, _, err := ReadState(sr.Read); err == nil && module == m.Module && module != holder.Module &&
|
||||
stateKeyName.MatchString(sr.Key) {
|
||||
seen[sr] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return sortedKeys(seen)
|
||||
out := make([]StateRead, 0, len(seen))
|
||||
for sr := range seen {
|
||||
out = append(out, sr)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Read != out[j].Read {
|
||||
return out[i].Read < out[j].Read
|
||||
}
|
||||
return out[i].Key < out[j].Key
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// wholeIn is v as a whole number within [lo, hi]; JSON gives every number as a float.
|
||||
|
||||
@@ -364,14 +364,15 @@ func TestAStateBlockIsRenderedForThisMachineAndTheHolderIsGrantedToReadIt(t *tes
|
||||
if bar := composedFile(t, r, "a-bar.bottom"); !strings.Contains(bar, `command = "show power.draw laptop"`) {
|
||||
t.Fatalf("the block does not read this machine's key:\n%s", bar)
|
||||
}
|
||||
if got := ReadsGranted(barHolder(), []Manifest{barHolder(), power}); len(got) != 1 || got[0] != "power.draw" {
|
||||
if got := ReadsGranted(barHolder(), []Manifest{barHolder(), power}, "laptop"); len(got) != 1 ||
|
||||
got[0] != (StateRead{Read: "power.draw", Key: "laptop"}) {
|
||||
t.Fatalf("the holder is granted %v", got)
|
||||
}
|
||||
// Only the holder, and only for what the modules on its machine offer.
|
||||
if got := ReadsGranted(power, []Manifest{barHolder(), power}); len(got) != 0 {
|
||||
if got := ReadsGranted(power, []Manifest{barHolder(), power}, "laptop"); len(got) != 0 {
|
||||
t.Fatalf("a contributor is granted %v", got)
|
||||
}
|
||||
if got := ReadsGranted(barHolder(), []Manifest{barHolder()}); len(got) != 0 {
|
||||
if got := ReadsGranted(barHolder(), []Manifest{barHolder()}, "laptop"); len(got) != 0 {
|
||||
t.Fatalf("a holder alone is granted %v", got)
|
||||
}
|
||||
}
|
||||
@@ -393,3 +394,14 @@ func TestAModuleOffersItsOwnStateOnly(t *testing.T) {
|
||||
t.Errorf("its own state was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerMachineStateIsDeclaredAndKeptInItsLongForm(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"power","state":[{"name":"draw","ttl-seconds":30,"per-machine":true}]}`))
|
||||
if err != nil || len(m.State) != 1 || !m.State[0].PerMachine {
|
||||
t.Fatalf("%+v %v", m.State, err)
|
||||
}
|
||||
raw, _ := json.Marshal(StateDeclaration{Name: "draw", PerMachine: true})
|
||||
if string(raw) != `{"name":"draw","per-machine":true}` {
|
||||
t.Fatalf("%s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,9 @@ type StateDeclaration struct {
|
||||
History int `json:"history,omitempty"`
|
||||
// TTLSeconds is how long a value lives once written; zero is until it is replaced or deleted.
|
||||
TTLSeconds int `json:"ttl-seconds,omitempty"`
|
||||
// PerMachine says the state is keyed by machine: each machine's instance writes the key named for
|
||||
// its machine and no other, and the bus grants it that key alone (novox/hq ADR 0260).
|
||||
PerMachine bool `json:"per-machine,omitempty"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads a bucket as its bare name, or as {name, history, ttl-seconds}.
|
||||
@@ -46,7 +49,7 @@ func (s *StateDeclaration) UnmarshalJSON(raw []byte) error {
|
||||
dec := json.NewDecoder(bytes.NewReader(trimmed))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return fmt.Errorf("a state is either a name or {name, history, ttl-seconds}: %w", err)
|
||||
return fmt.Errorf("a state is either a name or {name, history, ttl-seconds, per-machine}: %w", err)
|
||||
}
|
||||
*s = StateDeclaration(full)
|
||||
return nil
|
||||
@@ -54,7 +57,7 @@ func (s *StateDeclaration) UnmarshalJSON(raw []byte) error {
|
||||
|
||||
// MarshalJSON writes back the short form when there is nothing else to say.
|
||||
func (s StateDeclaration) MarshalJSON() ([]byte, error) {
|
||||
if s.History == 0 && s.TTLSeconds == 0 {
|
||||
if s.History == 0 && s.TTLSeconds == 0 && !s.PerMachine {
|
||||
return json.Marshal(s.Name)
|
||||
}
|
||||
type plain StateDeclaration
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestAStateNameIsLocalAndOneToken(t *testing.T) {
|
||||
{`{"module":"a.b","version":"1","state":["s"]}`, `no dot`},
|
||||
{`{"module":"a","version":"1","reads":["bindings"]}`, `a read is <module>.<name>`},
|
||||
{`{"module":"a","version":"1","reads":["a.s"]}`, `which is its own state`},
|
||||
{`{"module":"a","version":"1","state":[{"name":"s","shared":true}]}`, `{name, history, ttl-seconds}`},
|
||||
{`{"module":"a","version":"1","state":[{"name":"s","shared":true}]}`, `{name, history, ttl-seconds, per-machine}`},
|
||||
} {
|
||||
_, err := ParseManifest([]byte(c.manifest))
|
||||
if err == nil {
|
||||
|
||||
@@ -80,8 +80,9 @@ func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) {
|
||||
}
|
||||
d := declaredFor(m, seats)
|
||||
// And the state offered to it as a seat's holder by the modules beside it (novox/hq ADR 0255).
|
||||
if granted := catalogue.ReadsGranted(m, onMachine(declared, modules)); len(granted) > 0 {
|
||||
d.Reads = append(append([]string(nil), d.Reads...), granted...)
|
||||
// For this machine's key alone (novox/hq ADR 0260): a bar shows its own machine's draw.
|
||||
for _, sr := range catalogue.ReadsGranted(m, onMachine(declared, modules), n.Name) {
|
||||
d.KeyedReads = append(d.KeyedReads, broker.KeyedRead{Read: sr.Read, Key: sr.Key})
|
||||
}
|
||||
d.Holds = heldHere(d.Holds, holdings, n.Name, module)
|
||||
out.Assigned[n.Name] = append(out.Assigned[n.Name], d)
|
||||
@@ -182,7 +183,8 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
|
||||
func bucketsOf(m catalogue.Manifest) []broker.Bucket {
|
||||
var out []broker.Bucket
|
||||
for _, s := range m.State {
|
||||
out = append(out, broker.Bucket{Module: m.Module, Name: s.Name, History: s.History, TTLSeconds: s.TTLSeconds})
|
||||
out = append(out, broker.Bucket{Module: m.Module, Name: s.Name, History: s.History, TTLSeconds: s.TTLSeconds,
|
||||
PerMachine: s.PerMachine})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -322,16 +322,20 @@ func TestABarIsGrantedTheStateItsMachinesModulesOfferIt(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reads := func(node, module string) []string {
|
||||
reads := func(node, module string) []broker.KeyedRead {
|
||||
for _, d := range records.Assigned[node] {
|
||||
if d.Module == module {
|
||||
return d.Reads
|
||||
if len(d.Reads) != 0 {
|
||||
t.Fatalf("%s on %s reads whole buckets: %v", module, node, d.Reads)
|
||||
}
|
||||
return d.KeyedReads
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s is not on %s", module, node)
|
||||
return nil
|
||||
}
|
||||
if got := reads("laptop", "a-bar"); len(got) != 1 || got[0] != "power.draw" {
|
||||
// The laptop's own key, and no other (novox/hq ADR 0260).
|
||||
if got := reads("laptop", "a-bar"); len(got) != 1 || got[0] != (broker.KeyedRead{Read: "power.draw", Key: "laptop"}) {
|
||||
t.Fatalf("the laptop's bar reads %v", got)
|
||||
}
|
||||
if got := reads("desk", "a-bar"); len(got) != 0 {
|
||||
|
||||
@@ -68,3 +68,13 @@ func ForTest(t *testing.T) *Inventory {
|
||||
}
|
||||
return inv
|
||||
}
|
||||
|
||||
// RemoveNodeForTest removes a machine's record as a person removing it from the store would: no verb of the
|
||||
// mesh removes one yet, and what hangs off it must go with it.
|
||||
func (i *Inventory) RemoveNodeForTest(ctx context.Context, name string) (bool, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx, `delete from node where name = $1`, name)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
@@ -8,8 +8,10 @@
|
||||
|
||||
-- Every build the controller asked for: a merge's new module, a plan's tier, a person's `build`, an `assign`
|
||||
-- with build. Kept once the ask is made, never before. Its outcome is the `build` row of the same id;
|
||||
-- an ask without one is still running, or was lost. not_asked is why the ask could not be made or waited
|
||||
-- for; for a merge that could not ask, the id is the controller's and no build's.
|
||||
-- an ask without one is still running, or was lost. not_asked is why the ask could not be handed over; for a
|
||||
-- merge that could not ask, the id is the controller's and no build's. outcome_unknown is why an asker that
|
||||
-- handed it over stopped waiting: the build may still run, and is read as in flight until its outcome or
|
||||
-- its bound.
|
||||
create table build_request (
|
||||
id text primary key,
|
||||
repository text not null,
|
||||
@@ -19,6 +21,7 @@ create table build_request (
|
||||
commit_hash text not null default '',
|
||||
asked_for text not null default '',
|
||||
not_asked text,
|
||||
outcome_unknown text,
|
||||
asked_at timestamptz not null default now()
|
||||
);
|
||||
create index build_request_asked_at on build_request (asked_at);
|
||||
@@ -28,7 +31,7 @@ create index build_request_asked_at on build_request (asked_at);
|
||||
-- assignment was refused then), expired (the build failed, was not registered, or said nothing within its
|
||||
-- bound) or withdrawn (`unassign`). raised_at is when an expiry or refusal was raised as a condition,
|
||||
-- acknowledged_at when a person took it back with `unassign`, cleared_at when its condition was cleared.
|
||||
-- Gone with its machine. Ended rows are deleted after 30 days.
|
||||
-- Gone with its machine. Ended rows are deleted after 30 days, once any condition raised for them is cleared.
|
||||
create table pending_assignment (
|
||||
id bigserial primary key,
|
||||
node uuid not null references node (id) on delete cascade,
|
||||
|
||||
@@ -32,9 +32,12 @@ type BuildRequest struct {
|
||||
Commit string
|
||||
// For says who asked: "merge", "plan", "build" (a person), "assign".
|
||||
For string
|
||||
// NotAsked is why the ask could not be made, or not waited for; empty when it was.
|
||||
// NotAsked is why the ask could not be handed over; empty when it was.
|
||||
NotAsked string
|
||||
At time.Time
|
||||
// OutcomeUnknown is why an asker that handed the build over stopped waiting for it: asked, outcome
|
||||
// unknown. Read as in flight until its outcome or its bound.
|
||||
OutcomeUnknown string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// Name is the module this request is expected to register, read from its directory: the last element of
|
||||
@@ -80,8 +83,8 @@ func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) erro
|
||||
return err
|
||||
}
|
||||
|
||||
// MarkNotAsked says a build request was asked and its asker could not hand it over, or stopped waiting for
|
||||
// it: the words are kept, unless its outcome was heard first.
|
||||
// MarkNotAsked says a kept build request was never handed over: the words are kept, unless its outcome was
|
||||
// heard first.
|
||||
func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error {
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update build_request set not_asked = $2
|
||||
@@ -89,6 +92,15 @@ func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// MarkOutcomeUnknown says the asker of a build it handed over stopped waiting for it: asked, outcome unknown.
|
||||
// The build may still run; its outcome, when heard, is the last word.
|
||||
func (i *Inventory) MarkOutcomeUnknown(ctx context.Context, id, why string) error {
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update build_request set outcome_unknown = $2
|
||||
where id = $1 and not exists (select 1 from build where build.id = $1)`, id, why)
|
||||
return err
|
||||
}
|
||||
|
||||
// RequestsNamed is every build request kept whose directory names the module, or whose build said it built
|
||||
// it, newest first, each with its outcome.
|
||||
func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOutcome, error) {
|
||||
@@ -126,7 +138,7 @@ func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) {
|
||||
func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select a.id, a.repository, a.seat, a.source_path, a.ref, a.commit_hash, a.asked_for,
|
||||
coalesce(a.not_asked, ''), a.asked_at,
|
||||
coalesce(a.not_asked, ''), coalesce(a.outcome_unknown, ''), a.asked_at,
|
||||
b.id is not null, coalesce(b.failed, ''), coalesce(b.module, ''), coalesce(b.at, a.asked_at)
|
||||
from build_request a left join build b on b.id = a.id
|
||||
order by a.asked_at desc, a.id desc`)
|
||||
@@ -138,7 +150,7 @@ func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) {
|
||||
for rows.Next() {
|
||||
var a RequestOutcome
|
||||
if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked,
|
||||
&a.At, &a.Heard, &a.Failed, &a.Module, &a.HeardAt); err != nil {
|
||||
&a.OutcomeUnknown, &a.At, &a.Heard, &a.Failed, &a.Module, &a.HeardAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
@@ -342,13 +354,34 @@ func (i *Inventory) SettlePending(ctx context.Context, id int64, from, state, no
|
||||
return tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
// ForgetEndedPending deletes ended pending assignments settled more than KeptFor ago, and says how many.
|
||||
// ForgetEndedPending deletes ended pending assignments settled more than KeptFor ago, and says how many. One
|
||||
// raised as a condition is kept until that condition is cleared: deleting it would leave the condition
|
||||
// with nothing to answer it.
|
||||
func (i *Inventory) ForgetEndedPending(ctx context.Context, now time.Time) (int64, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`delete from pending_assignment where state not in ('waiting', 'applying') and settled_at < $1`,
|
||||
`delete from pending_assignment where state not in ('waiting', 'applying') and settled_at < $1
|
||||
and (raised_at is null or cleared_at is not null)`,
|
||||
now.Add(-KeptFor))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// PendingKnown says which of these pending assignments are still on record.
|
||||
func (i *Inventory) PendingKnown(ctx context.Context, ids []int64) (map[int64]bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select id from pending_assignment where id = any($1)`, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[int64]bool{}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[id] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
@@ -76,6 +76,11 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrNotHandedOver is a Submit that failed before the build was handed to the build seat: nothing runs. Any
|
||||
// other error from Submit came after: the build may still run, and its outcome may still come (novox/hq
|
||||
// issue 325).
|
||||
var ErrNotHandedOver = errors.New("the build was not handed over")
|
||||
|
||||
func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
|
||||
wait time.Duration) (BuildResult, error) {
|
||||
|
||||
@@ -84,23 +89,23 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
|
||||
// the same event on EVENTS, which the controller records.
|
||||
outcomes, err := b.js.Conn().SubscribeSync(BuildOutcomeOf(b.role()))
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("cannot listen for a build's outcome: %w", err)
|
||||
return BuildResult{}, fmt.Errorf("%w: cannot listen for a build's outcome: %w", ErrNotHandedOver, err)
|
||||
}
|
||||
defer func() { _ = outcomes.Unsubscribe() }()
|
||||
if err := b.js.Conn().Flush(); err != nil {
|
||||
return BuildResult{}, err
|
||||
return BuildResult{}, fmt.Errorf("%w: %w", ErrNotHandedOver, err)
|
||||
}
|
||||
|
||||
body, err := json.Marshal(request)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
return BuildResult{}, fmt.Errorf("%w: %w", ErrNotHandedOver, err)
|
||||
}
|
||||
// Into the role's work queue and awaited: work the bus never accepted must fail here rather than
|
||||
// be assumed, because nothing else will ever say so.
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("cannot submit a build: %w", err)
|
||||
return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err)
|
||||
}
|
||||
|
||||
waiting, cancelWait := context.WithTimeout(ctx, wait)
|
||||
|
||||
Reference in New Issue
Block a user