Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 jschoubben
|
6fe7b7dd6c |
@@ -1,4 +1 @@
|
||||
vendor/
|
||||
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
|
||||
modules/*/cmd/*/mesh-*
|
||||
!modules/*/cmd/*/mesh-*.go
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,21 +8,19 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -30,15 +28,6 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
if os.Getenv(process.BrokerFileEnv) == "" {
|
||||
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
|
||||
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
|
||||
if err := serveTools(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-delivery-assigner:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
@@ -50,12 +39,9 @@ func main() {
|
||||
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
|
||||
// holds the two lists equal), so the controller's broker grants exactly these.
|
||||
var readBuckets = []string{
|
||||
buscompose.ResolvedBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.BuildsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ManifestsBucket,
|
||||
@@ -69,56 +55,30 @@ var readBuckets = []string{
|
||||
// their steps in its steps state (novox/hq ADR 0482 clause 4).
|
||||
const goBuildModule = "mesh-build-go"
|
||||
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
|
||||
// (ADR 0567 §1), named after the assigner as every module state is.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket}
|
||||
|
||||
// seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants
|
||||
// them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind).
|
||||
var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposalsBucket, delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket}
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket}
|
||||
|
||||
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
|
||||
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9).
|
||||
// and the teller.
|
||||
var keptStates = []announce.State{
|
||||
{Name: delivery.PlacementState, Readable: true},
|
||||
{Name: delivery.PlacementProposalsState, Readable: true},
|
||||
{Name: delivery.PlacementCandidatesState, Readable: true},
|
||||
{Name: delivery.PlacementPulsesState, Readable: true},
|
||||
{Name: "asks", Readable: true},
|
||||
{Name: "compositions", Readable: true},
|
||||
{Name: "differences", Readable: true},
|
||||
{Name: "puts", Readable: true},
|
||||
{Name: "wants", Readable: true},
|
||||
}
|
||||
|
||||
// buckets is every bucket the assigner opened, by name.
|
||||
type buckets map[string]jetstream.KeyValue
|
||||
|
||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
|
||||
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
|
||||
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
|
||||
// without it and takes no put further until it starts again (ADR 0546).
|
||||
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
|
||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
|
||||
// without it would write wants from half the mesh.
|
||||
func open(ctx context.Context, js jetstream.JetStream) (buckets, error) {
|
||||
out := buckets{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if name == delivery.AssignerPutsBucket && err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if slices.Contains(seatBuckets, name) && err != nil {
|
||||
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement the controller's
|
||||
// assignments alone name where modules run, as before any placement was written (ADR 0549 §5).
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
|
||||
}
|
||||
out[name] = kv
|
||||
}
|
||||
@@ -128,16 +88,34 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
|
||||
account, err := process.ReadBrokerAccount(getenv)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
|
||||
}
|
||||
account, err := process.ReadAccount(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, js, err := account.Dial()
|
||||
opts := []nats.Option{
|
||||
nats.Name(account.Node + "." + account.Module),
|
||||
nats.UserInfo(account.User, account.Password),
|
||||
nats.CustomInboxPrefix(account.InboxPrefix()),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(account.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(account.URL, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
kvs, err := open(ctx, js, logger.Printf)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kvs, err := open(ctx, js)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -151,34 +129,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := said.Keeps(keptStates...); err != nil {
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
thisNode = account.Node
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, conn, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
return follow(ctx, js, kvs, logger)
|
||||
}
|
||||
|
||||
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
|
||||
var thisNode string
|
||||
|
||||
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||
type emitting struct{ *process.Conn }
|
||||
|
||||
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
|
||||
|
||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||
var watched = []string{
|
||||
buscompose.ResolvedBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.AssignerAsksBucket,
|
||||
@@ -189,7 +151,7 @@ var watched = []string{
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -198,15 +160,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
default:
|
||||
}
|
||||
}
|
||||
errs := make(chan error, len(watched)+2)
|
||||
settled := make(chan struct{}, len(watched)+2)
|
||||
errs := make(chan error, len(watched)+1)
|
||||
settled := make(chan struct{}, len(watched)+1)
|
||||
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
||||
kw, err := kvs[name].WatchAll(ctx, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
|
||||
}
|
||||
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
|
||||
optional := name == delivery.AssignerPutsBucket
|
||||
go func() {
|
||||
defer kw.Stop()
|
||||
for {
|
||||
@@ -216,69 +176,25 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
return
|
||||
case e, ok := <-kw.Updates():
|
||||
if !ok {
|
||||
if optional {
|
||||
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
|
||||
return
|
||||
}
|
||||
errs <- fmt.Errorf("the watch of %s ended", name)
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
if !optional { // the puts never gate the first pass, so never the wants (ADR 0546)
|
||||
settled <- struct{}{}
|
||||
}
|
||||
settled <- struct{}{}
|
||||
continue
|
||||
}
|
||||
var value []byte
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
w.put(name, e.Key(), value)
|
||||
wake()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
|
||||
// stops the puts, never the wants.
|
||||
if kvs[delivery.AssignerPutsBucket] != nil {
|
||||
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||
delete(kvs, delivery.AssignerPutsBucket)
|
||||
}
|
||||
}
|
||||
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
|
||||
// credential reads it once the bus's user list carries the grant: neither waits for the assigner. So while it is not open,
|
||||
// a pass started by a change opens it again, at most once a minute, and from then on the puts are watched and taken
|
||||
// further, without a restart.
|
||||
var triedPuts time.Time
|
||||
reopenPuts := func(p *passer) {
|
||||
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
|
||||
return
|
||||
}
|
||||
triedPuts = time.Now()
|
||||
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
kvs[delivery.AssignerPutsBucket] = kv
|
||||
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||
delete(kvs, delivery.AssignerPutsBucket)
|
||||
return
|
||||
}
|
||||
if bus != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && slices.Contains(seatBuckets, name) {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
var opts []jetstream.WatchOpt
|
||||
if name == delivery.HaveBucket {
|
||||
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
|
||||
@@ -290,29 +206,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||
return err
|
||||
}
|
||||
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
|
||||
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then the
|
||||
// controller's assignments alone name where modules run (ADR 0549 §5).
|
||||
type appeared struct {
|
||||
name string
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
placementAppeared := make(chan appeared, len(seatBuckets))
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
}
|
||||
takePlacement := func(a appeared) error {
|
||||
if kvs[a.name] != nil {
|
||||
return nil // bound already, by the other of its two events
|
||||
}
|
||||
kvs[a.name] = a.kv
|
||||
logger.Printf("the bucket %s is open and watched", a.name)
|
||||
return watch(a.name)
|
||||
}
|
||||
for range started + 1 {
|
||||
for range len(watched) + 1 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
@@ -321,17 +215,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
case <-settled:
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if bus != nil {
|
||||
p.seat = newSeatActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if js != nil {
|
||||
p.publish = streamPublisher(ctx, js, thisNode)
|
||||
}
|
||||
wake()
|
||||
for {
|
||||
select {
|
||||
@@ -340,50 +225,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
case err := <-errs:
|
||||
return err
|
||||
case <-settled:
|
||||
case kv := <-placementAppeared:
|
||||
if err := takePlacement(kv); err != nil {
|
||||
return err
|
||||
}
|
||||
case <-dirty:
|
||||
reopenPuts(p)
|
||||
// The grants part makes the seat's buckets in the pass that seals the list granting them, so their creation can
|
||||
// be said before this credential may hear it: every pass, itself started by a watched change, binds each one the
|
||||
// bus now holds, and says the seat waits while one is missing.
|
||||
var unbound []string
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] != nil || js == nil {
|
||||
continue
|
||||
}
|
||||
if kv, err := js.KeyValue(ctx, name); err == nil {
|
||||
if err := takePlacement(appeared{name, kv}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
unbound = append(unbound, name)
|
||||
}
|
||||
p.sayWaits(unbound)
|
||||
p.pass(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
|
||||
var PutsReopenEvery = time.Minute
|
||||
|
||||
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
|
||||
// (ADR 0437 clause 9).
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
subject, err := asks.ConditionSubject(delivery.AssignerModule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := asks.EncodeCondition(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Publish(ctx, subject, data)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,32 +2,20 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
|
||||
func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -60,111 +48,12 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("the program announces %v, keeps %v", got, want)
|
||||
}
|
||||
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
|
||||
for _, c := range []struct {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
}{
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
|
||||
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||
}
|
||||
}
|
||||
// And a watch of the puts that ends before it handed what stands (its subscription closed, its grant withdrawn) does not
|
||||
// keep the first pass from being made: the wants are written all the same.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
wrote := make(chan string, 8)
|
||||
kvs := buckets{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||
}
|
||||
ckey, _ := delivery.ComposedNodeKey("ace")
|
||||
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
t.Errorf("the first pass wrote %s", key)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Error("a puts watch that ended before its first marker kept the first pass from being made: no want was written")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingJS opens every bucket but one, which it refuses with err.
|
||||
type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
missing string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
if name == r.missing {
|
||||
return nil, r.err
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
func TestNoCredential(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" }, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
return &memKV{values: map[string][]byte{}}, nil
|
||||
}
|
||||
|
||||
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
|
||||
// closed; a put on it is said on wrote.
|
||||
type watchKV struct {
|
||||
*memKV
|
||||
entries []kvEntry
|
||||
closed bool
|
||||
wrote chan string
|
||||
}
|
||||
|
||||
func (w *watchKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
ch := make(chan jetstream.KeyValueEntry, len(w.entries)+1)
|
||||
if w.closed {
|
||||
close(ch)
|
||||
return keyWatcher{ch}, nil
|
||||
}
|
||||
for _, e := range w.entries {
|
||||
ch <- e
|
||||
}
|
||||
ch <- nil
|
||||
return keyWatcher{ch}, nil
|
||||
}
|
||||
|
||||
func (w *watchKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if w.wrote != nil {
|
||||
w.wrote <- key
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
type keyWatcher struct{ ch chan jetstream.KeyValueEntry }
|
||||
|
||||
func (k keyWatcher) Updates() <-chan jetstream.KeyValueEntry { return k.ch }
|
||||
func (k keyWatcher) Stop() error { return nil }
|
||||
|
||||
type kvEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
key string
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (e kvEntry) Key() string { return e.key }
|
||||
func (e kvEntry) Value() []byte { return e.value }
|
||||
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
func (e kvEntry) Revision() uint64 { return 1 }
|
||||
func (e kvEntry) Created() time.Time { return time.Time{} }
|
||||
|
||||
@@ -3,12 +3,8 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/sdkversion"
|
||||
@@ -23,89 +19,76 @@ type passer struct {
|
||||
w *world
|
||||
logf func(string, ...any)
|
||||
sdk string
|
||||
// say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(asks.Condition) error
|
||||
said map[string]bool
|
||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||
answeredAt map[string]uint64
|
||||
asks *composeAsks
|
||||
// now is the clock; nil is time.Now.
|
||||
now func() time.Time
|
||||
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
|
||||
// first pass after a start, which clears what a previous process may have raised (startClears).
|
||||
neither map[string]neitherOne
|
||||
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
|
||||
acts *putActs
|
||||
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
|
||||
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
|
||||
noWant map[string]string
|
||||
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
|
||||
// §1, issue 1496): raised once per words, cleared once when it ends.
|
||||
nodeFailed map[string]string
|
||||
// The seat mesh-placement's holder (seat.go, ADR 0567): its calls of the controller (nil: none is made), how it says
|
||||
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
|
||||
seat *seatActs
|
||||
publish publisher
|
||||
saidProposed map[string]bool
|
||||
// saidWaits is the last said of the seat's wait for its states (sayWaits).
|
||||
saidWaits string
|
||||
intended map[string]map[string]map[string]bool
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
if p.now != nil {
|
||||
return p.now()
|
||||
}
|
||||
return time.Now()
|
||||
asks *composeAsks
|
||||
}
|
||||
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
p.placementSeat(ctx)
|
||||
p.compositions(ctx)
|
||||
p.wants(ctx)
|
||||
p.puts(ctx)
|
||||
p.nodeWants(ctx)
|
||||
p.backups(ctx)
|
||||
rules := p.wants(ctx)
|
||||
p.differences(ctx, rules)
|
||||
}
|
||||
|
||||
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
placed := p.w.placedNodes()
|
||||
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
|
||||
for node := range placed {
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self {
|
||||
continue
|
||||
}
|
||||
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
// differences writes, for each node and module the assigner wants or the controller sends, the difference between the
|
||||
// two with the rule that explains it or unexplained (ADR 0528 clause 4), and deletes the key of a difference that
|
||||
// ended; its history keeps what it was.
|
||||
func (p *passer) differences(ctx context.Context, rules map[string]map[string]assign.Holding) {
|
||||
_, assignments := p.w.assigned()
|
||||
wanted := p.w.wantsOf()
|
||||
trunk := p.w.trunk()
|
||||
pairs := map[[2]string]bool{}
|
||||
for _, a := range assignments {
|
||||
pairs[[2]string{a.Node, a.Module}] = true
|
||||
}
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
if placed[node] {
|
||||
continue
|
||||
for module, byNode := range wanted {
|
||||
for node := range byNode {
|
||||
pairs[[2]string{node, module}] = true
|
||||
}
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
}
|
||||
written := p.w.read(delivery.AssignerDifferencesBucket)
|
||||
keep := map[string]bool{}
|
||||
for pair := range pairs {
|
||||
node, module := pair[0], pair[1]
|
||||
key, err := delivery.DifferenceKey(node, module)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
|
||||
var want *delivery.Want
|
||||
if w, ok := wanted[module][node]; ok {
|
||||
want = &w
|
||||
}
|
||||
var sent assign.Sent
|
||||
if a, ok := assignments[node+"."+module]; ok {
|
||||
sent = assign.Sent{Sends: true, Build: a.Build, FP: p.w.sentFP(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module, a.Build)}
|
||||
}
|
||||
d, ok := assign.Compare(want, sent, trunk[module].FP2, rules[module][node])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
keep[key] = true
|
||||
if before, err := delivery.DecodeDifference(written[key]); err == nil {
|
||||
before.V = 0
|
||||
if delivery.SameDifference(before, d) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.put(ctx, delivery.AssignerDifferencesBucket, key, func() ([]byte, error) { return delivery.EncodeDifference(d) })
|
||||
}
|
||||
for key := range written {
|
||||
if keep[key] {
|
||||
continue
|
||||
}
|
||||
if err := p.kvs[delivery.AssignerDifferencesBucket].Delete(ctx, key); err != nil {
|
||||
p.logf("the difference %s that ended could not be deleted: %v", key, err)
|
||||
continue
|
||||
}
|
||||
p.w.put(delivery.AssignerDifferencesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
|
||||
// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller
|
||||
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
|
||||
func (p *passer) wants(ctx context.Context) {
|
||||
// the hold of a fingerprint whose health check failed. It returns, by module then node, the rule that holds a node
|
||||
// back, for the comparison.
|
||||
func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding {
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
@@ -126,10 +109,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers()))
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
rules := map[string]map[string]assign.Holding{}
|
||||
names := map[string]bool{}
|
||||
for name := range trunk {
|
||||
names[name] = true
|
||||
@@ -137,72 +117,20 @@ func (p *passer) wants(ctx context.Context) {
|
||||
for name := range wanted {
|
||||
names[name] = true
|
||||
}
|
||||
neither := map[string]neitherOne{}
|
||||
// The first pass after a start remembers nothing a previous process said: a neither or held condition it raised would stay
|
||||
// raised for ever once the module has its want or is no longer held. That pass clears, once, the neither condition of every
|
||||
// assigned module not on neither list and the held condition of every module not held; a clear of a condition that is not open is
|
||||
// quiet at the router.
|
||||
first := p.neither == nil
|
||||
stale := map[string]neitherOne{}
|
||||
left := map[string]string{}
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Self: func(node string) bool { return placed[node] },
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if placed[node] {
|
||||
// The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want
|
||||
// names the build alone, and the node judges the move itself.
|
||||
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither, Words: "the node composes its own"}, true
|
||||
}
|
||||
return p.w.composed(node, name, fp)
|
||||
},
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||
},
|
||||
Restored: func(node string) bool {
|
||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||
return err == nil && p.answeredWith(key, assign.OptionRestore)
|
||||
},
|
||||
Allowed: func(node, digest string) bool {
|
||||
key, err := delivery.RecordAskKey(node, name, digest)
|
||||
return err == nil && p.answeredWith(key, assign.OptionYes)
|
||||
}}
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name]}
|
||||
if fp != "" {
|
||||
held, _ := delivery.HeldKey(name, fp)
|
||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||
hold, _ := delivery.HoldAskKey(name, fp)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Kept = p.answeredWith(hold, "keep")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||
}
|
||||
}
|
||||
if m.Released {
|
||||
p.clearHeld(name, fp)
|
||||
}
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held), p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
}
|
||||
last := map[string]string{}
|
||||
for _, node := range m.Nodes {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
p.yeses(ctx, name, m, wanted[name], &out)
|
||||
gatherNeither(neither, name, out)
|
||||
leftNeither(left, m, wanted[name], out)
|
||||
p.sayNoWant(name, m.Nodes, wanted[name], out)
|
||||
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
|
||||
if first {
|
||||
startClears(stale, m, out)
|
||||
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||
p.clearHeld(name, fp)
|
||||
}
|
||||
}
|
||||
rules[name] = out.Rules
|
||||
if out.Held != nil {
|
||||
key, err := delivery.HeldKey(name, fp)
|
||||
if err == nil {
|
||||
@@ -210,23 +138,10 @@ func (p *passer) wants(ctx context.Context) {
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
|
||||
}
|
||||
}
|
||||
if out.Lifted {
|
||||
p.lift(ctx, name, fp)
|
||||
}
|
||||
if out.Held != nil || (m.Held && !out.Lifted) {
|
||||
p.holdAsk(ctx, name, fp, out.Held)
|
||||
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
|
||||
}
|
||||
for _, node := range out.Restore {
|
||||
p.restoreAsk(ctx, name, node, fp)
|
||||
}
|
||||
for _, need := range out.Record {
|
||||
p.recordAsk(ctx, name, fp, need)
|
||||
if out.Held != nil || m.Held {
|
||||
p.holdAsk(ctx, name, fp)
|
||||
}
|
||||
for node, want := range out.Wants {
|
||||
if frozen[node+"."+name] {
|
||||
continue // one source alone names it (ADR 0549 §5): no want, and the one written before stays
|
||||
}
|
||||
key, err := delivery.WantKey(node, name)
|
||||
if err != nil {
|
||||
p.logf("no want of %s on %s is written: %v", name, node, err)
|
||||
@@ -236,216 +151,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
|
||||
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
|
||||
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
|
||||
func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) {
|
||||
if p.noWant == nil {
|
||||
p.noWant = map[string]string{}
|
||||
}
|
||||
assigned := map[string]bool{}
|
||||
for _, node := range nodes {
|
||||
assigned[node] = true
|
||||
key := node + "." + module
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
reason := ""
|
||||
if h, held := out.Rules[node]; held && (!written || w.Removed) {
|
||||
reason = h.Reason()
|
||||
}
|
||||
last, said := p.noWant[key]
|
||||
switch {
|
||||
case reason != "" && reason != last:
|
||||
p.logf("%s on %s gets no want: %s", module, node, reason)
|
||||
p.noWant[key] = reason
|
||||
case reason == "" && said && written && !w.Removed:
|
||||
p.logf("%s on %s has a want", module, node)
|
||||
delete(p.noWant, key)
|
||||
}
|
||||
}
|
||||
for key := range p.noWant {
|
||||
if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] {
|
||||
delete(p.noWant, key) // no longer assigned there: its removed want says so
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
|
||||
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
|
||||
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
|
||||
// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the
|
||||
// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to
|
||||
// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass.
|
||||
func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) {
|
||||
if p.nodeFailed == nil {
|
||||
p.nodeFailed = map[string]string{}
|
||||
}
|
||||
look := map[string]bool{}
|
||||
for _, node := range nodes {
|
||||
look[node] = true
|
||||
}
|
||||
for key := range p.nodeFailed {
|
||||
if node, m, _ := strings.Cut(key, "."); m == module {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
if first {
|
||||
for node := range known.Online {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(look)) {
|
||||
key := node + "." + module
|
||||
f, failing := out.NodeFailed[node]
|
||||
last, said := p.nodeFailed[key]
|
||||
switch {
|
||||
case failing && f.Words+"|"+strings.Join(f.Others, ",") != last:
|
||||
p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+
|
||||
"is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node)
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil {
|
||||
p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",")
|
||||
case !failing && (said || first):
|
||||
if said {
|
||||
p.logf("%s on %s no longer fails its health check alone", module, node)
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said {
|
||||
p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
delete(p.nodeFailed, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||
type neitherOne struct {
|
||||
node, module string
|
||||
holding assign.Holding
|
||||
}
|
||||
|
||||
// gatherNeither adds the nodes on which module is on neither list in its outcome, keyed by node and module.
|
||||
func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome) {
|
||||
for node, h := range out.Neither {
|
||||
into[node+"."+module] = neitherOne{node: node, module: module, holding: h}
|
||||
}
|
||||
}
|
||||
|
||||
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
|
||||
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
|
||||
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
|
||||
// left it with none).
|
||||
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if _, on := out.Neither[node]; on {
|
||||
continue
|
||||
}
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
why := ""
|
||||
h, ruled := out.Rules[node]
|
||||
switch {
|
||||
case written && !w.Removed:
|
||||
case (m.Held && !out.Lifted) || out.Held != nil:
|
||||
why = "it is held, and its held condition says so"
|
||||
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
|
||||
why = "its move waits for the operator's yes on its record ask"
|
||||
case putOn(m, node):
|
||||
why = "its put waits on its own steps and the operator's answer"
|
||||
case m.FP == "":
|
||||
why = "the trunk has no build of it"
|
||||
case ruled:
|
||||
why = h.Reason()
|
||||
default:
|
||||
why = "the node does not run it"
|
||||
}
|
||||
into[node+"."+m.Name] = why
|
||||
}
|
||||
}
|
||||
|
||||
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
|
||||
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
|
||||
func putOn(m assign.Module, node string) bool {
|
||||
p, put := m.Puts[node]
|
||||
_, owns := assign.PutOwns(p)
|
||||
return put && owns
|
||||
}
|
||||
|
||||
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||
// for ever) or a move waiting for its record ask. Clearing a condition that is not open is quiet at the router.
|
||||
func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if _, on := out.Neither[node]; !on {
|
||||
into[node+"."+m.Name] = neitherOne{node: node, module: m.Name}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayNeither raises, once, the condition of every module on neither list in this pass and writes it to the journal, and clears,
|
||||
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||
// each is cleared once.
|
||||
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
|
||||
// it does not hold is no longer assigned.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
|
||||
if p.neither == nil {
|
||||
p.neither = map[string]neitherOne{}
|
||||
for key, one := range stale {
|
||||
if _, on := now[key]; !on {
|
||||
p.neither[key] = one
|
||||
}
|
||||
}
|
||||
}
|
||||
for key, one := range now {
|
||||
if _, said := p.neither[key]; said {
|
||||
continue
|
||||
}
|
||||
c := assign.NeitherCondition(one.node, one.module, one.holding, true)
|
||||
if p.say != nil {
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be said, and is said at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Reason())
|
||||
p.neither[key] = one
|
||||
}
|
||||
for key, one := range p.neither {
|
||||
if _, still := now[key]; still {
|
||||
continue
|
||||
}
|
||||
why, assigned := left[key]
|
||||
if !assigned {
|
||||
why = "it is no longer assigned there"
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if why == "" {
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
} else {
|
||||
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
|
||||
}
|
||||
delete(p.neither, key)
|
||||
}
|
||||
return rules
|
||||
}
|
||||
|
||||
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
||||
@@ -516,393 +222,11 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
|
||||
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
|
||||
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
|
||||
// node (wants are only compared, ADR 0429 §7), and the options say so.
|
||||
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) {
|
||||
func (p *passer) holdAsk(ctx context.Context, module, fp string) {
|
||||
key, err := delivery.HoldAskKey(module, fp)
|
||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||
return
|
||||
}
|
||||
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
|
||||
// an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906).
|
||||
now := p.clock()
|
||||
asking := int(now.Unix())
|
||||
h := p.heldHold(module, fp, fresh)
|
||||
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
|
||||
a := assign.HoldAsk(key, module, fp, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
|
||||
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
|
||||
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
|
||||
func (p *passer) lift(ctx context.Context, module, fp string) {
|
||||
held, err := delivery.HeldKey(module, fp)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ask, err := delivery.HoldAskKey(module, fp)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c := assign.LiftedCondition(module, fp)
|
||||
if !p.said[c.Key+".lifted"] {
|
||||
if p.say != nil {
|
||||
if err := p.say(c); err != nil {
|
||||
// The hold stays written, so the next pass lifts it again and says so then.
|
||||
p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.logf("%s", assign.LiftedWords(module, fp))
|
||||
p.said[c.Key+".lifted"] = true
|
||||
delete(p.said, c.Key+".raised") // a hold written again is said again
|
||||
}
|
||||
p.remove(ctx, delivery.AssignerWantsBucket, held)
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, ask)
|
||||
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
|
||||
// set back from it (issue 1906, note 28).
|
||||
for key := range p.w.read(delivery.AssignerAsksBucket) {
|
||||
if k, err := delivery.ParseAnyAskKey(key); err == nil && k.Kind == delivery.AskKindRestore && k.Module == module && k.FP == fp {
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// staleAsk says whether the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
|
||||
// behind: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, or a composition the
|
||||
// controller no longer composes for the node at that fingerprint. A backup ask and an ask of another kind are never stale here.
|
||||
// A module of the trunk whose fingerprint is not read yet, and a composition not read yet, are not stale: nothing known is
|
||||
// nothing taken away.
|
||||
func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
|
||||
k, err := delivery.ParseAnyAskKey(key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
switch k.Kind {
|
||||
case delivery.AskKindHold, delivery.AskKindRestore:
|
||||
m, on := trunk[k.Module]
|
||||
return !on || (m.FP2 != "" && m.FP2 != k.FP)
|
||||
case delivery.AskKindRecord:
|
||||
m, on := trunk[k.Module]
|
||||
if !on {
|
||||
return true
|
||||
}
|
||||
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
|
||||
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
|
||||
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
|
||||
// trunk's, its want's, its have's), and only when every one of them is read.
|
||||
for _, fp := range p.fingerprintsOf(k.Node, k.Module, m.FP2) {
|
||||
d, ok := p.w.composed(k.Node, k.Module, fp)
|
||||
if !ok || d.Digest == k.Composed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p.fingerprintsOf(k.Node, k.Module, m.FP2)) > 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fingerprintsOf are the fingerprints a node may still want of module: the trunk's, the one its want names and the one its have
|
||||
// names, each once.
|
||||
func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
|
||||
var out []string
|
||||
add := func(fp string) {
|
||||
if fp != "" && !slices.Contains(out, fp) {
|
||||
out = append(out, fp)
|
||||
}
|
||||
}
|
||||
add(trunkFP)
|
||||
if key, err := delivery.WantKey(node, module); err == nil {
|
||||
if w, err := delivery.DecodeWant(p.w.value(delivery.AssignerWantsBucket, key)); err == nil {
|
||||
add(w.FP)
|
||||
}
|
||||
}
|
||||
if key, err := delivery.HaveKey(node, module); err == nil {
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil {
|
||||
add(h.FP)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// staleAsks takes away, at every pass, each hold, restore and record ask that staleAsk names (issue 1906, note 28): an ask
|
||||
// outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
|
||||
// read at all (no module), nothing is taken away.
|
||||
func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module) {
|
||||
if len(trunk) == 0 {
|
||||
return
|
||||
}
|
||||
for key := range p.w.read(delivery.AssignerAsksBucket) {
|
||||
if p.staleAsk(key, trunk) {
|
||||
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
|
||||
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
|
||||
k, kerr := delivery.ParseAnyAskKey(key)
|
||||
held := kerr == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore)
|
||||
if held && !p.clearHeld(k.Module, k.FP) {
|
||||
continue
|
||||
}
|
||||
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, key)
|
||||
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
|
||||
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
|
||||
// raised again if the trunk returns to the fingerprint.
|
||||
if held {
|
||||
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
|
||||
// pass.
|
||||
func (p *passer) remove(ctx context.Context, bucket, key string) {
|
||||
if !p.w.has(bucket, key) {
|
||||
return
|
||||
}
|
||||
if err := p.kvs[bucket].Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("%s %s could not be taken away: %v", bucket, key, err)
|
||||
return
|
||||
}
|
||||
p.w.put(bucket, key, nil)
|
||||
}
|
||||
|
||||
// heldHold is the hold of fp as written: the node whose health check failed and its words.
|
||||
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
|
||||
if fresh != nil {
|
||||
return *fresh
|
||||
}
|
||||
key, err := delivery.HeldKey(module, fp)
|
||||
if err != nil {
|
||||
return delivery.Held{}
|
||||
}
|
||||
h, _ := delivery.DecodeHeld(p.w.read(delivery.AssignerWantsBucket)[key])
|
||||
return h
|
||||
}
|
||||
|
||||
// heldCondition raises, once, the condition for a fingerprint the assigner holds (ADR 0429 §6, ADR 0539 §19): the module, the
|
||||
// node whose health check failed and the check's words. A module holding data needs the operator's answer to its restore ask.
|
||||
func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *delivery.Held, data bool) {
|
||||
h := p.heldHold(module, fp, fresh)
|
||||
c := assign.HeldCondition(module, fp, h.Node, h.Words, true, data)
|
||||
if p.say == nil || p.said[c.Key+".raised"] {
|
||||
return
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the held %s could not be said, and is said at the next pass: %v", module, err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key+".raised"] = true
|
||||
delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again
|
||||
delete(p.said, c.Key+".cleared") // and released again is cleared again
|
||||
}
|
||||
|
||||
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
|
||||
func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
|
||||
key, err := delivery.RestoreAskKey(node, module, fp)
|
||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||
return
|
||||
}
|
||||
words := p.heldHold(module, fp, nil).Words
|
||||
a := assign.RestoreAsk(key, node, module, fp, words, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
|
||||
// answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only
|
||||
// for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of
|
||||
// the same fingerprint, does not carry over to a later one (issue 1906).
|
||||
func (p *passer) answeredWith(key, option string) bool {
|
||||
a, ok := p.w.answer(key)
|
||||
if !ok || a.Option != option {
|
||||
return false
|
||||
}
|
||||
ask, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
id, _, _ := strings.Cut(a.Warrant, "@")
|
||||
return id == ask.Ask.ID
|
||||
}
|
||||
|
||||
// clearHeld clears, once, the condition of a hold the operator released.
|
||||
// It says whether nothing is left to say: the clear was said now or before, or there is no router to say it to.
|
||||
func (p *passer) clearHeld(module, fp string) bool {
|
||||
c := assign.HeldCondition(module, fp, "", "", false, false)
|
||||
if p.say == nil || p.said[c.Key+".cleared"] {
|
||||
return true
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||
return false
|
||||
}
|
||||
p.said[c.Key+".cleared"] = true
|
||||
return true
|
||||
}
|
||||
|
||||
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
|
||||
func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) {
|
||||
key, err := delivery.RecordAskKey(need.Node, module, need.Digest)
|
||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||
return
|
||||
}
|
||||
a := assign.RecordAsk(key, need.Node, module, fp, need, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// backups sees to the modules held on a node because the backup before an apply failed (the have's phase held, ADR 0539 §16):
|
||||
// it asks whether to back up again or go on without a backup, raises the condition naming node, module and the backup's words,
|
||||
// and writes the answer into the node's want for the module, which the node agent reads from it and from nothing else.
|
||||
//
|
||||
// The ask is bound to the composition (it names the digest, and a new revision at the same fingerprint is asked again under a
|
||||
// new number) and an answer counts only if the router's warrant names the id of the ask now at the key (the warrant's first part,
|
||||
// before the `@`). Each answer is another want value (the asking's number is in the want's words), so a backup that fails again
|
||||
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
|
||||
// in a new have value after the answer was written.
|
||||
func (p *passer) backups(ctx context.Context) {
|
||||
wanted := p.w.wantsOf()
|
||||
for key, value := range p.w.read(delivery.HaveBucket) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
have, err := delivery.DecodeHave(value)
|
||||
if err != nil || have.Phase != delivery.PhaseHeld {
|
||||
continue
|
||||
}
|
||||
want, ok := wanted[module][node]
|
||||
if !ok || want.Removed {
|
||||
continue
|
||||
}
|
||||
akey, err := delivery.BackupAskKey(node, module, want.FP)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
current, there := delivery.DecodeAssignerAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey])
|
||||
if there != nil || assign.BackupAskDigest(current) != want.Composed {
|
||||
after := -1
|
||||
if there == nil {
|
||||
after = current.Asking
|
||||
}
|
||||
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(after))
|
||||
continue
|
||||
}
|
||||
answer, answered := p.w.answer(akey)
|
||||
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != current.Ask.ID ||
|
||||
(answer.Option != delivery.BackupAgain && answer.Option != delivery.BackupSkip) {
|
||||
continue // not answered yet: the ask stands
|
||||
}
|
||||
words := assign.BackupAnswerWords(answer.Option, current.Asking)
|
||||
haveChanges := p.w.changed(delivery.HaveBucket, key)
|
||||
if want.Backup != answer.Option || want.Words != words {
|
||||
want.Backup, want.Words = answer.Option, words
|
||||
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||
w := want
|
||||
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) })
|
||||
p.answeredAt[current.Ask.ID] = haveChanges
|
||||
}
|
||||
continue
|
||||
}
|
||||
if answer.Option != delivery.BackupAgain {
|
||||
continue
|
||||
}
|
||||
// The answer is in the want. The node retried and the backup failed again when the have took another value since.
|
||||
if seen, known := p.answeredAt[current.Ask.ID]; !known {
|
||||
p.answeredAt[current.Ask.ID] = haveChanges // after a restart: wait for the next failure
|
||||
} else if haveChanges > seen {
|
||||
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(current.Asking))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once.
|
||||
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want delivery.Want, words string, asking int) {
|
||||
a := assign.BackupAsk(key, node, module, want.FP, want.Composed, words, asking, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
c := assign.BackupCondition(node, module, want.FP, words, true)
|
||||
if p.say == nil || p.said[c.Key] {
|
||||
return
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the failed backup of %s on %s could not be said: %v", module, node, err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
// sayPlacement raises, once, the condition of every pair the controller's assignments and the catalogue's placement do not
|
||||
// both name, clears it once they agree, and answers those pairs by node and module (ADR 0549 §5): the pass writes no want
|
||||
// for them and keeps the one written before. A say that fails is made again at the next pass.
|
||||
func (p *passer) sayPlacement(differ []assign.Pair) map[string]bool {
|
||||
frozen := map[string]bool{}
|
||||
now := map[string]bool{}
|
||||
for _, d := range differ {
|
||||
frozen[d.Node+"."+d.Module] = true
|
||||
c := assign.PlacementCondition(d, true)
|
||||
now[c.Key] = true
|
||||
if p.said[c.Key+".raised"] {
|
||||
continue
|
||||
}
|
||||
p.logf("%s on %s is named by %s alone: no want is written for it there until both sources name it (ADR 0549 §5)", d.Module, d.Node, d.Only)
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be said, and is said at the next pass: %v", d.Module, d.Node, err)
|
||||
continue
|
||||
}
|
||||
p.said[c.Key+".raised"] = true
|
||||
}
|
||||
prefix := delivery.AssignerModule + ".placement."
|
||||
for key := range p.said {
|
||||
c, raised := strings.CutSuffix(key, ".raised")
|
||||
if !raised || !strings.HasPrefix(c, prefix) || now[c] {
|
||||
continue
|
||||
}
|
||||
node, module, _ := strings.Cut(strings.TrimPrefix(c, prefix), ".")
|
||||
if err := p.say(assign.PlacementCondition(assign.Pair{Node: node, Module: module}, false)); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
delete(p.said, key)
|
||||
}
|
||||
return frozen
|
||||
}
|
||||
|
||||
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
|
||||
// person must allow (ADR 0547 §3): for a have of phase
|
||||
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
|
||||
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again.
|
||||
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if m.Self == nil || !m.Self(node) {
|
||||
continue
|
||||
}
|
||||
was, had := before[node]
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
if !had || was.Removed {
|
||||
continue
|
||||
}
|
||||
w = was
|
||||
}
|
||||
if w.Removed {
|
||||
continue
|
||||
}
|
||||
yes := ""
|
||||
if had && was.FP == w.FP {
|
||||
yes = was.Yes
|
||||
}
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
|
||||
if m.Allowed != nil && m.Allowed(node, h.Waits) {
|
||||
yes = h.Waits
|
||||
} else {
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
}
|
||||
}
|
||||
// A want carried over from before the node composed its own may still name the controller's composition; a placed
|
||||
// node's want names the build alone (ADR 0547 §2).
|
||||
if yes != w.Yes || w.Composed != "" || !w.Self {
|
||||
w.Yes, w.Composed, w.Self = yes, "", true
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,946 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// memKV is the part of a bucket a pass writes: puts and deletes.
|
||||
type memKV struct {
|
||||
jetstream.KeyValue
|
||||
values map[string][]byte
|
||||
}
|
||||
|
||||
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
m.values[key] = value
|
||||
return uint64(len(m.values)), nil
|
||||
}
|
||||
|
||||
func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||
delete(m.values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
|
||||
func newPasser(kvs buckets, at time.Time) *passer {
|
||||
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
|
||||
now: func() time.Time { return at }}
|
||||
}
|
||||
|
||||
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
|
||||
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
||||
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
|
||||
// an id it keeps).
|
||||
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||
held := &delivery.Held{Node: "ace", Words: "health failed"}
|
||||
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
|
||||
first.holdAsk(context.Background(), "web", fpTest, held)
|
||||
a1, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.lift(context.Background(), "web", fpTest)
|
||||
if _, there := asksKV.values[key]; there {
|
||||
t.Fatal("the lift left the hold ask")
|
||||
}
|
||||
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
|
||||
again.holdAsk(context.Background(), "web", fpTest, held)
|
||||
a2, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a2.Ask.ID == a1.Ask.ID {
|
||||
t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
|
||||
// taken away, is not taken for the hold that stands now.
|
||||
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
put := func(asking int) string {
|
||||
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
|
||||
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
return a.Ask.ID
|
||||
}
|
||||
first := put(100)
|
||||
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
|
||||
p.w.put(delivery.AnswersBucket, key, v)
|
||||
if !p.answeredWith(key, "release") {
|
||||
t.Fatal("the answer to the ask that stands is not taken")
|
||||
}
|
||||
if p.answeredWith(key, "keep") {
|
||||
t.Error("an answer was taken for an option not chosen")
|
||||
}
|
||||
put(200) // the same fingerprint held again, after a lift
|
||||
if p.answeredWith(key, "release") {
|
||||
t.Error("the answer to an earlier hold carried over to a later one")
|
||||
}
|
||||
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
|
||||
if p.answeredWith(key, "release") {
|
||||
t.Error("an answer to an ask taken away was taken")
|
||||
}
|
||||
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
|
||||
}
|
||||
|
||||
// memEntry is a value a memKV holds, as a bucket's Get answers it.
|
||||
type memEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
value []byte
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (e memEntry) Value() []byte { return e.value }
|
||||
func (e memEntry) Revision() uint64 { return e.revision }
|
||||
func (m *memKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||
v, ok := m.values[key]
|
||||
if !ok {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
return memEntry{value: v, revision: uint64(len(v))}, nil
|
||||
}
|
||||
func (m *memKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if uint64(len(m.values[key])) != revision {
|
||||
return 0, errors.New("wrong last sequence")
|
||||
}
|
||||
m.values[key] = value
|
||||
return uint64(len(value)), nil
|
||||
}
|
||||
|
||||
// memStore is the runtime's states for the put's tools, over the same memKVs a pass writes.
|
||||
type memStore map[string]*memKV
|
||||
|
||||
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := s[state].values[key]
|
||||
return v, uint64(len(v)), ok, nil
|
||||
}
|
||||
func (s memStore) put(state, key string, value []byte) error {
|
||||
s[state].values[key] = value
|
||||
return nil
|
||||
}
|
||||
func (s memStore) keys(state string) ([]string, error) {
|
||||
var out []string
|
||||
for k := range s[state].values {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// fakeBus is the assigner's process account in a test: every call recorded, each answered at once.
|
||||
type fakeBus struct {
|
||||
calls chan string
|
||||
events []string
|
||||
}
|
||||
|
||||
func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
|
||||
b.calls <- fmt.Sprintf("%s %v", key, body)
|
||||
return json.RawMessage(`{"ok":true,"output":""}`), nil
|
||||
}
|
||||
func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
return b.Call(node+"/"+key, body)
|
||||
}
|
||||
func (b *fakeBus) Emit(eventType string, body any) error {
|
||||
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ADR 0546, the first put (2026-10-25, want.novox.nats): the tool writes the put; a pass waits for the composition and grant
|
||||
// the controller's take-in writes; once they are there it asks the operator with a yes that approves; only on that warrant does it have
|
||||
// the streams snapshotted, then writes the want held and records the warrant; the have running it healthy holds it; a release
|
||||
// gives the want back to the rules. Each step is said as the event put.
|
||||
func putOfTheBus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
||||
fp := delivery.SourcePrefix2 + "fd497ab8" + strings.Repeat("0", 56)
|
||||
digest := "sha256:" + strings.Repeat("d", 64)
|
||||
mem := func() *memKV { return &memKV{values: map[string][]byte{}} }
|
||||
puts, asksKV, wantsKV := mem(), mem(), mem()
|
||||
kvs := buckets{delivery.AssignerPutsBucket: puts, delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
||||
p := newPasser(kvs, now)
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
|
||||
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
|
||||
p.acts.snapshot = func(context.Context, string, string) (string, error) {
|
||||
return "novox's restore point of nats (streams) taken 22:01", nil
|
||||
}
|
||||
// The world: nats assigned to novox, built at fp, the bus by its manifest, novox running it from before the switch.
|
||||
world := func(bucket, key string, v any) {
|
||||
var data []byte
|
||||
switch x := v.(type) {
|
||||
case []byte:
|
||||
data = x
|
||||
default:
|
||||
data, _ = json.Marshal(x)
|
||||
}
|
||||
p.w.put(bucket, key, data)
|
||||
}
|
||||
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
world(delivery.ModulesBucket, "nats", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
|
||||
bkey, _ := delivery.BuilderBuildKey("nats", fp)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mdigest})
|
||||
world(delivery.BuilderBuildsBucket, bkey, built)
|
||||
hkey, _ := delivery.HaveKey("novox", "nats")
|
||||
before, _ := delivery.EncodeHave(delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
world(delivery.HaveBucket, hkey, before)
|
||||
|
||||
builds := mem()
|
||||
builds.values[bkey] = built
|
||||
modules := mem()
|
||||
modules.values["nats"] = mod
|
||||
s := newPutTools(memStore{putsState: puts, delivery.BuilderModule + ".builds": builds, delivery.IndexerModule + ".modules": modules})
|
||||
s.now = func() time.Time { return now }
|
||||
was := answerWithin
|
||||
answerWithin = 0
|
||||
defer func() { answerWithin = was }()
|
||||
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil {
|
||||
t.Error("a prefix of a fingerprint the builder never built was taken")
|
||||
}
|
||||
if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp {
|
||||
t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err)
|
||||
}
|
||||
if got, err := s.version("nats", "newest"); err != nil || got != fp {
|
||||
t.Errorf("newest reads %q, %v", got, err)
|
||||
}
|
||||
ans, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "the bus's user list"}, "shanks.node-tools")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := ans.(putAnswer); a.Step != delivery.PutAsked || !a.Hold || !strings.Contains(a.Follow, "mesh_wait") {
|
||||
t.Fatalf("put answered %+v", a)
|
||||
}
|
||||
pkey, _ := delivery.PutKey("novox", "nats")
|
||||
step := func() delivery.Put {
|
||||
t.Helper()
|
||||
world(delivery.AssignerPutsBucket, pkey, puts.values[pkey])
|
||||
p.pass(ctx)
|
||||
got, err := delivery.DecodePut(puts.values[pkey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
if got := step(); got.Step != delivery.PutCompose || !strings.Contains(got.Words, "no composition") {
|
||||
t.Fatalf("the first pass left the put at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
select {
|
||||
case call := <-bus.calls:
|
||||
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
|
||||
default:
|
||||
}
|
||||
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
|
||||
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
|
||||
world(delivery.ControllerComposedBucket, ckey, cd)
|
||||
gkey, _ := delivery.ControllerGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
world(delivery.ControllerGrantsBucket, gkey, g)
|
||||
got := step()
|
||||
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
|
||||
if got.Step != delivery.PutPerson || got.Ask != akey {
|
||||
t.Fatalf("composed, the put stands at %s (%s), ask %q", got.Step, got.Words, got.Ask)
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil || a.Ask.Options[0].Level != asks.Approve {
|
||||
t.Fatalf("the ask is %+v, %v; want a yes that approves", a, err)
|
||||
}
|
||||
if _, there := wantsKV.values["want.novox.nats"]; there {
|
||||
t.Fatal("the bus's want was written before the operator's warrant")
|
||||
}
|
||||
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-25T22:01:00Z", Words: "the operator, verified"})
|
||||
world(delivery.AnswersBucket, akey, yes)
|
||||
if got := step(); got.Step != delivery.PutSnapshot {
|
||||
t.Fatalf("on the warrant the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
<-woke
|
||||
got = step()
|
||||
w, err := delivery.DecodeWant(wantsKV.values["want.novox.nats"])
|
||||
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
|
||||
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||
}
|
||||
if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) {
|
||||
t.Errorf("the warrant was recorded as %q", call)
|
||||
}
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
|
||||
running, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
world(delivery.HaveBucket, hkey, running)
|
||||
if got := step(); got.Step != delivery.PutHolding {
|
||||
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "done"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := step(); got.Step != delivery.PutReleased {
|
||||
t.Fatalf("released, the put stands at %s", got.Step)
|
||||
}
|
||||
listed, _ := s.doPuts(map[string]any{"module": "nats"})
|
||||
if l := listed.(map[string]any)["puts"].([]putAnswer); len(l) != 1 || l[0].Step != delivery.PutReleased || len(l[0].Steps) < 6 {
|
||||
t.Errorf("puts lists %+v", l)
|
||||
}
|
||||
if len(bus.events) < 5 || !strings.Contains(strings.Join(bus.events, "\n"), "holding") {
|
||||
t.Errorf("the steps said as events: %v", bus.events)
|
||||
}
|
||||
// The same bus put again at the same composition after the release is its own asking: its ask is written again under an
|
||||
// id of its own, and the earlier yes, still at the answers' key, does not count for it (ADR 0546 clause 8).
|
||||
world(delivery.HaveBucket, hkey, before)
|
||||
s.now = func() time.Time { return now.Add(time.Minute) }
|
||||
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "again"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := step()
|
||||
a2, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
|
||||
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
|
||||
}
|
||||
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes2, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a2.Ask.ID + "@2026-10-25T22:03:00Z", Words: "the operator, verified"})
|
||||
world(delivery.AnswersBucket, akey, yes2)
|
||||
if got := step(); got.Step != delivery.PutSnapshot {
|
||||
t.Fatalf("on the second warrant the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
<-woke
|
||||
// Released between the pass's read and its write: the pass writes no want for the put that no longer stands.
|
||||
oldWant, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64)})
|
||||
wantsKV.values["want.novox.nats"] = oldWant
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", oldWant)
|
||||
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "changed my mind"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.pass(ctx)
|
||||
if string(wantsKV.values["want.novox.nats"]) != string(oldWant) {
|
||||
t.Errorf("a want was written for a put released meanwhile: %s", wantsKV.values["want.novox.nats"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it
|
||||
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
|
||||
aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t)
|
||||
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
failing := true
|
||||
p.say = func(c asks.Condition) error {
|
||||
if failing {
|
||||
failing = false
|
||||
return errors.New("the channel is away")
|
||||
}
|
||||
said = append(said, c)
|
||||
return nil
|
||||
}
|
||||
var logged []string
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||
p.sayNeither(one, nil, nil) // the say fails
|
||||
p.sayNeither(one, nil, nil)
|
||||
p.sayNeither(one, nil, nil)
|
||||
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||
}
|
||||
wants := map[string]string{"g14.screen-lock": ""}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
|
||||
t.Errorf("the clear said %q", last)
|
||||
}
|
||||
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
|
||||
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
|
||||
p.sayNeither(bus, nil, nil)
|
||||
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
|
||||
t.Errorf("the bus's neither condition = %+v", c)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
|
||||
if c := said[len(said)-1]; c.State != asks.StateCleared || strings.Contains(c.Resolved, "has a want") {
|
||||
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
|
||||
t.Errorf("the clear without a want said %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
got := map[string]neitherOne{}
|
||||
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
|
||||
gatherNeither(got, "rofi", assign.Outcome{})
|
||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||
t.Errorf("gathered = %+v", got)
|
||||
}
|
||||
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
|
||||
// through its steps owns the want, so it is named, never "the node does not run it".
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
puts map[string]delivery.Put
|
||||
before map[string]delivery.Want
|
||||
why string
|
||||
}{
|
||||
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
|
||||
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
|
||||
{"nothing", nil, nil, "the node does not run it"},
|
||||
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
|
||||
} {
|
||||
left := map[string]string{}
|
||||
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
|
||||
if why, there := left["novox.nats"]; !there || why != c.why {
|
||||
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
|
||||
}
|
||||
}
|
||||
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
|
||||
// when it has a want again: one line per change, not per pass.
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
|
||||
var logged []string
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
|
||||
wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
|
||||
for i, c := range []struct {
|
||||
before map[string]delivery.Want
|
||||
out assign.Outcome
|
||||
line string // the line said in this pass, "" for none
|
||||
}{
|
||||
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
|
||||
{nil, notDone, ""},
|
||||
{map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""},
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
|
||||
{nil, wanted, "rofi on shanks has a want"},
|
||||
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
|
||||
} {
|
||||
logged = nil
|
||||
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
|
||||
if got := strings.Join(logged, "\n"); got != c.line {
|
||||
t.Errorf("pass %d said %q, want %q", i, got, c.line)
|
||||
}
|
||||
}
|
||||
// 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition
|
||||
// once, naming the node, the module, F and the have's words, and clears it once when it ends.
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}}
|
||||
for i, c := range []struct {
|
||||
failed map[string]assign.NodeFailure
|
||||
state string // the condition said in this pass, "" for none
|
||||
}{
|
||||
{failing, asks.StateRaised},
|
||||
{failing, ""},
|
||||
{nil, asks.StateCleared},
|
||||
{nil, ""},
|
||||
} {
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false)
|
||||
switch {
|
||||
case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state):
|
||||
t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state)
|
||||
case c.state == asks.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" ||
|
||||
!strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")):
|
||||
t.Errorf("node failed, raised %+v", conds[0])
|
||||
}
|
||||
}
|
||||
// D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every
|
||||
// node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none).
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" {
|
||||
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
|
||||
}
|
||||
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
|
||||
Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
known assign.Nodes
|
||||
keys []string
|
||||
}{
|
||||
{"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
} {
|
||||
conds = nil
|
||||
fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
|
||||
fresh.logf = func(string, ...any) {}
|
||||
fresh.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
|
||||
var keys []string
|
||||
for _, cond := range conds {
|
||||
if cond.State == asks.StateCleared {
|
||||
keys = append(keys, cond.Key)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) {
|
||||
t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
|
||||
// every module that is not held. Clearing a condition that is not open is quiet at the router.
|
||||
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
p.w.put(delivery.HaveBucket, "g14.web", have)
|
||||
key, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
p.w.put(delivery.AssignerWantsBucket, key, want)
|
||||
p.wants(context.Background())
|
||||
cleared := map[string]bool{}
|
||||
for _, c := range said {
|
||||
if c.State != asks.StateCleared {
|
||||
t.Errorf("the first pass raised %+v", c)
|
||||
}
|
||||
cleared[c.Key] = true
|
||||
}
|
||||
neither := assign.NeitherCondition("g14", "web", assign.Holding{}, false).Key
|
||||
held := assign.HeldCondition("web", fpTest, "", "", false, false).Key
|
||||
if !cleared[neither] || !cleared[held] {
|
||||
t.Errorf("cleared = %v, want %s and %s", cleared, neither, held)
|
||||
}
|
||||
n := len(said)
|
||||
p.wants(context.Background())
|
||||
if len(said) != n {
|
||||
t.Errorf("a second pass said again: %+v", said[n:])
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
|
||||
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
|
||||
// raised (the first pass after a start), each once.
|
||||
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
setup := func() (*passer, *[]asks.Condition) {
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
said := &[]asks.Condition{}
|
||||
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "shanks", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
p.w.put(delivery.HaveBucket, "shanks.systemd-resolved", have)
|
||||
hkey, _ := delivery.HeldKey("systemd-resolved", fpTest)
|
||||
held, _ := delivery.EncodeHeld(delivery.Held{Node: "shanks", Words: "health failed"})
|
||||
p.w.put(delivery.AssignerWantsBucket, hkey, held)
|
||||
return p, said
|
||||
}
|
||||
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
|
||||
neithers := func(said []asks.Condition) (raised, cleared int) {
|
||||
for _, c := range said {
|
||||
if c.Key != key {
|
||||
continue
|
||||
}
|
||||
if c.State == asks.StateRaised {
|
||||
raised++
|
||||
} else {
|
||||
cleared++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
// a standing neither this process raised
|
||||
p, said := setup()
|
||||
p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: delivery.RuleNotHeld}}}
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
if r, c := neithers(*said); r != 0 || c != 1 {
|
||||
t.Errorf("standing in this process: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||
}
|
||||
// one a previous process raised: the first pass after a start clears it
|
||||
p, said = setup()
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
if r, c := neithers(*said); r != 0 || c != 1 {
|
||||
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose
|
||||
// fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
|
||||
// backup ask is left; and a lift takes the restore asks of its fingerprint on every node.
|
||||
func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("a", 64), delivery.SourcePrefix2+strings.Repeat("c", 64)
|
||||
digOld, digNew := "sha256:"+strings.Repeat("d", 64), "sha256:"+strings.Repeat("e", 64)
|
||||
trunkOf := func(fp string) map[string]delivery.Module {
|
||||
return map[string]delivery.Module{"web": {FP2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}}
|
||||
}
|
||||
type ask struct{ name, key string }
|
||||
holdOld, _ := delivery.HoldAskKey("web", fpOld)
|
||||
holdNew, _ := delivery.HoldAskKey("web", fpNew)
|
||||
holdGone, _ := delivery.HoldAskKey("gone", fpOld)
|
||||
restOld, _ := delivery.RestoreAskKey("novox", "web", fpOld)
|
||||
restNew, _ := delivery.RestoreAskKey("novox", "web", fpNew)
|
||||
recOld, _ := delivery.RecordAskKey("novox", "web", digOld)
|
||||
recNew, _ := delivery.RecordAskKey("novox", "web", digNew)
|
||||
backOld, _ := delivery.BackupAskKey("novox", "web", fpOld)
|
||||
all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
trunk map[string]delivery.Module
|
||||
gone []string // the keys taken away
|
||||
noDigit bool // the controller's composition is not read
|
||||
heldAt string // the node's want is at this fingerprint, whose composition is digOld
|
||||
}{
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
|
||||
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
|
||||
{"the trunk is not read at all", nil, nil, false, ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
for _, key := range all {
|
||||
k, _ := delivery.ParseAnyAskKey(key)
|
||||
var a delivery.AssignerAsk
|
||||
switch k.Kind {
|
||||
case delivery.AskKindHold:
|
||||
a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now())
|
||||
case delivery.AskKindRestore:
|
||||
a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", time.Now())
|
||||
case delivery.AskKindRecord:
|
||||
a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, time.Now())
|
||||
default:
|
||||
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
|
||||
}
|
||||
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksKV.values[key] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
}
|
||||
if !c.noDigit {
|
||||
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
|
||||
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
p.w.put(delivery.ControllerComposedBucket, ckey, v)
|
||||
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
|
||||
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
if c.heldAt != "" {
|
||||
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
|
||||
}
|
||||
p.w.put(delivery.ControllerComposedBucket, okey, old)
|
||||
}
|
||||
if c.heldAt != "" {
|
||||
wkey, _ := delivery.WantKey("novox", "web")
|
||||
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
|
||||
p.w.put(delivery.AssignerWantsBucket, wkey, w)
|
||||
}
|
||||
heldOld, _ := delivery.HeldKey("web", fpOld)
|
||||
wantsKV.values[heldOld] = []byte("x")
|
||||
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
|
||||
firstAsk, _ := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||
p.staleAsks(context.Background(), c.trunk)
|
||||
// the held record and condition of a fingerprint whose hold ask went are taken away with it
|
||||
heldOld, _ = delivery.HeldKey("web", fpOld)
|
||||
_, heldThere := wantsKV.values[heldOld]
|
||||
cleared := slices.ContainsFunc(said, func(c asks.Condition) bool {
|
||||
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == asks.StateCleared
|
||||
})
|
||||
if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
|
||||
t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
|
||||
}
|
||||
for _, key := range all {
|
||||
_, there := asksKV.values[key]
|
||||
want := !slices.Contains(c.gone, key)
|
||||
if there != want {
|
||||
t.Errorf("%s: left in the bucket = %v, want %v", key, there, want)
|
||||
}
|
||||
}
|
||||
if slices.Contains(c.gone, holdOld) {
|
||||
// the trunk returns to the fingerprint (a revert): the held record still holds it, and it is asked and said again
|
||||
p.now = func() time.Time { return time.Now().Add(time.Hour) }
|
||||
p.holdAsk(context.Background(), "web", fpOld, nil)
|
||||
p.heldCondition(context.Background(), "web", fpOld, nil, false)
|
||||
again, err := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||
raised := slices.ContainsFunc(said, func(c asks.Condition) bool { return c.State == asks.StateRaised })
|
||||
if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised {
|
||||
t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
var said []asks.Condition
|
||||
fail := true
|
||||
p.say = func(c asks.Condition) error {
|
||||
if fail {
|
||||
fail = false
|
||||
return errors.New("router unreachable")
|
||||
}
|
||||
said = append(said, c)
|
||||
return nil
|
||||
}
|
||||
a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now())
|
||||
v, err := delivery.EncodeAssignerAsk(holdOld, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksKV.values[holdOld] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, holdOld, v)
|
||||
p.staleAsks(context.Background(), trunkOf(fpNew))
|
||||
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
|
||||
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
|
||||
}
|
||||
p.staleAsks(context.Background(), trunkOf(fpNew))
|
||||
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != asks.StateCleared {
|
||||
t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
keep, _ := delivery.RestoreAskKey("ace", "web", fpNew)
|
||||
keepOther, _ := delivery.RestoreAskKey("ace", "other", fpOld)
|
||||
for _, key := range []string{restOld, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
|
||||
asksKV.values[key] = []byte("x")
|
||||
p.w.put(delivery.AssignerAsksBucket, key, []byte("x"))
|
||||
}
|
||||
p.lift(context.Background(), "web", fpOld)
|
||||
for key, gone := range map[string]bool{restOld: true, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
|
||||
if _, there := asksKV.values[key]; there == gone {
|
||||
t.Errorf("%s: left in the bucket = %v after the lift", key, there)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func mustKey(key string, err error) string {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// 1906 note 28, review: a node held back at an earlier fingerprint is asked about the revised composition of that fingerprint
|
||||
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
|
||||
// yes the want names the revised digest.
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
|
||||
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
|
||||
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
|
||||
if cerr != nil {
|
||||
t.Fatal(cerr)
|
||||
}
|
||||
put(delivery.ControllerComposedBucket, ckey, cd)
|
||||
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
|
||||
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
|
||||
put(delivery.ControllerComposedBucket, tkey, tcd)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", revised)
|
||||
for i := 0; i < 2; i++ {
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
|
||||
}
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != revised {
|
||||
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
|
||||
}
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Error("the answered record ask was taken away")
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0549 §5): until the third milestone a pair the controller's assignments and the catalogue's placement do not
|
||||
// both name gets no want and one condition, raised once and cleared once both name it; before any placement was read the
|
||||
// assignments are the one source and nothing is raised.
|
||||
func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
assigned := func(node, module string) {
|
||||
raw, _ := json.Marshal(map[string]string{"node": node, "module": module})
|
||||
p.w.put(provider.AssignmentsBucket, node+"."+module, raw)
|
||||
}
|
||||
place := func(node string, modules ...string) {
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
raw, err := delivery.EncodePlacement(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PlacementKey(node)
|
||||
p.w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
assigned("g14", "a")
|
||||
assigned("g14", "b")
|
||||
if got, _ := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "b"}) || len(p.sayPlacement(p.w.placementDiffers())) != 0 || len(said) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v, said %v", got, said)
|
||||
}
|
||||
none := func(context.Context, string) ([]byte, error) { return nil, errNotThere }
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" {
|
||||
t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
place("g14", "a", "c")
|
||||
got, _ := p.w.assigned()
|
||||
frozen := p.sayPlacement(p.w.placementDiffers())
|
||||
p.sayPlacement(p.w.placementDiffers())
|
||||
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
|
||||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
|
||||
!strings.Contains(said[1].Explanation, "the seat mesh-placement's placement names c on g14") {
|
||||
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
|
||||
}
|
||||
place("g14", "a", "b", "c")
|
||||
assigned("g14", "c")
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 || said[2].State != asks.StateCleared || said[3].State != asks.StateCleared {
|
||||
t.Errorf("both agree again: frozen %v, said %+v", frozen, said)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2): placed, the check composes over the input the nodes compose from (frombus), once each placed node
|
||||
// wrote its record; until then it says why and composes the assignments alone.
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") {
|
||||
t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread)
|
||||
}
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}})
|
||||
p.w.put(delivery.HaveBucket, "g14.node", rec)
|
||||
m := p.w.mesh(context.Background(), none)
|
||||
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
|
||||
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
|
||||
// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and
|
||||
// kept there while the want stays at its build.
|
||||
func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
put(delivery.PlacementBucket, "placement.g14", pl)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
sync := func() {
|
||||
for k, v := range wantsKV.values {
|
||||
put(delivery.AssignerWantsBucket, k, v)
|
||||
}
|
||||
}
|
||||
|
||||
p.wants(context.Background())
|
||||
p.nodeWants(context.Background())
|
||||
sync()
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self {
|
||||
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
|
||||
}
|
||||
nkey, _ := delivery.WantNodeKey("g14")
|
||||
if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
|
||||
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
|
||||
}
|
||||
|
||||
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
|
||||
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
|
||||
put(delivery.HaveBucket, "g14.web", waits)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatal("a have waiting for a yes raised no record ask")
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
sync()
|
||||
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
|
||||
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
|
||||
}
|
||||
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
|
||||
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", ran)
|
||||
p.wants(context.Background())
|
||||
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
|
||||
t.Errorf("a pass after the move dropped the yes: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,449 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The puts (novox/hq ADR 0546): the tool `assigner_put` writes the key `<node>.<module>` of the assigner's puts, and each pass
|
||||
// takes every put that has not ended one step further (assign.StepPut), doing the acts the step asks for: the put's ask, the
|
||||
// bus's snapshot, the warranted record. Each act's outcome is heard back as a change of the world (the composition, the grant,
|
||||
// the answer) or, for the calls, as a wake once they answer; nothing polls but the snapshot's read-back of the backup holder,
|
||||
// as the controller's bus step reads it (ADR 0235), bounded. Every step is said as the event mesh-delivery-assigner.put, so a
|
||||
// caller waits on it.
|
||||
|
||||
// PutEvent is the event a put's every step is said as: mesh.mod.mesh-delivery-assigner.event.put.
|
||||
const PutEvent = "put"
|
||||
|
||||
// caller is what the puts need of the bus as the assigner's own process account: a call of a seat's verb, on one node or on
|
||||
// the seat held once, and an event said. process.Conn, or a test's.
|
||||
type caller interface {
|
||||
Call(key string, body any) (json.RawMessage, error)
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
Emit(eventType string, body any) error
|
||||
}
|
||||
|
||||
// putActs keeps, by put (its key and when it was asked), what the acts of a put came to while this process runs: a restart
|
||||
// takes the snapshot again, without harm.
|
||||
type putActs struct {
|
||||
mu sync.Mutex
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
snapping map[string]bool
|
||||
snapped map[string]string
|
||||
snapNo map[string]string
|
||||
warranted map[string]bool
|
||||
warrantNo map[string]string
|
||||
said map[string]string
|
||||
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
|
||||
snapshot func(ctx context.Context, node, module string) (string, error)
|
||||
}
|
||||
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
said: map[string]string{}}
|
||||
a.snapshot = a.snapshotTheBus
|
||||
return a
|
||||
}
|
||||
|
||||
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
|
||||
func identity(key string, p delivery.Put) string {
|
||||
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// putsByModule is every put the assigner holds, by module then node.
|
||||
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
|
||||
out := map[string]map[string]delivery.Put{}
|
||||
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
put, err := delivery.DecodePut(value)
|
||||
if err != nil {
|
||||
p.logf("the put %s cannot be read, and is left: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string]delivery.Put{}
|
||||
}
|
||||
out[module][node] = put
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// puts takes every put that has not ended one step further.
|
||||
func (p *passer) puts(ctx context.Context) {
|
||||
if p.acts == nil {
|
||||
return
|
||||
}
|
||||
all := p.putsByModule()
|
||||
if len(all) == 0 {
|
||||
return
|
||||
}
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
wanted := p.w.wantsOf()
|
||||
modules := make([]string, 0, len(all))
|
||||
for m := range all {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
var manifest catalogue.Manifest
|
||||
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
|
||||
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
|
||||
manifest = got.manifest
|
||||
}
|
||||
}
|
||||
for node, put := range all[module] {
|
||||
if delivery.PutEnded(put.Step) {
|
||||
continue
|
||||
}
|
||||
key, _ := delivery.PutKey(node, module)
|
||||
id := identity(key, put)
|
||||
world := p.putWorld(node, module, put, manifest, assigned[node])
|
||||
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
|
||||
next := assign.StepPut(world)
|
||||
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
|
||||
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
|
||||
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
|
||||
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
|
||||
next.Put.Composed = w.Composed
|
||||
}
|
||||
}
|
||||
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
|
||||
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
|
||||
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
|
||||
}
|
||||
}
|
||||
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
|
||||
!strings.Contains(next.Put.Words, failed) {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
|
||||
}
|
||||
p.actOn(ctx, node, module, id, world, next)
|
||||
written := samePut(put, next.Put) || p.writePut(ctx, key, put, next.Put)
|
||||
if !written {
|
||||
continue // released or put again meanwhile: nothing is done for the put that no longer stands
|
||||
}
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
|
||||
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
|
||||
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||
want := assign.PutWant(next.Put)
|
||||
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
|
||||
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||
}
|
||||
}
|
||||
if next.Warranted {
|
||||
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func normalWant(w delivery.Want) delivery.Want {
|
||||
w.V = 0
|
||||
return w
|
||||
}
|
||||
|
||||
// putWorld is what one step of a put reads from the world.
|
||||
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
|
||||
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
|
||||
for _, m := range assignedHere {
|
||||
if m == module {
|
||||
w.Assigned = true
|
||||
}
|
||||
}
|
||||
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
|
||||
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
|
||||
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
|
||||
w.Build, w.BuildWords = b.State, b.Error
|
||||
} else {
|
||||
w.Build = "unreadable: " + err.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
if policy == "" {
|
||||
policy, _, _ = catalogue.DerivedUpgrade(manifest)
|
||||
}
|
||||
w.Recorded = policy == catalogue.PolicyRecord
|
||||
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
|
||||
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
|
||||
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
w.Answer = assign.OptionYes
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
w.Answer = assign.OptionNo
|
||||
default:
|
||||
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if hkey, err := delivery.HaveKey(node, module); err == nil {
|
||||
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
|
||||
if h, err := delivery.DecodeHave(raw); err == nil {
|
||||
w.Have = &h
|
||||
}
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
|
||||
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
|
||||
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
|
||||
// from their new owner and nothing else changes.
|
||||
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
|
||||
var out *delivery.ComposedDigest
|
||||
if d, ok := p.w.composed(node, module, fp); ok {
|
||||
out = &d
|
||||
}
|
||||
granted := false
|
||||
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
granted = g.NamesNode(node, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, granted
|
||||
}
|
||||
|
||||
// actOn does the acts a step asks for.
|
||||
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
|
||||
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
|
||||
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
|
||||
// raises it anew under its own id.
|
||||
akey := next.Put.Ask
|
||||
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || current.Asking != assign.PutAsking(next.Put) {
|
||||
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
|
||||
if err != nil {
|
||||
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
|
||||
} else {
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
}
|
||||
}
|
||||
}
|
||||
if next.Snapshot {
|
||||
p.acts.snap(ctx, id, node, module)
|
||||
}
|
||||
}
|
||||
|
||||
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
|
||||
func (p *passer) performs(key string, act asks.Act) error {
|
||||
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, o := range a.Ask.Options {
|
||||
if o.ID == assign.OptionYes {
|
||||
return o.Performs(act)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the ask %s offers no yes", key)
|
||||
}
|
||||
|
||||
// samePut says two puts say the same: their step, words and what they found.
|
||||
func samePut(a, b delivery.Put) bool {
|
||||
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
|
||||
}
|
||||
|
||||
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
|
||||
// released it since; and says it as the event put. It says whether it wrote.
|
||||
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
|
||||
kv := p.kvs[delivery.AssignerPutsBucket]
|
||||
if kv == nil {
|
||||
return false
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
|
||||
return false
|
||||
}
|
||||
now, err := delivery.DecodePut(e.Value())
|
||||
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
|
||||
return false // asked again or released meanwhile: the next pass reads that
|
||||
}
|
||||
data, err := delivery.EncodePut(next)
|
||||
if err != nil {
|
||||
p.logf("the put %s is not written: %v", key, err)
|
||||
return false
|
||||
}
|
||||
if _, err := kv.Update(ctx, key, data, e.Revision()); err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
p.logf("the put %s could not be written, and is taken further at the next pass: %v", key, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.put(delivery.AssignerPutsBucket, key, data)
|
||||
p.logf("put %s: %s: %s", key, next.Step, next.Words)
|
||||
p.acts.say(key, next)
|
||||
return true
|
||||
}
|
||||
|
||||
// state is what this process knows of a put's acts.
|
||||
func (a *putActs) state(id string) (snapshot string, snapping bool, snapNo string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.snapped[id], a.snapping[id], a.snapNo[id]
|
||||
}
|
||||
|
||||
// snap has the bus's snapshot taken once for a put, in the background.
|
||||
func (a *putActs) snap(ctx context.Context, id, node, module string) {
|
||||
a.mu.Lock()
|
||||
if a.snapping[id] || a.snapped[id] != "" || a.snapNo[id] != "" {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.snapping[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
defer a.wake()
|
||||
where, err := a.snapshot(ctx, node, module)
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.snapping[id] = false
|
||||
if err != nil {
|
||||
a.snapNo[id] = err.Error()
|
||||
return
|
||||
}
|
||||
a.snapped[id] = where
|
||||
}()
|
||||
}
|
||||
|
||||
// SnapshotWithin is how long the bus node's backup holder is given to take the bus's snapshot (as the controller's bus step).
|
||||
var SnapshotWithin = 15 * time.Minute
|
||||
|
||||
// snapshotLook is how often the backup holder's record is read back while the snapshot is taken.
|
||||
var snapshotLook = 10 * time.Second
|
||||
|
||||
// backedUp is one module as the backup holder's `backed-up` answers it.
|
||||
type backedUp struct {
|
||||
Module string `json:"module"`
|
||||
Data []struct {
|
||||
Item string `json:"item"`
|
||||
LastBackup *time.Time `json:"last_backup"`
|
||||
Error string `json:"error"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// snapshotTheBus asks the bus node's backup holder to back the bus module up now, whose dump is the streams' snapshot
|
||||
// (ADR 0235), and reads its record back until it names a backup newer than the ask: where the snapshot is.
|
||||
func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (string, error) {
|
||||
asked := time.Now()
|
||||
if _, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".now", map[string]any{"module": module}); err != nil {
|
||||
return "", fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
|
||||
}
|
||||
deadline := asked.Add(SnapshotWithin)
|
||||
for {
|
||||
if raw, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".backed-up", map[string]any{}); err == nil {
|
||||
var modules []backedUp
|
||||
if json.Unmarshal(raw, &modules) == nil {
|
||||
for _, m := range modules {
|
||||
for _, it := range m.Data {
|
||||
if m.Module == module && it.LastBackup != nil && it.LastBackup.After(asked) && it.Error == "" {
|
||||
return fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, it.Item, it.LastBackup.UTC().Format(time.RFC3339)), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return "", fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s", node, SnapshotWithin)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(snapshotLook):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
|
||||
// router's record of the teller's ask says who chose and how (ADR 0274).
|
||||
func (a *putActs) warrant(id, ask string) {
|
||||
a.mu.Lock()
|
||||
if a.warranted[id] {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.warranted[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
|
||||
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.mu.Lock()
|
||||
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// warrantFailed says why the put's warrant was not recorded, "" when it was or is not yet known.
|
||||
func (a *putActs) warrantFailed(id string) string {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.warrantNo[id]
|
||||
}
|
||||
|
||||
// putSaid is the body of the event put.
|
||||
type putSaid struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
Hold bool `json:"hold"`
|
||||
Step string `json:"step"`
|
||||
Words string `json:"words"`
|
||||
Composed string `json:"composed,omitempty"`
|
||||
}
|
||||
|
||||
// say says a put's step as the event put, once per step and words.
|
||||
func (a *putActs) say(key string, p delivery.Put) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || a.bus == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.said[key] == p.Step+p.Words {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.said[key] = p.Step + p.Words
|
||||
a.mu.Unlock()
|
||||
if err := a.bus.Emit(PutEvent, putSaid{Node: node, Module: module, FP: p.FP, Hold: p.Hold, Step: p.Step, Words: p.Words, Composed: p.Composed}); err != nil {
|
||||
a.logf("the put %s's step %s is not said as an event: %v", key, p.Step, err)
|
||||
}
|
||||
}
|
||||
|
||||
// shortHex is a digest's first twelve hex digits.
|
||||
func shortHex(digest string) string {
|
||||
h := strings.TrimPrefix(digest, "sha256:")
|
||||
if len(h) > 12 {
|
||||
return h[:12]
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -1,810 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/judge"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
|
||||
// proposal one step, from what the world holds. A proposal the verbs wrote is said (placement-proposed); the nodes it
|
||||
// must hear from are found by composing every node with and without it, and those its key does not change get a reached
|
||||
// candidate; their answers are judged (placement-judged) and, on a passing verdict, the operator is asked to approve it;
|
||||
// on the warrant each changed key is written by compare-and-set at the revision it was laid over (placement-changed),
|
||||
// and the controller's assignments are kept in step through its verbs until the lease write. A key moved meanwhile
|
||||
// lays the proposal again; a decline, an expiry or a withdrawal ends it (placement-withdrawn). Every event is said on the
|
||||
// assigner's own stream, so only this process says them. Nothing polls: the wake at an answer's bound is the one wake.
|
||||
|
||||
// publisher says one of the seat's events on the assigner's stream, under a message id the bus keeps one of.
|
||||
type publisher func(event string, body any, id string) error
|
||||
|
||||
// streamPublisher publishes on mesh.stream.mesh-delivery-assigner.event.<event>, with the headers an emit carries.
|
||||
func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
|
||||
return func(event string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(process.ModuleStreamSubject(delivery.AssignerModule, event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", delivery.AssignerModule)
|
||||
msg.Header.Set("x-node", node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// seatActs keeps what the seat's calls of the controller came to while this process runs: the export read again at a
|
||||
// seed's warrant, and each call that keeps the controller's assignments in step (§8).
|
||||
type seatActs struct {
|
||||
mu sync.Mutex
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
export map[string]exportCheck
|
||||
// calls is, by pair (node and module, or node and provision for a pin), what the last call for it came to.
|
||||
calls map[string]string
|
||||
}
|
||||
|
||||
// exportCheck is the export read again for one proposal: running, its answer, or why it was not read.
|
||||
type exportCheck struct {
|
||||
running bool
|
||||
export map[string]delivery.Placement
|
||||
err string
|
||||
}
|
||||
|
||||
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
|
||||
return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{}}
|
||||
}
|
||||
|
||||
// exportOf is the export read again for a proposal, asked once in the background.
|
||||
func (a *seatActs) exportOf(id string) exportCheck {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if c, ok := a.export[id]; ok {
|
||||
return c
|
||||
}
|
||||
a.export[id] = exportCheck{running: true}
|
||||
go func() {
|
||||
defer a.wake()
|
||||
raw, err := a.bus.Call(exportCall, map[string]any{})
|
||||
c := exportCheck{}
|
||||
if err != nil {
|
||||
c.err = "the controller's placement-export did not answer: " + err.Error()
|
||||
} else if c.export, err = assign.ExportOf(raw); err != nil {
|
||||
c.err = err.Error()
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.export[id] = c
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
return exportCheck{running: true}
|
||||
}
|
||||
|
||||
// forget drops what is kept of a proposal's export, so a later warrant reads it again.
|
||||
func (a *seatActs) forget(id string) {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
delete(a.export, id)
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
// record records once, in the controller's hand-act log, that placement was written on the operator's warrant (§5): the
|
||||
// router's record of the teller's ask says who chose and how (ADR 0274).
|
||||
func (a *seatActs) record(warrant string) {
|
||||
if a == nil || warrant == "" {
|
||||
return
|
||||
}
|
||||
ask, _, _ := strings.Cut(warrant, "@")
|
||||
go func() {
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
|
||||
a.logf("the placement's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// handAct records a hand-back in the controller's hand-act log (§7).
|
||||
func (a *seatActs) handAct(prop delivery.PlacementProposal) {
|
||||
go func() {
|
||||
if _, err := a.bus.Call(handActCall, map[string]any{"what": "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "),
|
||||
"why": prop.Why, "cause": "asked by " + prop.By + ", approved by the operator"}); err != nil {
|
||||
a.logf("the hand-back %s is not recorded in the hand-act log: %v", prop.ID, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// callAll makes the controller's calls in order in the background, each said in the journal and kept by its pair.
|
||||
func (a *seatActs) callAll(calls []assign.ControllerCall) {
|
||||
if len(calls) == 0 {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
for _, c := range calls {
|
||||
a.calls[c.Node+"."+c.Module] = c.Key() + " is asked"
|
||||
}
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
defer a.wake()
|
||||
for _, c := range calls {
|
||||
words := c.Key() + " answered"
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+"."+c.Verb, c.Args); err != nil {
|
||||
words = c.Key() + " refused: " + err.Error()
|
||||
}
|
||||
a.logf("keeping the controller in step with placement: %s", words)
|
||||
a.mu.Lock()
|
||||
a.calls[c.Node+"."+c.Module] = words
|
||||
a.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// callOf is what the last call for a pair came to, "" when none was made.
|
||||
func (a *seatActs) callOf(node, module string) string {
|
||||
if a == nil {
|
||||
return ""
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.calls[node+"."+module]
|
||||
}
|
||||
|
||||
// withCalls names, on each pair the two sources do not both name, the call the assigner made for it (§8).
|
||||
func (p *passer) withCalls(pairs []assign.Pair) []assign.Pair {
|
||||
for i := range pairs {
|
||||
pairs[i].Call = p.seat.callOf(pairs[i].Node, pairs[i].Module)
|
||||
}
|
||||
return pairs
|
||||
}
|
||||
|
||||
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
|
||||
func (p *passer) sayWaits(unbound []string) {
|
||||
c := assign.SeatWaitsCondition(unbound)
|
||||
state := string(c.State) + c.Summary
|
||||
if p.say == nil || p.said[c.Key] && p.saidWaits == state {
|
||||
return
|
||||
}
|
||||
if c.State == asks.StateCleared && p.saidWaits == "" {
|
||||
return // nothing was raised by this process
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("that the seat waits for its states could not be said, and is said at the next pass: %v", err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key], p.saidWaits = true, state
|
||||
}
|
||||
|
||||
// seatPlacements is every node's placement key and its revision, as the watch holds them.
|
||||
func (p *passer) seatPlacements() (map[string]delivery.Placement, map[string]uint64) {
|
||||
out, revs := map[string]delivery.Placement{}, map[string]uint64{}
|
||||
for key, value := range p.w.read(delivery.PlacementBucket) {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if pl, err := delivery.DecodePlacement(value); err == nil && pl.Node == node {
|
||||
out[node], revs[node] = pl, p.w.metaOf(delivery.PlacementBucket, key).revision
|
||||
}
|
||||
}
|
||||
return out, revs
|
||||
}
|
||||
|
||||
// candidatesOf is a proposal's candidates by node, with their keys' revisions and the newest write.
|
||||
func (p *passer) candidatesOf(id string) (map[string]delivery.PlacementCandidate, map[string]uint64, time.Time) {
|
||||
out, revs := map[string]delivery.PlacementCandidate{}, map[string]uint64{}
|
||||
var newest time.Time
|
||||
for key, value := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
meta := p.w.metaOf(delivery.PlacementCandidatesBucket, key)
|
||||
out[node], revs[node] = c, meta.revision
|
||||
if meta.at.After(newest) {
|
||||
newest = meta.at
|
||||
}
|
||||
}
|
||||
return out, revs, newest
|
||||
}
|
||||
|
||||
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
|
||||
func (p *passer) answersTo(id string) map[string]*delivery.CandidateAnswer {
|
||||
out := map[string]*delivery.CandidateAnswer{}
|
||||
for key, value := range p.w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := h.Candidates[id]; ok {
|
||||
out[node] = &a
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// placementSeat takes every open proposal one step further.
|
||||
func (p *passer) placementSeat(ctx context.Context) {
|
||||
for _, b := range seatBuckets {
|
||||
if p.kvs[b] == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if p.saidProposed == nil {
|
||||
p.saidProposed, p.intended = map[string]bool{}, map[string]map[string]map[string]bool{}
|
||||
}
|
||||
current, revs := p.seatPlacements()
|
||||
ids := make([]string, 0)
|
||||
raws := p.w.read(delivery.PlacementProposalsBucket)
|
||||
for id := range raws {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
prop, err := delivery.DecodePlacementProposal(raws[id])
|
||||
if err != nil || prop.ID != id {
|
||||
p.logf("the proposal %s cannot be read, so it is left: %v", id, err)
|
||||
continue
|
||||
}
|
||||
p.stepProposal(ctx, prop, p.w.metaOf(delivery.PlacementProposalsBucket, id).revision, current, revs)
|
||||
}
|
||||
}
|
||||
|
||||
// pulse writes a bound's key with its time to live (§4): its expiry, heard on the watch as the server's delete marker,
|
||||
// is the event that ends the wait. Written again over one that stands, so its time starts anew.
|
||||
func (p *passer) pulse(ctx context.Context, key string, ttl time.Duration) {
|
||||
kv := p.kvs[delivery.PlacementPulsesBucket]
|
||||
_ = kv.Delete(ctx, key)
|
||||
rev, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl))
|
||||
if err != nil {
|
||||
p.logf("the bound %s is not written, so its wait ends at once: %v", key, err)
|
||||
return
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementPulsesBucket, key, []byte(`{}`), rev, p.clock())
|
||||
}
|
||||
|
||||
// pulsing says whether a bound's key stands.
|
||||
func (p *passer) pulsing(key string) bool { return p.w.has(delivery.PlacementPulsesBucket, key) }
|
||||
|
||||
// deletePulses deletes every bound of a proposal.
|
||||
func (p *passer) deletePulses(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementPulsesBucket) {
|
||||
if strings.HasSuffix(key, "."+id) || strings.Contains(key, "."+id+".") {
|
||||
if err := p.kvs[delivery.PlacementPulsesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementPulsesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ask raises (or raises again) a proposal's authorising ask and its bound, and answers the proposal asked.
|
||||
func (p *passer) ask(ctx context.Context, prop delivery.PlacementProposal, akey, brief string) (delivery.PlacementProposal, bool) {
|
||||
a, err := assign.PlacementAsk(akey, prop, brief, assign.PlacementAsking(prop, p.clock()), p.clock())
|
||||
if err != nil {
|
||||
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
|
||||
return prop, false
|
||||
}
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
if pk, err := delivery.PulseAskKey(prop.ID); err == nil {
|
||||
p.pulse(ctx, pk, a.Ask.Expires.Sub(p.clock()))
|
||||
}
|
||||
prop.State, prop.Ask = delivery.ProposalAsked, a.Ask.ID
|
||||
return prop, true
|
||||
}
|
||||
|
||||
// stepProposal takes one proposal one step further.
|
||||
func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if !p.saidProposed[prop.ID] && prop.State == delivery.ProposalAnswering && prop.Verdict == nil {
|
||||
p.sayEvent(delivery.PlacementProposedEvent, delivery.PlacementEvent{Proposal: prop}, prop.ID)
|
||||
p.saidProposed[prop.ID] = true
|
||||
}
|
||||
if prop.Withdrawn != "" {
|
||||
p.endProposal(ctx, prop, prop.Withdrawn)
|
||||
return
|
||||
}
|
||||
akey, _ := delivery.PlacementAskKey(prop.ID)
|
||||
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
|
||||
if prop.State == delivery.ProposalAsked {
|
||||
askPulse, _ := delivery.PulseAskKey(prop.ID)
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
if prop.HandBack {
|
||||
p.handBack(ctx, prop, akey, revs)
|
||||
} else {
|
||||
p.warranted(ctx, prop, rev, akey, current, revs)
|
||||
}
|
||||
return
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
p.endProposal(ctx, prop, "declined by the operator")
|
||||
return
|
||||
case !p.pulsing(askPulse) && !again:
|
||||
p.endProposal(ctx, prop, "its ask expired unanswered")
|
||||
return
|
||||
}
|
||||
}
|
||||
if prop.HandBack {
|
||||
// The operator's alone (§7): no node answers it; its ask is raised at once, and again on again.
|
||||
if prop.State != delivery.ProposalAsked || again {
|
||||
brief := fmt.Sprintf("%s (asked by %s): every node's placement key is deleted, %s, and each node returns to the "+
|
||||
"controller's compositions; the history is kept", prop.Why, prop.By, strings.Join(prop.Nodes(), ", "))
|
||||
now := p.clock().UTC()
|
||||
prop.Verdict = &delivery.PlacementVerdict{State: delivery.VerdictPass, At: now}
|
||||
if prop, ok := p.ask(ctx, prop, akey, brief); ok {
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
|
||||
return
|
||||
}
|
||||
cands, candRevs, written := p.candidatesOf(prop.ID)
|
||||
if len(prop.Must) == 0 {
|
||||
p.mustAnswer(ctx, prop, rev, current, cands)
|
||||
return
|
||||
}
|
||||
for _, node := range prop.Must {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // a candidate just written is not heard yet: its watch wakes the next pass
|
||||
}
|
||||
}
|
||||
if written.After(prop.Written) {
|
||||
prop.Written = written
|
||||
}
|
||||
pulsing := map[string]bool{}
|
||||
for _, node := range prop.Must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
pulsing[node] = p.pulsing(k)
|
||||
}
|
||||
}
|
||||
v := judge.JudgeAnswers(prop, p.intendedOf(prop, current, cands), p.answersTo(prop.ID), candRevs, p.w.online(), pulsing, p.clock())
|
||||
if prop.Verdict != nil && sameVerdict(*prop.Verdict, v) && !again {
|
||||
return
|
||||
}
|
||||
prop.Verdict = &v
|
||||
switch v.State {
|
||||
case delivery.VerdictFail:
|
||||
prop.State, prop.Ask = delivery.ProposalFailed, ""
|
||||
case delivery.VerdictWaits:
|
||||
prop.State, prop.Ask = delivery.ProposalAnswering, ""
|
||||
case delivery.VerdictPass:
|
||||
if prop.State != delivery.ProposalAsked || again {
|
||||
var ok bool
|
||||
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
|
||||
return
|
||||
}
|
||||
if again {
|
||||
prop.Words = "judged and asked again (again)"
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.writeProposal(ctx, prop, rev) {
|
||||
p.sayEvent(delivery.PlacementJudgedEvent, delivery.PlacementEvent{Proposal: prop},
|
||||
fmt.Sprintf("%s.judged.%d", prop.ID, v.At.UnixNano()))
|
||||
}
|
||||
}
|
||||
|
||||
// sameVerdict says two verdicts say the same, their time aside.
|
||||
func sameVerdict(a, b delivery.PlacementVerdict) bool {
|
||||
a.At, b.At = time.Time{}, time.Time{}
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
// moved says whether a key the proposal changes is no longer at the revision it was laid over.
|
||||
func moved(prop delivery.PlacementProposal, revs map[string]uint64) bool {
|
||||
for node, base := range prop.Base {
|
||||
if revs[node] != base {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// intended is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
|
||||
func (p *passer) intendedOf(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) map[string]map[string]bool {
|
||||
key := fmt.Sprint(prop.ID, prop.Base)
|
||||
if got, ok := p.intended[key]; ok {
|
||||
return got
|
||||
}
|
||||
got, _, _ := p.compute(prop, current, cands)
|
||||
p.intended[key] = got
|
||||
return got
|
||||
}
|
||||
|
||||
// compute composes every node with the current placement and with the proposal laid over it (judge.Intended).
|
||||
func (p *passer) compute(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) (map[string]map[string]bool, []string, []string) {
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
if c, ok := cands[node]; ok && !c.Reached {
|
||||
laid[node] = c.Placement
|
||||
}
|
||||
}
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
r := worldReader{w: p.w, manifests: get}
|
||||
var now, with *compose.Input
|
||||
if in, err := frombus.Input(context.Background(), r, frombus.Mesh{}, frombus.Scope{Every: true}); err == nil {
|
||||
now = &in
|
||||
}
|
||||
if in, err := frombus.Input(context.Background(), r, frombus.Mesh{}, frombus.Scope{Every: true, Placements: laid}); err == nil {
|
||||
with = &in
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for n := range current {
|
||||
nodes[n] = true
|
||||
}
|
||||
if records, err := r.Records(context.Background()); err == nil {
|
||||
for n := range records {
|
||||
nodes[n] = true
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for n := range nodes {
|
||||
names = append(names, n)
|
||||
}
|
||||
fps := map[string]string{}
|
||||
for name, m := range p.w.trunk() {
|
||||
if m.FP2 != "" {
|
||||
fps[name] = m.FP2
|
||||
}
|
||||
}
|
||||
return judge.Intended(now, with, current, laid, names, fps)
|
||||
}
|
||||
|
||||
// mustAnswer finds the nodes the proposal must hear from (§3), writes a reached candidate for each whose key it does not
|
||||
// change, and writes them into the proposal.
|
||||
func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) {
|
||||
for _, node := range prop.Nodes() {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // the verbs write the candidates before the proposal: wait to hear them
|
||||
}
|
||||
}
|
||||
intended, must, notes := p.compute(prop, current, cands)
|
||||
p.intended[fmt.Sprint(prop.ID, prop.Base)] = intended
|
||||
for _, node := range must {
|
||||
if _, ok := cands[node]; ok {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Reached: true}
|
||||
if pl, ok := current[node]; ok {
|
||||
c.Placement = &pl
|
||||
}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
for _, node := range must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
p.pulse(ctx, k, judge.AnswerBound)
|
||||
}
|
||||
}
|
||||
prop.Must = must
|
||||
if len(notes) > 0 {
|
||||
prop.Words = strings.TrimPrefix(prop.Words+"; "+strings.Join(notes, "; "), "; ")
|
||||
}
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
|
||||
// layAgain lays a proposal over the keys as they stand now (§5: a key moved): its candidates are written again, the
|
||||
// nodes that must answer found again, and it is judged and asked again; a proposal that no longer lays ends.
|
||||
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
|
||||
held, recorded := p.lays(ctx)
|
||||
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded,
|
||||
Seed: prop.Seed, ExportDigest: prop.ExportDigest})
|
||||
if len(findings) > 0 {
|
||||
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
for _, node := range again.Nodes() {
|
||||
key, _ := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Placement: laid[node]}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
|
||||
p.seat.forget(prop.ID)
|
||||
p.writeProposal(ctx, again, rev)
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against, as the verbs check it: the modules the indexer holds, by key, and the
|
||||
// nodes with a record.
|
||||
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
|
||||
held := map[string]bool{}
|
||||
for key := range p.w.read(delivery.ModulesBucket) {
|
||||
if name, err := delivery.ParseModulesKey(key); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
records, _ := worldReader{w: p.w}.Records(ctx)
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
|
||||
}
|
||||
|
||||
// warranted writes an approved proposal (§5): for the seed, after the export is read again and found unchanged (§7);
|
||||
// each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
|
||||
// proposal deleted, placement-changed said, and the controller kept in step (§8).
|
||||
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
if prop.Seed {
|
||||
if p.seat == nil {
|
||||
return
|
||||
}
|
||||
c := p.seat.exportOf(prop.ID)
|
||||
switch {
|
||||
case c.running:
|
||||
return // its answer wakes the next pass
|
||||
case c.err != "":
|
||||
if words := "the export cannot be read again at the warrant: " + c.err; prop.Words != words {
|
||||
prop.Words = words
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
return
|
||||
case delivery.ExportDigest(c.export) != prop.ExportDigest:
|
||||
p.endProposal(ctx, prop, "the controller's export changed since the seed was proposed: the new export is proposed as the seed")
|
||||
p.proposeSeed(ctx, prop, c.export)
|
||||
return
|
||||
}
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
|
||||
return
|
||||
}
|
||||
cands, _, _ := p.candidatesOf(prop.ID)
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
laid[node] = cands[node].Placement
|
||||
}
|
||||
writes, err := assign.Writes(prop, laid)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s cannot be written: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementBucket]
|
||||
var changes []delivery.PlacementChange
|
||||
writtenLaid := map[string]*delivery.Placement{}
|
||||
after := maps.Clone(current)
|
||||
afterRevs := maps.Clone(revs)
|
||||
for _, w := range writes {
|
||||
var rev2 uint64
|
||||
var err error
|
||||
switch {
|
||||
case w.Value == nil:
|
||||
err = kv.Delete(ctx, w.Key, jetstream.LastRevision(w.Base))
|
||||
default:
|
||||
raw, encErr := delivery.EncodePlacement(*w.Value)
|
||||
if encErr != nil {
|
||||
err = encErr
|
||||
} else if w.Base == 0 {
|
||||
rev2, err = kv.Create(ctx, w.Key, raw)
|
||||
} else {
|
||||
rev2, err = kv.Update(ctx, w.Key, raw, w.Base)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
|
||||
// again over the keys as this write left them.
|
||||
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
|
||||
p.followWrite(prop, answer, changes, current, writtenLaid)
|
||||
if rest, ok := assign.Rest(prop, writtenLaid); ok {
|
||||
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
|
||||
} else {
|
||||
p.endProposal(ctx, prop, "a key moved while it was written: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
before := current[w.Node]
|
||||
now := delivery.Placement{Node: w.Node}
|
||||
if w.Value != nil {
|
||||
now = *w.Value
|
||||
after[w.Node] = *w.Value
|
||||
} else {
|
||||
delete(after, w.Node)
|
||||
}
|
||||
afterRevs[w.Node] = rev2
|
||||
c := delivery.ChangeOf(before, now)
|
||||
c.Before, c.After = w.Base, rev2
|
||||
changes = append(changes, c)
|
||||
writtenLaid[w.Node] = w.Value
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.followWrite(prop, answer, changes, current, writtenLaid)
|
||||
}
|
||||
|
||||
// followWrite follows the keys a warrant wrote: the warrant recorded, placement-changed said, and, until the lease
|
||||
// write, the controller's assignments kept in step for them (§8). The seed calls no controller verb: it is what the
|
||||
// controller holds already (§9).
|
||||
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange,
|
||||
before map[string]delivery.Placement, written map[string]*delivery.Placement) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
if p.seat != nil && !assign.Leased && !prop.Seed {
|
||||
p.seat.callAll(assign.ControllerCalls(before, written, prop.ID, answer.Warrant))
|
||||
}
|
||||
}
|
||||
|
||||
func sortedNodes(m map[string]*delivery.Placement) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for n := range m {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// handBack deletes every key on the operator's warrant (§7) at the revision the hand-back read, records it in the
|
||||
// controller's hand-act log and says placement-changed; every node returns to the controller's compositions. A key moved
|
||||
// since ends it, and the operator hands back again.
|
||||
func (p *passer) handBack(ctx context.Context, prop delivery.PlacementProposal, akey string, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind this hand-back, so nothing is deleted: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
current, _ := p.seatPlacements()
|
||||
var changes []delivery.PlacementChange
|
||||
for _, node := range prop.Nodes() {
|
||||
key, _ := delivery.PlacementKey(node)
|
||||
if err := p.kvs[delivery.PlacementBucket].Delete(ctx, key, jetstream.LastRevision(prop.Base[node])); err != nil {
|
||||
p.sayChanged(prop, answer, changes)
|
||||
p.endProposal(ctx, prop, fmt.Sprintf("%s moved since the hand-back was asked (%v): hand back again", key, err))
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(current[node], delivery.Placement{Node: node})
|
||||
c.Before = prop.Base[node]
|
||||
changes = append(changes, c)
|
||||
}
|
||||
if p.seat != nil {
|
||||
p.seat.handAct(prop)
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
}
|
||||
|
||||
// sayChanged says placement-changed for the keys written.
|
||||
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
at := p.clock().UTC()
|
||||
if _, when, ok := strings.Cut(answer.Warrant, "@"); ok {
|
||||
if t, err := time.Parse(time.RFC3339Nano, when); err == nil {
|
||||
at = t
|
||||
}
|
||||
}
|
||||
delivery.SortChanges(changes)
|
||||
p.sayEvent(delivery.PlacementChangedEvent, delivery.PlacementEvent{Proposal: prop, Nodes: changes,
|
||||
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
|
||||
}
|
||||
|
||||
// proposeSeed proposes an export as the seed again, by the same proposer and why.
|
||||
func (p *passer) proposeSeed(ctx context.Context, was delivery.PlacementProposal, export map[string]delivery.Placement) {
|
||||
held, recorded := p.lays(ctx)
|
||||
now := p.clock()
|
||||
seed, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: was.By, Why: was.Why, At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)})
|
||||
if len(findings) > 0 {
|
||||
p.logf("the new export is not proposed as the seed: %s", strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
for _, node := range seed.Nodes() {
|
||||
key, _ := delivery.PlacementCandidateKey(seed.ID, node)
|
||||
c := delivery.PlacementCandidate{Proposal: seed.ID, Placement: laid[node]}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
seed.Written = now.UTC()
|
||||
p.writeProposal(ctx, seed, 0)
|
||||
}
|
||||
|
||||
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
|
||||
// whole with why it ended.
|
||||
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
|
||||
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(delivery.AssignerAsksBucket, akey) {
|
||||
if err := p.kvs[delivery.AssignerAsksBucket].Delete(ctx, akey); err == nil {
|
||||
p.w.put(delivery.AssignerAsksBucket, akey, nil)
|
||||
}
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
if err := p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not deleted, and is ended at the next pass: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.seat.forget(prop.ID)
|
||||
p.logf("the proposal %s ends: %s", prop.ID, why)
|
||||
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
|
||||
}
|
||||
|
||||
// deleteCandidates deletes every candidate of a proposal.
|
||||
func (p *passer) deleteCandidates(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
if pid, _, err := delivery.ParsePlacementCandidateKey(key); err == nil && pid == id {
|
||||
if err := p.kvs[delivery.PlacementCandidatesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementCandidatesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeProposal writes a proposal over the revision the pass read, by compare-and-set: a verb that marked it meanwhile
|
||||
// (withdraw, again) wins, and the next pass reads that. It says whether it wrote.
|
||||
func (p *passer) writeProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64) bool {
|
||||
raw, err := delivery.EncodePlacementProposal(prop)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s is not written: %v", prop.ID, err)
|
||||
return false
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementProposalsBucket]
|
||||
var at uint64
|
||||
if rev == 0 {
|
||||
at, err = kv.Create(ctx, prop.ID, raw)
|
||||
} else {
|
||||
at, err = kv.Update(ctx, prop.ID, raw, rev)
|
||||
}
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not written, and is taken further at the next pass: %v", prop.ID, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementProposalsBucket, prop.ID, raw, at, p.clock())
|
||||
return true
|
||||
}
|
||||
|
||||
// say says one of the seat's events, said in the journal when the bus refuses it.
|
||||
func (p *passer) sayEvent(event string, body delivery.PlacementEvent, id string) {
|
||||
if p.publish == nil {
|
||||
return
|
||||
}
|
||||
if err := p.publish(event, body, id); err != nil {
|
||||
p.logf("%s of %s is not said: %v", event, body.Proposal.ID, err)
|
||||
}
|
||||
}
|
||||
@@ -1,532 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
|
||||
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
|
||||
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
|
||||
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
|
||||
// warrant and the seat's events on its stream — is the assigner's process's (seat.go), started by these writes.
|
||||
|
||||
// seatStore is what the seat's verbs need of the states: the runtime's, or a test's.
|
||||
type seatStore interface {
|
||||
store
|
||||
del(state, key string) error
|
||||
history(state, key string) ([]stdio.StateRevision, error)
|
||||
}
|
||||
|
||||
func (runtimeStore) del(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
return stdio.State(state).History(key)
|
||||
}
|
||||
|
||||
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
|
||||
// `<module>.<state>`.
|
||||
const (
|
||||
modulesState = delivery.IndexerModule + ".modules"
|
||||
haveState = "node-engine.have"
|
||||
)
|
||||
|
||||
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
|
||||
type seatTools struct {
|
||||
st seatStore
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
|
||||
return &seatTools{st: st, ask: ask, now: time.Now}
|
||||
}
|
||||
|
||||
// keyed is one placement key as a verb answers it.
|
||||
type keyed struct {
|
||||
Placement delivery.Placement `json:"placement"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// placements reads every node's placement and its key's revision.
|
||||
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
|
||||
keys, err := s.st.keys(delivery.PlacementState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
|
||||
}
|
||||
out, revs := map[string]delivery.Placement{}, map[string]uint64{}
|
||||
for _, key := range keys {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement of %s cannot be read: %w", node, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePlacement(raw)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement of %s cannot be read: %w", node, err)
|
||||
}
|
||||
out[node], revs[node] = p, rev
|
||||
}
|
||||
return out, revs, nil
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
|
||||
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
|
||||
mods, err := s.st.keys(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, k := range mods {
|
||||
if name, err := delivery.ParseModulesKey(k); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
|
||||
}
|
||||
recorded := map[string]bool{}
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
|
||||
}
|
||||
|
||||
// proposal reads one open proposal.
|
||||
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
|
||||
if err := delivery.CheckProposalID(id); err != nil {
|
||||
return delivery.PlacementProposal{}, err
|
||||
}
|
||||
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
|
||||
if err != nil {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
|
||||
}
|
||||
if !found {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
|
||||
}
|
||||
return delivery.DecodePlacementProposal(raw)
|
||||
}
|
||||
|
||||
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
|
||||
// takes as the event to compose and judge it.
|
||||
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
|
||||
p.Written = s.now().UTC()
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementCandidateKey(p.ID, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
|
||||
return fmt.Errorf("the candidate %s is not written: %w", key, err)
|
||||
}
|
||||
}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// proposed is what propose and adopt answer.
|
||||
type proposed struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Follow string `json:"follow"`
|
||||
}
|
||||
|
||||
func followProposal(id string) string {
|
||||
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
|
||||
"operator is asked to approve it once its verdict passes", id)
|
||||
}
|
||||
|
||||
// changesOf reads propose's changes: a JSON array, given as one or as its text.
|
||||
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
|
||||
var raw []byte
|
||||
switch c := v.(type) {
|
||||
case string:
|
||||
raw = []byte(c)
|
||||
case nil:
|
||||
return nil, errors.New("propose names its changes")
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out []delivery.PlacementProposalChange
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the changes are no list of changes: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// doPropose is the verb propose (§2).
|
||||
func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
changes, err := changesOf(args["changes"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
return nil, errors.New("the seed is adopt's alone: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
current, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
|
||||
// placement-withdrawn carrying it whole.
|
||||
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("withdraw says why: it is kept in the event that ends the proposal")
|
||||
}
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
|
||||
}
|
||||
|
||||
// doAgain is the verb again (§10): the proposal is marked, and the assigner judges it now and raises its ask again
|
||||
// when its verdict passes.
|
||||
func (s *seatTools) doAgain(args map[string]any) (any, error) {
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Again = s.now().UTC()
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// exportCall is the controller's verb the seed is read from, by request and reply on its seat (§7).
|
||||
const exportCall = "seat:" + catalogue.ControllerSeatName + ".placement-export"
|
||||
|
||||
// doAdopt is the verb adopt (§7): the controller's export proposed whole as the seed.
|
||||
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
|
||||
current, _, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(current) > 0 {
|
||||
return nil, fmt.Errorf("placement is taken over already (%d keys): propose a change instead, or hand it back first", len(current))
|
||||
}
|
||||
if ids, err := s.st.keys(delivery.PlacementProposalsState); err == nil {
|
||||
for _, id := range ids {
|
||||
if p, err := s.proposal(id); err == nil && p.Seed {
|
||||
return nil, fmt.Errorf("the seed %s is open: withdraw it first, or answer its ask", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
raw, err := s.ask(exportCall, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export did not answer: %w", err)
|
||||
}
|
||||
export, err := assign.ExportOf(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// handActCall records a hand act in the controller's hand-act log.
|
||||
const handActCall = "seat:" + catalogue.ControllerSeatName + ".hand-act"
|
||||
|
||||
// doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back
|
||||
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
|
||||
// records the hand act and says placement-changed; every node returns to the controller's compositions.
|
||||
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
|
||||
if assign.Leased {
|
||||
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
|
||||
}
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("hand-back says why: it is recorded with the hand act")
|
||||
}
|
||||
if by == "" {
|
||||
return nil, errors.New("the runtime names no caller: a hand-back is made by someone")
|
||||
}
|
||||
_, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(revs) == 0 {
|
||||
return nil, errors.New("there is no placement key to hand back")
|
||||
}
|
||||
now := s.now()
|
||||
p := delivery.PlacementProposal{ID: delivery.NewProposalID(now), By: by, Why: why, At: now.UTC(), Written: now.UTC(),
|
||||
HandBack: true, Base: revs, State: delivery.ProposalAnswering}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the hand-back is not written: %w", err)
|
||||
}
|
||||
return proposed{Proposal: p, Follow: "the operator is asked to approve it; on the warrant every key is deleted and each " +
|
||||
"node returns to the controller's compositions"}, nil
|
||||
}
|
||||
|
||||
// placementAnswer is what the verb placement answers.
|
||||
type placementAnswer struct {
|
||||
Keys map[string]keyed `json:"keys,omitempty"`
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
Count int `json:"count"`
|
||||
History int `json:"history-per-key"`
|
||||
Changes []historyEntry `json:"changes,omitempty"`
|
||||
Older string `json:"older,omitempty"`
|
||||
}
|
||||
|
||||
// historyEntry is one change of a key in the state's revisions.
|
||||
type historyEntry struct {
|
||||
Node string `json:"node"`
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Change delivery.PlacementChange `json:"change"`
|
||||
Deleted bool `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
|
||||
// changes in order from the revisions the state keeps.
|
||||
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
|
||||
all, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, module := arg(args, "node"), arg(args, "module")
|
||||
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
|
||||
for n, p := range all {
|
||||
if node != "" && n != node {
|
||||
continue
|
||||
}
|
||||
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
|
||||
out.Nodes = append(out.Nodes, n)
|
||||
}
|
||||
sort.Strings(out.Nodes)
|
||||
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
|
||||
return nil, fmt.Errorf("%s has no placement key", node)
|
||||
}
|
||||
if arg(args, "history") != "true" {
|
||||
return out, nil
|
||||
}
|
||||
nodes := out.Nodes
|
||||
if node != "" {
|
||||
nodes = []string{node}
|
||||
} else if module == "" {
|
||||
nodes = nil
|
||||
for n := range all {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
}
|
||||
for _, n := range nodes {
|
||||
key, err := delivery.PlacementKey(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
revisions, err := s.st.history(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
|
||||
}
|
||||
var before delivery.Placement
|
||||
for _, r := range revisions {
|
||||
e := historyEntry{Node: n, Revision: r.Revision, At: r.At}
|
||||
after := delivery.Placement{}
|
||||
if r.Op == "" || r.Op == "put" || r.Op == "PUT" {
|
||||
if p, err := delivery.DecodePlacement(r.Value); err == nil {
|
||||
after = p
|
||||
}
|
||||
} else {
|
||||
e.Deleted = true
|
||||
}
|
||||
e.Change = delivery.ChangeOf(before, after)
|
||||
e.Change.Before, e.Change.After = 0, r.Revision
|
||||
before = after
|
||||
if module != "" && !strings.Contains(" "+strings.Join(append(append(e.Change.Added, e.Change.Removed...), e.Change.Changed...), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Changes = append(out.Changes, e)
|
||||
}
|
||||
}
|
||||
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
|
||||
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// openProposal is one open proposal as the verb proposals answers it: the proposal (its changes, base revisions,
|
||||
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
|
||||
// every node's answer now.
|
||||
type openProposal struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Candidates map[string]uint64 `json:"candidates"`
|
||||
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
|
||||
}
|
||||
|
||||
// doProposals is the verb proposals (§10).
|
||||
func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
ids, err := s.st.keys(delivery.PlacementProposalsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
|
||||
}
|
||||
cands, err := s.st.keys(delivery.PlacementCandidatesState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
|
||||
}
|
||||
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(haveState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for id, a := range h.Candidates {
|
||||
if answers[id] == nil {
|
||||
answers[id] = map[string]delivery.CandidateAnswer{}
|
||||
}
|
||||
answers[id][node] = a
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := []openProposal{}
|
||||
for _, id := range ids {
|
||||
if want := arg(args, "id"); want != "" && want != id {
|
||||
continue
|
||||
}
|
||||
p, err := s.proposal(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
|
||||
for _, k := range cands {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(k)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
|
||||
o.Candidates[node] = rev
|
||||
}
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
if arg(args, "id") != "" && len(out) == 0 {
|
||||
return nil, fmt.Errorf("no proposal %s is open", arg(args, "id"))
|
||||
}
|
||||
return map[string]any{"proposals": out}, nil
|
||||
}
|
||||
|
||||
// tools are the seat's verbs as the runtime serves them, under the seat's verb names, with the seat's own schemas.
|
||||
func (s *seatTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
|
||||
input := map[string]map[string]any{}
|
||||
description := map[string]string{}
|
||||
for _, v := range seat.Serves {
|
||||
input[v.Name], description[v.Name] = v.Input, v.Description
|
||||
}
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
catalogue.PlacementVerbPlacement: s.doPlacement,
|
||||
catalogue.PlacementVerbProposals: s.doProposals,
|
||||
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbAgain: s.doAgain,
|
||||
catalogue.PlacementVerbAdopt: func(a map[string]any) (any, error) { return s.doAdopt(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbHandBack: func(a map[string]any) (any, error) { return s.doHandBack(a, stdio.CallerOf(a)) },
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func (s memStore) del(state, key string) error {
|
||||
delete(s[state].values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
v, ok := s[state].values[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
|
||||
}
|
||||
|
||||
// ADR 0567 §2, §7, §10: the seat's seven verbs over the states. propose lays a proposal and writes it with its candidates,
|
||||
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt proposes the
|
||||
// controller's export whole as the seed, once; hand-back records the hand act and deletes every key; placement and
|
||||
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
|
||||
func TestTheSeatsVerbs(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
st := memStore{}
|
||||
for _, state := range []string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState, modulesState, haveState} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
for _, m := range []string{"web", "shell"} {
|
||||
st[modulesState].values[m] = []byte("{}")
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
|
||||
st[haveState].values[n+".node"] = raw
|
||||
}
|
||||
export, _ := json.Marshal(map[string]any{"anchor": anchor})
|
||||
var asked []string
|
||||
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
if key == exportCall {
|
||||
out, _ := json.Marshal(map[string]any{"ok": true, "output": string(export)})
|
||||
return out, nil
|
||||
}
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
})
|
||||
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
|
||||
|
||||
// adopt: the export proposed whole as the seed, once.
|
||||
got, err := s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := got.(proposed).Proposal
|
||||
if !seed.Seed || seed.ExportDigest != delivery.ExportDigest(map[string]delivery.Placement{"anchor": anchor}) || !slices.Equal(asked, []string{exportCall}) ||
|
||||
len(st[delivery.PlacementCandidatesState].values) != 1 || st[delivery.PlacementProposalsState].values[seed.ID] == nil {
|
||||
t.Fatalf("adopt = %+v, asked %v", seed, asked)
|
||||
}
|
||||
if _, err := s.doAdopt(map[string]any{"why": "again"}, "op"); err == nil || !strings.Contains(err.Error(), "is open") {
|
||||
t.Errorf("a second adopt with a seed open: %v", err)
|
||||
}
|
||||
// withdraw and again mark the proposal; the holder ends or judges it.
|
||||
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.doAgain(map[string]any{"id": seed.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again.IsZero() {
|
||||
t.Errorf("marked = %+v, %v", p, err)
|
||||
}
|
||||
if _, err := s.doWithdraw(map[string]any{"id": "p-none", "why": "x"}, "op"); err == nil {
|
||||
t.Error("an unknown proposal was withdrawn")
|
||||
}
|
||||
delete(st[delivery.PlacementProposalsState].values, seed.ID) // the holder ended it
|
||||
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
|
||||
|
||||
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
|
||||
raw, _ := delivery.EncodePlacement(anchor)
|
||||
st[delivery.PlacementState].values["placement.anchor"] = raw
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
|
||||
!strings.Contains(err.Error(), "ghost") || !strings.Contains(err.Error(), "lease write") || len(st[delivery.PlacementProposalsState].values) != 0 {
|
||||
t.Errorf("a refused proposal: %v", err)
|
||||
}
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
|
||||
t.Error("an unknown change was taken")
|
||||
}
|
||||
got, err = s.doPropose(map[string]any{"why": "move web", "changes": []any{
|
||||
map[string]any{"op": "take-off", "node": "anchor", "module": "web"}, map[string]any{"op": "place", "node": "laptop", "module": "web"}}}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
move := got.(proposed).Proposal
|
||||
if move.By != "operator@laptop" || move.Base["anchor"] != uint64(len(raw)) || move.Base["laptop"] != 0 || len(st[delivery.PlacementCandidatesState].values) != 2 {
|
||||
t.Fatalf("propose = %+v", move)
|
||||
}
|
||||
|
||||
// placement and proposals read every state the seat keeps (§10); history names the stream for what is older.
|
||||
reads := map[string]bool{}
|
||||
pa, err := s.doPlacement(map[string]any{"module": "web", "history": "true"})
|
||||
if a := pa.(placementAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, delivery.PlacementStream) {
|
||||
t.Errorf("placement of web = %+v, %v", pa, err)
|
||||
} else {
|
||||
reads[delivery.PlacementState], reads["stream "+delivery.PlacementStream] = true, true
|
||||
}
|
||||
if _, err := s.doPlacement(map[string]any{"node": "laptop"}); err == nil {
|
||||
t.Error("a node with no key was answered")
|
||||
}
|
||||
ps, err := s.doProposals(map[string]any{"id": move.ID})
|
||||
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 2 {
|
||||
t.Errorf("proposals = %+v, %v", ps, err)
|
||||
} else {
|
||||
reads[delivery.PlacementProposalsState], reads[delivery.PlacementCandidatesState] = true, true
|
||||
}
|
||||
raw0, err := readManifest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range raw0.State {
|
||||
if slices.Contains([]string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState}, state.Name) && !reads[state.Name] {
|
||||
t.Errorf("no verb reads the seat's state %s", state.Name)
|
||||
}
|
||||
}
|
||||
for _, stream := range raw0.Streams {
|
||||
if !reads["stream "+stream.Name] {
|
||||
t.Errorf("no verb reads the seat's stream %s", stream.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// hand-back: the operator's alone, written as a hand-back proposal naming every key at its revision, for the assigner
|
||||
// to ask; nothing is deleted by the verb.
|
||||
got, err = s.doHandBack(map[string]any{"why": "back"}, "op")
|
||||
if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 {
|
||||
t.Errorf("hand-back = %+v, %v", got, err)
|
||||
}
|
||||
if tools := s.tools(); len(tools) != 7 || tools[2].Name != catalogue.PlacementVerbPropose || tools[2].Run == nil || tools[2].Input == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
}
|
||||
|
||||
// readManifest is the module's manifest, read as the controller reads it.
|
||||
func readManifest() (catalogue.Manifest, error) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.ParseManifest(raw)
|
||||
}
|
||||
@@ -1,364 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
|
||||
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
|
||||
// (seat_tools.go, ADR 0567 §10).
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
const putsState = "puts"
|
||||
|
||||
// answerWithin is how long assigner_put waits for the assigner's process to take the put up before it answers what stands.
|
||||
var answerWithin = 20 * time.Second
|
||||
|
||||
// store is what the tools need of the states: the runtime's (stdio.State), or a test's.
|
||||
type store interface {
|
||||
get(state, key string) ([]byte, uint64, bool, error)
|
||||
put(state, key string, value []byte) error
|
||||
keys(state string) ([]string, error)
|
||||
}
|
||||
|
||||
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
|
||||
type putTools struct {
|
||||
st store
|
||||
now func() time.Time
|
||||
mu sync.Mutex
|
||||
// heard is every put the watch handed, by key; changed is closed and replaced at each.
|
||||
heard map[string][]byte
|
||||
changed chan struct{}
|
||||
}
|
||||
|
||||
func newPutTools(st store) *putTools {
|
||||
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
// hear takes a put's change in from the watch.
|
||||
func (s *putTools) hear(key string, value []byte) {
|
||||
s.mu.Lock()
|
||||
s.heard[key] = value
|
||||
close(s.changed)
|
||||
s.changed = make(chan struct{})
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
|
||||
var (
|
||||
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
|
||||
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
|
||||
)
|
||||
|
||||
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
|
||||
// of one the builder built, which must name one build alone.
|
||||
func (s *putTools) version(module, version string) (string, error) {
|
||||
v := strings.ToLower(strings.TrimSpace(version))
|
||||
switch {
|
||||
case v == "newest":
|
||||
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
|
||||
}
|
||||
if m.FP2 == "" {
|
||||
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
|
||||
}
|
||||
return m.FP2, nil
|
||||
case fullFP.MatchString(v):
|
||||
return v, nil
|
||||
case prefix.MatchString(v):
|
||||
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
|
||||
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
|
||||
}
|
||||
var found []string
|
||||
for _, k := range keys {
|
||||
m, fp, err := delivery.ParseBuilderBuildKey(k)
|
||||
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
|
||||
found = append(found, fp)
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
|
||||
case 1:
|
||||
return found[0], nil
|
||||
}
|
||||
sort.Strings(found)
|
||||
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
|
||||
}
|
||||
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
|
||||
}
|
||||
|
||||
// putAnswer is what assigner_put and assigner_puts answer of one put.
|
||||
type putAnswer struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
delivery.Put
|
||||
Follow string `json:"follow,omitempty"`
|
||||
}
|
||||
|
||||
// doPut is the tool assigner_put.
|
||||
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("put says why: it is kept with the put and said in any ask")
|
||||
}
|
||||
hold := false
|
||||
switch arg(args, "hold") {
|
||||
case "true":
|
||||
hold = true
|
||||
case "", "false":
|
||||
default:
|
||||
return nil, fmt.Errorf("hold is true or false, not %q", arg(args, "hold"))
|
||||
}
|
||||
fp, err := s.version(module, arg(args, "version"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now().UTC()
|
||||
words := fmt.Sprintf("asked by %s: %s", by, why)
|
||||
if raw, _, found, err := s.st.get(putsState, key); err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
} else if found {
|
||||
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
|
||||
if before.FP == fp && before.Hold == hold {
|
||||
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
|
||||
}
|
||||
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
|
||||
}
|
||||
}
|
||||
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(putsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
|
||||
}
|
||||
|
||||
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
|
||||
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
|
||||
deadline := time.After(answerWithin)
|
||||
for {
|
||||
s.mu.Lock()
|
||||
raw, changed := s.heard[key], s.changed
|
||||
s.mu.Unlock()
|
||||
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
|
||||
p = now
|
||||
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
|
||||
return p // it waits on a person or the node, or ended
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-changed:
|
||||
case <-deadline:
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// followPut says how a caller follows a put after the answer.
|
||||
func followPut(node, module string) string {
|
||||
return fmt.Sprintf("the put goes on: mesh_wait on mesh-delivery-assigner.put with filter {\"node\": %q, \"module\": %q}, then "+
|
||||
"mesh-delivery-assigner.assigner_puts for where it stands", node, module)
|
||||
}
|
||||
|
||||
// doRelease is the tool assigner_release.
|
||||
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("release says why: it is kept with the put")
|
||||
}
|
||||
raw, _, found, err := s.st.get(putsState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
|
||||
}
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
}
|
||||
if delivery.PutEnded(p.Step) {
|
||||
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
|
||||
}
|
||||
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(putsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
return putAnswer{Node: node, Module: module, Put: p}, nil
|
||||
}
|
||||
|
||||
// doPuts is the tool assigner_puts.
|
||||
func (s *putTools) doPuts(args map[string]any) (any, error) {
|
||||
keys, err := s.st.keys(putsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the puts cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []putAnswer{}
|
||||
for _, key := range keys {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(putsState, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, putAnswer{Node: node, Module: module, Put: p})
|
||||
}
|
||||
return map[string]any{"puts": out}, nil
|
||||
}
|
||||
|
||||
func arg(args map[string]any, name string) string {
|
||||
switch v := args[name].(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(v)
|
||||
case bool:
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// The tools' names, as the manifest lists them.
|
||||
const (
|
||||
toolPut = "assigner_put"
|
||||
toolRelease = "assigner_release"
|
||||
toolPuts = "assigner_puts"
|
||||
)
|
||||
|
||||
// schema is a tool's arguments, each a string: required ones first named, and switches true or false.
|
||||
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
|
||||
props := map[string]any{}
|
||||
for name, description := range properties {
|
||||
p := map[string]any{"type": "string", "description": description}
|
||||
if slices.Contains(switches, name) {
|
||||
p["enum"] = []string{"true", "false"}
|
||||
}
|
||||
props[name] = p
|
||||
}
|
||||
out := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
out["required"] = required
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tools are the assigner's tools as the runtime serves them (ADR 0546 clause 1).
|
||||
func (s *putTools) tools() []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: toolPut, Description: "Put one version of a module on one machine (novox/hq ADR 0546): the build at that fingerprint, " +
|
||||
"the composition and grant the controller wrote for that machine, then the machine's want naming it, held there until " +
|
||||
"released when hold is true. A module whose moves a person takes (the bus, a module of policy record) is asked of the " +
|
||||
"operator first, and the bus has its streams snapshotted after the yes. Answers what was done or what it waits on; the " +
|
||||
"put goes on after the answer, each step said as the event mesh-delivery-assigner.put, and assigner_puts says where it stands.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine, where the module is assigned",
|
||||
"module": "the module",
|
||||
"version": "newest, a fingerprint (src2: and 64 hex digits), or at least its first 12 hex digits",
|
||||
"hold": "\"true\": the want stays at this version until released; \"false\" (the default): the rules move it again once it runs",
|
||||
"why": "why, kept with the put and said in any ask",
|
||||
}, []string{"node", "module", "version", "why"}, "hold"),
|
||||
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
|
||||
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
|
||||
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
|
||||
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
|
||||
[]string{"node", "module", "why"}),
|
||||
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
|
||||
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
|
||||
"its time and words. Narrowed by machine or module. Read-only.",
|
||||
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
|
||||
Run: s.doPuts},
|
||||
}
|
||||
}
|
||||
|
||||
// runtimeStore is the runtime's states.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) put(state, key string, value []byte) error {
|
||||
_, err := stdio.State(state).Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
|
||||
func serveTools() error {
|
||||
s := newPutTools(runtimeStore{})
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := stdio.State(putsState).Watch("", func(c stdio.StateChange) error {
|
||||
s.hear(c.Key, c.Value)
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: the puts are not heard yet (%v); asked again in %s\n", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
return stdio.Serve("", append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...))
|
||||
}
|
||||
|
||||
// shortFP is a fingerprint's first twelve hex digits.
|
||||
func shortFP(fp string) string {
|
||||
part, err := delivery.FingerprintKeyPart(fp)
|
||||
if err != nil || len(part) < 12 {
|
||||
return fp
|
||||
}
|
||||
return part[:12]
|
||||
}
|
||||
@@ -13,10 +13,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
@@ -47,16 +44,8 @@ type world struct {
|
||||
manifests map[string]parsed
|
||||
// healthy is, by have key, every fingerprint the node ran healthy, oldest first, each once: what a set-back goes to.
|
||||
healthy map[string][]string
|
||||
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
|
||||
changes map[string]uint64
|
||||
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
|
||||
meta map[string]entryMeta
|
||||
}
|
||||
|
||||
// entryMeta is a value's revision in its bucket and when it was written.
|
||||
type entryMeta struct {
|
||||
revision uint64
|
||||
at time.Time
|
||||
// sent is, by build id, the fingerprint of the new form the controller's build record names, once it names one.
|
||||
sent map[string]string
|
||||
}
|
||||
|
||||
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
|
||||
@@ -67,27 +56,7 @@ type parsed struct {
|
||||
}
|
||||
|
||||
func newWorld() *world {
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
|
||||
meta: map[string]entryMeta{}}
|
||||
}
|
||||
|
||||
// putEntry is put, keeping the value's revision and the time it was written.
|
||||
func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at time.Time) {
|
||||
w.put(bucket, key, value)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if value == nil {
|
||||
delete(w.meta, bucket+"|"+key)
|
||||
return
|
||||
}
|
||||
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
|
||||
}
|
||||
|
||||
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
|
||||
func (w *world) metaOf(bucket, key string) entryMeta {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.meta[bucket+"|"+key]
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, sent: map[string]string{}}
|
||||
}
|
||||
|
||||
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
|
||||
@@ -102,7 +71,6 @@ func (w *world) put(bucket, key string, value []byte) {
|
||||
return
|
||||
}
|
||||
w.values[bucket][key] = value
|
||||
w.changes[bucket+"|"+key]++
|
||||
if bucket == delivery.HaveBucket {
|
||||
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
|
||||
@@ -193,56 +161,8 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
return p
|
||||
}
|
||||
|
||||
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
|
||||
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
|
||||
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
byAssignment, all := w.assignedByController()
|
||||
placed, seeded := w.placed()
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded)
|
||||
return both, all
|
||||
}
|
||||
|
||||
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name.
|
||||
func (w *world) placementDiffers() []assign.Pair {
|
||||
byAssignment, _ := w.assignedByController()
|
||||
placed, seeded := w.placed()
|
||||
_, differ := assign.Placed(byAssignment, placed, seeded)
|
||||
return differ
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
|
||||
// that does not read places nothing on its node and is said by the indexer's own read tool.
|
||||
func (w *world) placed() (map[string][]string, bool) {
|
||||
out := map[string][]string{}
|
||||
values := w.read(delivery.PlacementBucket)
|
||||
for key, value := range values {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil || p.Node != node {
|
||||
continue
|
||||
}
|
||||
out[node] = p.Assigned()
|
||||
}
|
||||
return out, len(values) > 0
|
||||
}
|
||||
|
||||
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
|
||||
// so their wants name the build alone and the seventh rule does not hold them.
|
||||
func (w *world) placedNodes() map[string]bool {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string]bool, len(placed))
|
||||
for node := range placed {
|
||||
out[node] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
|
||||
// name order.
|
||||
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
|
||||
out, all := map[string][]string{}, map[string]assignment{}
|
||||
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||
var a assignment
|
||||
@@ -289,128 +209,9 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
}
|
||||
m.Manifests[name] = p.manifest
|
||||
}
|
||||
// Once the catalogue places the nodes, the check composes over the input they compose from (ADR 0547 §2, ADR 0549
|
||||
// §3): one composer's input for the check and the nodes. An input that cannot be read is said, and the nodes and their
|
||||
// assignments alone are composed, as before any placement.
|
||||
if _, seeded := w.placed(); seeded {
|
||||
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true})
|
||||
if err != nil {
|
||||
m.InputUnread = err.Error()
|
||||
} else {
|
||||
m.Input = &in
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// worldReader is what the assigner watched as frombus reads the bus: the seat's placement, the indexer's modules, each node's
|
||||
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
|
||||
// composition it asks resolves, and stands in for what it does not carry).
|
||||
type worldReader struct {
|
||||
w *world
|
||||
manifests getter
|
||||
}
|
||||
|
||||
func (r worldReader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
out := map[string]delivery.Placement{}
|
||||
for key, value := range r.w.read(delivery.PlacementBucket) {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if p, err := delivery.DecodePlacement(value); err == nil && p.Node == node {
|
||||
out[node] = p
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
|
||||
out := map[string]delivery.NodeRecord{}
|
||||
for key, value := range r.w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
out[node] = *h.Record
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r worldReader) Modules(context.Context) (map[string]delivery.Module, error) {
|
||||
return r.w.trunk(), nil
|
||||
}
|
||||
|
||||
func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := r.manifests(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(m.JSON), nil
|
||||
}
|
||||
|
||||
// Values is one module's config values on one node as mesh-config resolved them, each a value its source lays
|
||||
// (catalogue.LaidValue): the holder composes every node a proposal touches with them (ADR 0567 §9).
|
||||
func (r worldReader) Values(_ context.Context, node, module string) (map[string]any, error) {
|
||||
var out map[string]any
|
||||
prefix := node + "." + module + "."
|
||||
for key, raw := range r.w.read(buscompose.ResolvedBucket) {
|
||||
name, ok := strings.CutPrefix(key, prefix)
|
||||
if !ok || strings.Contains(name, ".") {
|
||||
continue
|
||||
}
|
||||
if v, laid := catalogue.LaidValue(raw); laid {
|
||||
if out == nil {
|
||||
out = map[string]any{}
|
||||
}
|
||||
out[name] = v
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (worldReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// Build and Archive are read only for a node composing itself (frombus.Scope without Every): the check of every node
|
||||
// composes from the manifests the indexer read, and reads no build and no archive.
|
||||
func (worldReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (worldReader) Archive(_ context.Context, url, _ string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("the composition check reads no archive (%s)", url)
|
||||
}
|
||||
|
||||
func (r worldReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
|
||||
all, err := r.Values(ctx, node, module)
|
||||
var out map[string]any
|
||||
for _, k := range keys {
|
||||
if v, ok := all[k]; ok {
|
||||
if out == nil {
|
||||
out = map[string]any{}
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// change is a candidate's change for the composition check, or why the composition waits for it.
|
||||
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
|
||||
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
|
||||
@@ -463,21 +264,6 @@ func (w *world) change(ctx context.Context, manifests getter, c delivery.Candida
|
||||
return ch, ""
|
||||
}
|
||||
|
||||
// online is, by node, whether the watchman holds the node online.
|
||||
func (w *world) online() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for key, value := range w.read(delivery.PresenceBucket) {
|
||||
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || node == "" {
|
||||
continue
|
||||
}
|
||||
if p, err := delivery.DecodePresence(value); err == nil {
|
||||
out[node] = p.Online
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// wantsOf is the wants the assigner wrote, by module then node.
|
||||
func (w *world) wantsOf() map[string]map[string]delivery.Want {
|
||||
out := map[string]map[string]delivery.Want{}
|
||||
@@ -539,13 +325,6 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
||||
return n
|
||||
}
|
||||
|
||||
// value is what a bucket holds at a key now, nil when nothing.
|
||||
func (w *world) value(bucket, key string) []byte {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.values[bucket][key]
|
||||
}
|
||||
|
||||
// has says whether a bucket holds a key now.
|
||||
func (w *world) has(bucket, key string) bool {
|
||||
w.mu.Lock()
|
||||
@@ -570,55 +349,30 @@ func (w *world) built(module, fp string) bool {
|
||||
return err == nil && b.State == delivery.BuildDone
|
||||
}
|
||||
|
||||
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
|
||||
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
|
||||
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
// sentFP is the fingerprint of the new form of the build the controller's assignment names, read from its build record
|
||||
// (mesh-controller_builds `<module>.build.<id>`) once and kept; empty when the record names none.
|
||||
func (w *world) sentFP(ctx context.Context, get getter, module, build string) string {
|
||||
if build == "" {
|
||||
return ""
|
||||
}
|
||||
w.mu.Lock()
|
||||
fp, ok := w.sent[build]
|
||||
w.mu.Unlock()
|
||||
if ok {
|
||||
return fp
|
||||
}
|
||||
value, err := get(ctx, delivery.BuildKey(module, build))
|
||||
if err != nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
return ""
|
||||
}
|
||||
var rec delivery.BuildRecord
|
||||
if json.Unmarshal(value, &rec) != nil || !strings.HasPrefix(rec.SourceFingerprint, delivery.SourcePrefix2) {
|
||||
return ""
|
||||
}
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.ControllerComposedBucket][key]
|
||||
w.sent[build] = rec.SourceFingerprint
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(value)
|
||||
return d, err == nil
|
||||
}
|
||||
|
||||
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
|
||||
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
|
||||
out := map[string]delivery.ComposedDigest{}
|
||||
for key, value := range w.read(delivery.ControllerComposedBucket) {
|
||||
node, _, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if d, err := delivery.DecodeComposedDigest(value); err == nil {
|
||||
out[node] = d
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
|
||||
func (w *world) answer(key string) (delivery.Answer, bool) {
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.AnswersBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.Answer{}, false
|
||||
}
|
||||
a, err := delivery.DecodeAnswer(value)
|
||||
return a, err == nil
|
||||
}
|
||||
|
||||
// changed is how many values the key has taken in this process (see world.changes).
|
||||
func (w *world) changed(bucket, key string) uint64 {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.changes[bucket+"|"+key]
|
||||
return rec.SourceFingerprint
|
||||
}
|
||||
|
||||
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
|
||||
|
||||
@@ -48,7 +48,6 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
|
||||
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
|
||||
// another repository as defined twice.
|
||||
func TestTheChangeOfACandidate(t *testing.T) {
|
||||
theWorldCountsTheValuesAKeyTakes(t)
|
||||
f := fakeManifests{}
|
||||
web := f.add(t, "web", `{"module":"web","version":"2"}`)
|
||||
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
|
||||
@@ -112,17 +111,3 @@ func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
|
||||
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
|
||||
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
|
||||
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
|
||||
t.Errorf("changed = %d, want 2", got)
|
||||
}
|
||||
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
|
||||
t.Errorf("changed = %d for a key never seen", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c h1:ngcItoLuh2uP24toZvkCXGA/3+TBrLTwTbE28XpYDWY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980 h1:ZEEeYibEPD1rngOAqbODzRY5rUStGC46OpyQwtNCYbs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -2,28 +2,24 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
|
||||
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
|
||||
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
|
||||
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
|
||||
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
|
||||
// it held or release it. Both options acknowledge and bind nothing: in release 5 the wants are only compared, so no answer
|
||||
// changes what a node runs (ADR 0429 §7); the node agent acts on the hold from release 6.
|
||||
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
|
||||
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
|
||||
"lets the wants move to it now.", module, node, short(fp), words, module),
|
||||
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
|
||||
"no node to move to it. Until release 6 this changes nothing a node runs: the controller still sends.", module, short(fp)),
|
||||
Options: []asks.Option{
|
||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released when wants are applied, from release 6", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
@@ -31,218 +27,3 @@ func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) del
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
|
||||
const (
|
||||
OptionRestore = "restore"
|
||||
OptionLeaveStopped = "leave-stopped"
|
||||
)
|
||||
|
||||
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
|
||||
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
|
||||
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
|
||||
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
|
||||
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
|
||||
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module stays as it is on the node",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
|
||||
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
|
||||
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
|
||||
// data, whose restore ask waits for the operator.
|
||||
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
|
||||
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
|
||||
if needsYou {
|
||||
c.Needs = "answer whether to restore " + module + " on " + node
|
||||
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
|
||||
} else {
|
||||
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
|
||||
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
|
||||
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
|
||||
// (raised false) once the node no longer fails alone.
|
||||
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s fails on %s alone", module, node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
|
||||
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
|
||||
"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
|
||||
node, node)}
|
||||
}
|
||||
|
||||
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||
// node and module, and cleared (raised false) when the node has a want for it again.
|
||||
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".neither." + node + "." + module
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is on no list", module, node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
if h.Bus {
|
||||
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
|
||||
asks.NeedsYou, needs, module, node, h.Words)}
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
|
||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
|
||||
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
|
||||
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
|
||||
func NeitherLeftCondition(node, module, why string) asks.Condition {
|
||||
c := NeitherCondition(node, module, Holding{}, false)
|
||||
if why != "" {
|
||||
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) asks.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
c.Resolved = LiftedWords(module, fp) + "."
|
||||
return c
|
||||
}
|
||||
|
||||
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
||||
const (
|
||||
OptionYes = "yes"
|
||||
OptionNo = "no"
|
||||
OptionBackupAgain = delivery.BackupAgain
|
||||
OptionBackupSkip = delivery.BackupSkip
|
||||
)
|
||||
|
||||
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
|
||||
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
|
||||
// after the answer yes. The way the move was judged is in the ask.
|
||||
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
|
||||
what := map[delivery.Judgement]string{
|
||||
delivery.JudgedRecreates: "recreates a container",
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
delivery.JudgedCannot: "cannot be judged: " + need.Words,
|
||||
}[need.Judgement]
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
|
||||
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
|
||||
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
|
||||
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
|
||||
func NextBackupAsking(after int) int {
|
||||
if after < 0 {
|
||||
return 0
|
||||
}
|
||||
return max(after+1, 2)
|
||||
}
|
||||
|
||||
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
|
||||
|
||||
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
|
||||
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
|
||||
// under a new number (ADR 0539 §16).
|
||||
func BackupAskDigest(a delivery.AssignerAsk) string {
|
||||
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
|
||||
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
|
||||
func BackupAnswerWords(option string, asking int) string {
|
||||
return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1)
|
||||
}
|
||||
|
||||
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
|
||||
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
|
||||
// backup's own words, from the node's have.
|
||||
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
|
||||
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
|
||||
module, short(fp), node, words, module, digest),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
|
||||
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module keeps running what it has",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
|
||||
// naming the node, the module and the backup's words; cleared under the same key.
|
||||
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
|
||||
needs := "answer whether to back up again or go on without a backup"
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -17,39 +15,7 @@ func TestTheHoldAskIsValid(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
// 1906/H3: the ask names the machine whose check failed and the hold's words.
|
||||
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
|
||||
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
|
||||
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
|
||||
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
|
||||
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
|
||||
t.Errorf("the ask asked again cannot be written: %v", err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
|
||||
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
|
||||
func TestTheRestoreAskIsValid(t *testing.T) {
|
||||
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
|
||||
a := HoldAsk(key, "mail", fNew, now)
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
@@ -59,145 +25,4 @@ func TestTheRestoreAskIsValid(t *testing.T) {
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
var ids []string
|
||||
for _, o := range a.Ask.Options {
|
||||
ids = append(ids, o.ID)
|
||||
}
|
||||
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
|
||||
t.Errorf("options = %v", ids)
|
||||
}
|
||||
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
|
||||
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
|
||||
// name (ADR 0437 clause 9), and is cleared under the same key.
|
||||
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
||||
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
c := LiftedCondition("mail", fNew)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
||||
// the view, and name the move and the node.
|
||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
now := time.Now()
|
||||
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
|
||||
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dgst := "sha256:" + strings.Repeat("b", 64)
|
||||
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
|
||||
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("%s: the router refuses the ask: %v", key, err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("%s: cannot be written: %v", key, err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("%s: the view may not answer it: %v", key, err)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
|
||||
t.Errorf("the record ask = %+v", r.Ask)
|
||||
}
|
||||
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
|
||||
t.Errorf("the backup ask = %+v", b.Ask)
|
||||
}
|
||||
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
|
||||
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
|
||||
t.Errorf("asked again under the same id")
|
||||
}
|
||||
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
|
||||
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
|
||||
}
|
||||
if BackupAskDigest(b) != dgst {
|
||||
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst)
|
||||
}
|
||||
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
|
||||
t.Errorf("two answers of the same option are one want value")
|
||||
}
|
||||
c := BackupCondition("ace", "mail", fNew, "no space left", true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Errorf("the backup condition is refused: %v", err)
|
||||
}
|
||||
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
|
||||
// so the view cannot answer it, and it lasts no longer than an approving ask may.
|
||||
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
|
||||
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||
for _, bus := range []bool{false, true} {
|
||||
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
|
||||
if err != nil {
|
||||
t.Fatalf("bus %v: %v", bus, err)
|
||||
}
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
|
||||
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
|
||||
}
|
||||
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
|
||||
yes := a.Ask.Options[0]
|
||||
switch {
|
||||
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
|
||||
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
|
||||
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
|
||||
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
|
||||
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
|
||||
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
|
||||
}
|
||||
later := put
|
||||
later.At = now.Add(time.Minute)
|
||||
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
|
||||
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
|
||||
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
|
||||
}
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
|
||||
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
|
||||
}
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
|
||||
t.Errorf("bus: the yes performs another composition's act")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
|
||||
// under the assigner's own name keyed by node and module, and is cleared under the same key.
|
||||
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
|
||||
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
|
||||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Sent is what the controller sends a node of a module: whether it sends it at all (an open assignment), the build its
|
||||
// assignment names, and that build's fingerprint of the new form (empty when its record names none).
|
||||
type Sent struct {
|
||||
Sends bool
|
||||
Build string
|
||||
FP string
|
||||
}
|
||||
|
||||
// Compare is the difference between a node's want for a module and what the controller sends it (ADR 0528 clause 4),
|
||||
// and whether there is one. want is nil when the assigner wrote none; newest is the module's newest fingerprint and
|
||||
// hold the rule that holds the node back from it (Rule 0 when none does). A difference is explained by a rule when the
|
||||
// controller sent the newest fingerprint while that rule holds the node's want back; every other is unexplained, said
|
||||
// in words, for acceptance test 17 of design 47 to read.
|
||||
func Compare(want *delivery.Want, sent Sent, newest string, hold Holding) (delivery.Difference, bool) {
|
||||
var d delivery.Difference
|
||||
removed := want != nil && want.Removed
|
||||
if want != nil && !removed {
|
||||
d.Want = want.FP
|
||||
}
|
||||
d.WantRemoved = removed
|
||||
if !sent.Sends {
|
||||
if want == nil || removed {
|
||||
return delivery.Difference{}, false
|
||||
}
|
||||
d.Unexplained, d.Words = true, "the node wants "+short(want.FP)+" and the controller sends it nothing"
|
||||
return d, true
|
||||
}
|
||||
d.Sent, d.SentBuild = sent.FP, sent.Build
|
||||
switch {
|
||||
case sent.FP == "":
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the build %s the controller sends names no fingerprint of the new form", sent.Build)
|
||||
case removed:
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the want is removed and the controller still sends %s", short(sent.FP))
|
||||
case sent.FP == d.Want:
|
||||
return delivery.Difference{}, false
|
||||
case hold.Rule != 0 && sent.FP == newest:
|
||||
d.Rule, d.Words = hold.Rule, fmt.Sprintf("the controller sent %s while %s", short(sent.FP), hold.Words)
|
||||
case d.Want == "":
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node has no want", short(sent.FP))
|
||||
default:
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node wants %s", short(sent.FP), short(d.Want))
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1297/B5: given the controller sent F to a node while the canary is not yet healthy on F, the difference names rule 3.
|
||||
func TestADifferenceExplainedByTheCanary(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
want := delivery.Want{FP: fOld, Newest: fNew, Rule: delivery.RuleCanary}
|
||||
d, ok := Compare(&want, Sent{Sends: true, Build: "b-9", FP: fNew}, fNew, out.Rules["g14"])
|
||||
if !ok || d.Rule != delivery.RuleCanary || d.Unexplained || d.Want != fOld || d.Sent != fNew {
|
||||
t.Fatalf("difference = %+v, %v; want rule 3", d, ok)
|
||||
}
|
||||
if _, err := delivery.EncodeDifference(d); err != nil {
|
||||
t.Errorf("the difference cannot be written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B5: a want equal to what is sent is no difference; nothing sent and no want is none; every other is written
|
||||
// unexplained, with words, and is writable.
|
||||
func TestCompare(t *testing.T) {
|
||||
w := delivery.Want{FP: fNew}
|
||||
if _, ok := Compare(&w, Sent{Sends: true, Build: "b", FP: fNew}, fNew, Holding{}); ok {
|
||||
t.Errorf("an equal want and send is a difference")
|
||||
}
|
||||
if _, ok := Compare(nil, Sent{}, fNew, Holding{}); ok {
|
||||
t.Errorf("no want and nothing sent is a difference")
|
||||
}
|
||||
removed := delivery.Want{Removed: true}
|
||||
if _, ok := Compare(&removed, Sent{}, fNew, Holding{}); ok {
|
||||
t.Errorf("a removed want and nothing sent is a difference")
|
||||
}
|
||||
old := delivery.Want{FP: fOld}
|
||||
for name, c := range map[string]struct {
|
||||
want *delivery.Want
|
||||
sent Sent
|
||||
}{
|
||||
"sends an older one": {&w, Sent{Sends: true, Build: "b", FP: fOld}},
|
||||
"names no fingerprint": {&w, Sent{Sends: true, Build: "b"}},
|
||||
"sends a removed module": {&removed, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||
"sends nothing": {&w, Sent{}},
|
||||
"no want": {nil, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||
"no rule holds it back": {&old, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||
} {
|
||||
d, ok := Compare(c.want, c.sent, fNew, Holding{})
|
||||
if !ok || !d.Unexplained || d.Words == "" {
|
||||
t.Errorf("%s: difference = %+v, %v; want it unexplained, with words", name, d, ok)
|
||||
continue
|
||||
}
|
||||
if _, err := delivery.EncodeDifference(d); err != nil {
|
||||
t.Errorf("%s: the difference cannot be written: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -35,13 +35,6 @@ type Mesh struct {
|
||||
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
|
||||
// composer at another mesh-sdk reads (issue 1622).
|
||||
Digests map[string]string
|
||||
// Input is the rest of the mesh as the nodes compose from it, filled by mesh-sdk's frombus from the catalogue's
|
||||
// placement and the nodes' records (novox/hq ADR 0547 §2, ADR 0549 §3, issue 1126), so the check and the nodes share
|
||||
// one composer's input; nil before any placement is read, when the nodes and their assignments alone are composed.
|
||||
Input *compose.Input
|
||||
// InputUnread says why that input could not be read from the bus, when it could not: the check then composes the
|
||||
// nodes and their assignments alone, and says so in its warnings.
|
||||
InputUnread string
|
||||
}
|
||||
|
||||
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
|
||||
@@ -64,8 +57,7 @@ type Change struct {
|
||||
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
|
||||
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
|
||||
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
|
||||
// touches no machine. A change of placement is no change of a repository (ADR 0567 §4): the seat
|
||||
// mesh-placement judges it.
|
||||
// touches no machine.
|
||||
func Touched(m Mesh, c Change) []string {
|
||||
moved := map[string]bool{}
|
||||
for name := range c.Manifests {
|
||||
@@ -280,18 +272,6 @@ func GrantsAlone(m Mesh, c Change, composed []string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// withOverlay is base with every manifest of over laid over it, a copy.
|
||||
func withOverlay(base, over map[string]catalogue.Manifest) map[string]catalogue.Manifest {
|
||||
out := make(map[string]catalogue.Manifest, len(base)+len(over))
|
||||
for name, man := range base {
|
||||
out[name] = man
|
||||
}
|
||||
for name, man := range over {
|
||||
out[name] = man
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
|
||||
@@ -42,17 +42,9 @@ func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
|
||||
for _, name := range c.Removed {
|
||||
delete(change, name)
|
||||
}
|
||||
in, fromClaims := plainInput(m.Assigned), true
|
||||
if m.Input != nil {
|
||||
// The input the nodes compose from, without its catalogue: the cases carry the manifests (issue 1126).
|
||||
in = *m.Input
|
||||
in.Catalogue = nil
|
||||
fromClaims = len(in.Holdings) == 0
|
||||
}
|
||||
changeCase := composer.Case{Name: CaseChange, Manifests: change}
|
||||
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, changeCase}}
|
||||
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: true, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
|
||||
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
|
||||
}
|
||||
|
||||
@@ -203,10 +195,6 @@ func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Durati
|
||||
readerFails, readerWarnings := judgeReaders(c, change, readers)
|
||||
fails = append(fails, readerFails...)
|
||||
v.Warnings = append(v.Warnings, readerWarnings...)
|
||||
if m.InputUnread != "" {
|
||||
v.Warnings = append(v.Warnings, "composed over the nodes and their assignments alone: the input the nodes compose from "+
|
||||
"cannot be read from the bus ("+m.InputUnread+")")
|
||||
}
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -101,25 +100,6 @@ func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
|
||||
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
|
||||
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2, ADR 0549 §3): once the catalogue places the nodes, the check composes over the input the nodes
|
||||
// compose from (frombus), its catalogue left to the cases, and the seats held as placement says.
|
||||
m := digestedMesh()
|
||||
m.Input = &compose.Input{Catalogue: map[string]catalogue.Manifest{"web": {Module: "web"}},
|
||||
Nodes: []compose.Node{{Name: "g14", Assigned: []string{"web"}, Place: compose.Place{Endpoint: "g14.example:51820"}}},
|
||||
Holdings: []catalogue.Held{{Claim: "web-seat", Scope: "mesh", Node: "g14", Module: "web"}}}
|
||||
placedJob, err := HeadJob(m, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed, err := composer.Decode(placedJob.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if placed.Input.Catalogue != nil || len(placed.Input.Nodes) != 1 || placed.Input.Nodes[0].Place.Endpoint != "g14.example:51820" ||
|
||||
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
|
||||
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
|
||||
// the seat mesh-placement's placement (ADR 0549 §5, ADR 0567 §8). The assigner writes wants only for a pair both name:
|
||||
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
|
||||
// removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal
|
||||
// the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not
|
||||
// yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key
|
||||
// is written the controller's assignments are the one source, as before.
|
||||
|
||||
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
|
||||
// OnlyPlacement.
|
||||
type Pair struct {
|
||||
Node, Module, Only string
|
||||
// Call is what became of the assigner's call of the controller's verb for the pair, in words, when it made one.
|
||||
Call string
|
||||
}
|
||||
|
||||
// The source that names a pair alone.
|
||||
const (
|
||||
OnlyAssignments = "the controller's assignments"
|
||||
OnlyPlacement = "the seat mesh-placement's placement"
|
||||
)
|
||||
|
||||
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
|
||||
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs.
|
||||
func Placed(assigned, placed map[string][]string, seeded bool) (map[string][]string, []Pair) {
|
||||
if !seeded {
|
||||
return assigned, nil
|
||||
}
|
||||
set := func(m map[string][]string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for node, modules := range m {
|
||||
for _, module := range modules {
|
||||
out[node+"\x00"+module] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
inAssigned, inPlaced := set(assigned), set(placed)
|
||||
both := map[string][]string{}
|
||||
var differ []Pair
|
||||
for node, modules := range assigned {
|
||||
for _, module := range modules {
|
||||
if inPlaced[node+"\x00"+module] {
|
||||
both[node] = append(both[node], module)
|
||||
} else {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyAssignments})
|
||||
}
|
||||
}
|
||||
}
|
||||
for node, modules := range placed {
|
||||
for _, module := range modules {
|
||||
if !inAssigned[node+"\x00"+module] {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyPlacement})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, modules := range both {
|
||||
sort.Strings(modules)
|
||||
}
|
||||
sort.Slice(differ, func(i, j int) bool {
|
||||
if differ[i].Node != differ[j].Node {
|
||||
return differ[i].Node < differ[j].Node
|
||||
}
|
||||
return differ[i].Module < differ[j].Module
|
||||
})
|
||||
return both, differ
|
||||
}
|
||||
|
||||
// PlacementCondition is the condition of a pair the two sources do not both name: raised while they differ, cleared once
|
||||
// both name it or neither does.
|
||||
func PlacementCondition(p Pair, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".placement." + p.Node + "." + p.Module
|
||||
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("the controller's assignments and the seat's placement agree on %s on %s.", p.Module, p.Node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
var needs string
|
||||
if p.Only == OnlyAssignments {
|
||||
needs = fmt.Sprintf("propose placing %s on %s (mesh-placement.propose), or unassign it at the controller", p.Module, p.Node)
|
||||
} else {
|
||||
needs = fmt.Sprintf("assign %s to %s at the controller, as the approved placement says, or propose taking it off", p.Module, p.Node)
|
||||
}
|
||||
if p.Call != "" {
|
||||
needs += "; the assigner's call: " + p.Call
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s names %s on %s and the other source does not. Until ADR 0547's third milestone both must "+
|
||||
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
|
||||
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
|
||||
}
|
||||
|
||||
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
|
||||
// §1): raised naming each, cleared when none is left.
|
||||
func SeatWaitsCondition(unbound []string) asks.Condition {
|
||||
key := delivery.AssignerModule + ".seat-states"
|
||||
subject := asks.Scope{Scope: "module", ID: delivery.AssignerModule}
|
||||
if len(unbound) == 0 {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: "the seat mesh-placement holds all its states."}
|
||||
}
|
||||
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
|
||||
Explanation: asks.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
|
||||
}
|
||||
@@ -1,252 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
|
||||
// key; each pass takes it one step further from what the world holds (StepPut), and the rules pass over the node's want for
|
||||
// the module from its asking until it ends (Module.Puts).
|
||||
|
||||
// ComposeWithin is how long a put waits for the composition and grant it needs (a take-in of a build just done writes them)
|
||||
// before it is refused.
|
||||
const ComposeWithin = 10 * time.Minute
|
||||
|
||||
// PutWorld is what one step of a put reads.
|
||||
type PutWorld struct {
|
||||
Node, Module string
|
||||
Put delivery.Put
|
||||
// Assigned says the module stands assigned to the node.
|
||||
Assigned bool
|
||||
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
|
||||
Bus, Recorded bool
|
||||
// Build is the state of the builder's build at the put's fingerprint, "" when no key is written; BuildWords its error.
|
||||
Build, BuildWords string
|
||||
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
|
||||
Newest string
|
||||
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
|
||||
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
|
||||
Composed *delivery.ComposedDigest
|
||||
Granted bool
|
||||
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
|
||||
// that ask expired unanswered.
|
||||
Answer string
|
||||
AskExpired bool
|
||||
// Snapshot says where the bus's streams were snapshotted for this put, once they were; Snapshotting that it was
|
||||
// asked; SnapshotFailed why it was not taken.
|
||||
Snapshot string
|
||||
Snapshotting bool
|
||||
SnapshotFailed string
|
||||
// Have is what the node says it runs of the module; nil when it says nothing.
|
||||
Have *delivery.Have
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
|
||||
type PutNext struct {
|
||||
Put delivery.Put
|
||||
// Ask writes the put's ask about the move (PutAsk).
|
||||
Ask bool
|
||||
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
|
||||
Snapshot bool
|
||||
// Warranted records, once, in the controller's hand-act log that the bus's want was written on the operator's warrant.
|
||||
Warranted bool
|
||||
}
|
||||
|
||||
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
|
||||
func StepPut(w PutWorld) PutNext {
|
||||
p := w.Put
|
||||
if delivery.PutEnded(p.Step) {
|
||||
return PutNext{Put: p}
|
||||
}
|
||||
say := func(step, format string, args ...any) PutNext {
|
||||
return PutNext{Put: p.Said(w.Now, step, fmt.Sprintf(format, args...))}
|
||||
}
|
||||
fp := short(p.FP)
|
||||
if !w.Assigned {
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
|
||||
}
|
||||
switch w.Build {
|
||||
case delivery.BuildDone:
|
||||
case delivery.BuildFailed:
|
||||
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
|
||||
case "":
|
||||
if p.FP != w.Newest {
|
||||
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
|
||||
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
|
||||
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
|
||||
}
|
||||
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
|
||||
default:
|
||||
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
|
||||
}
|
||||
if w.Composed == nil || !w.Granted {
|
||||
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
|
||||
if w.Composed != nil {
|
||||
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
|
||||
}
|
||||
since := w.Now
|
||||
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
|
||||
since = p.Steps[len(p.Steps)-1].At
|
||||
}
|
||||
if w.Now.Sub(since) > ComposeWithin {
|
||||
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
|
||||
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
|
||||
}
|
||||
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
|
||||
}
|
||||
d := *w.Composed
|
||||
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
|
||||
return PutNext{Put: applied(w, p)}
|
||||
}
|
||||
p.Composed = d.Digest
|
||||
if needsPerson(w, d) {
|
||||
switch w.Answer {
|
||||
case OptionNo:
|
||||
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
|
||||
case OptionYes:
|
||||
default:
|
||||
if w.AskExpired {
|
||||
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
|
||||
}
|
||||
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
|
||||
if p.Ask == "" {
|
||||
next.Ask = true
|
||||
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
|
||||
}
|
||||
return next
|
||||
}
|
||||
if w.Bus {
|
||||
switch {
|
||||
case w.Snapshot != "":
|
||||
p.Snapshot = w.Snapshot
|
||||
case w.SnapshotFailed != "":
|
||||
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
|
||||
case w.Snapshotting:
|
||||
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
|
||||
default:
|
||||
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
|
||||
next.Snapshot = true
|
||||
return next
|
||||
}
|
||||
}
|
||||
}
|
||||
words := fmt.Sprintf("the want of %s on %s names %s (composition %s)", w.Module, w.Node, fp, short(d.Digest))
|
||||
if p.Hold {
|
||||
words += ", held until released"
|
||||
}
|
||||
if p.Snapshot != "" {
|
||||
words += "; the streams were snapshotted: " + p.Snapshot
|
||||
}
|
||||
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
|
||||
next.Warranted = w.Bus && p.Ask != ""
|
||||
return next
|
||||
}
|
||||
|
||||
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
|
||||
func applied(w PutWorld, p delivery.Put) delivery.Put {
|
||||
h := w.Have
|
||||
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
|
||||
return p
|
||||
}
|
||||
switch {
|
||||
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
|
||||
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
|
||||
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
|
||||
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
|
||||
case failed(*h):
|
||||
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
|
||||
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
|
||||
// container, move data, or that cannot be judged (§14).
|
||||
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
|
||||
runs := w.Have != nil && w.Have.FP == w.Put.FP
|
||||
if runs && w.Have.Composed == d.Digest {
|
||||
return false // the node runs this composition already: nothing moves
|
||||
}
|
||||
if w.Bus {
|
||||
return !(runs && d.UserListOnly)
|
||||
}
|
||||
if !w.Recorded {
|
||||
return false
|
||||
}
|
||||
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
|
||||
}
|
||||
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
|
||||
func PutWant(p delivery.Put) delivery.Want {
|
||||
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
}
|
||||
|
||||
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
|
||||
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
|
||||
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
|
||||
switch p.Step {
|
||||
case delivery.PutWanted, delivery.PutHolding:
|
||||
w := PutWant(p)
|
||||
return &w, true
|
||||
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
|
||||
return nil, false
|
||||
}
|
||||
return nil, true
|
||||
}
|
||||
|
||||
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
|
||||
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
|
||||
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
|
||||
func PutAsking(p delivery.Put) int {
|
||||
if n := p.At.UnixMilli(); n > 0 {
|
||||
return int(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
|
||||
func PutAct(node, module, fp, composed string) asks.Act {
|
||||
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
|
||||
}
|
||||
|
||||
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
|
||||
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
|
||||
// channel that proves who answered takes it (ADR 0259), never the view.
|
||||
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
|
||||
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
|
||||
what := map[delivery.Judgement]string{
|
||||
delivery.JudgedRecreates: "recreates a container",
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
delivery.JudgedCannot: "cannot be judged: " + d.Words,
|
||||
delivery.JudgedNeither: "recreates no container and moves no data",
|
||||
delivery.JudgedUnchanged: "changes nothing the node runs",
|
||||
}[d.Judgement]
|
||||
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
|
||||
expires := now.Add(7 * 24 * time.Hour)
|
||||
if bus {
|
||||
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
yes.Level, yes.Binds = asks.Approve, binds
|
||||
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
|
||||
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
|
||||
expires = now.Add(asks.ApproveLasts)
|
||||
}
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, PutAsking(p)),
|
||||
Headline: fmt.Sprintf("Put %s on %s?", module, node),
|
||||
Explanation: explanation,
|
||||
Options: []asks.Option{yes,
|
||||
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
|
||||
Who: asks.Operator,
|
||||
Expires: expires,
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
}, Asking: PutAsking(p)}, nil
|
||||
}
|
||||
@@ -1,296 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The seat mesh-placement (novox/hq ADR 0567), held by the assigner beside its wants: placement is its state
|
||||
// `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
|
||||
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
|
||||
|
||||
// Leased says whether the lease write of ADR 0547 has happened, after which the controller's assignments are read no
|
||||
// more (§8). Not before L3: until then the holder keeps the controller in step and refuses what its verbs cannot take.
|
||||
var Leased = false
|
||||
|
||||
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the
|
||||
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
|
||||
// digest.
|
||||
type ProposeIn struct {
|
||||
ID, By, Why string
|
||||
At time.Time
|
||||
Changes []delivery.PlacementProposalChange
|
||||
Current map[string]delivery.Placement
|
||||
Revisions map[string]uint64
|
||||
Held func(module string) bool
|
||||
Recorded func(node string) bool
|
||||
Seed bool
|
||||
ExportDigest string
|
||||
}
|
||||
|
||||
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
|
||||
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change the
|
||||
// controller's verbs cannot take before the lease write (§8), a change that does not fit its node, and what
|
||||
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
|
||||
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
|
||||
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
|
||||
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
|
||||
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
|
||||
Changes: in.Changes, Seed: in.Seed, ExportDigest: in.ExportDigest, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
|
||||
var findings []string
|
||||
if p.By == "" {
|
||||
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
|
||||
}
|
||||
if p.Why == "" {
|
||||
findings = append(findings, "a proposal says why: it is shown to the operator")
|
||||
}
|
||||
if len(in.Changes) == 0 {
|
||||
findings = append(findings, "a proposal changes something: it names no change")
|
||||
}
|
||||
for i, c := range in.Changes {
|
||||
if (c.Op == delivery.OpSeed) != in.Seed {
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1))
|
||||
continue
|
||||
}
|
||||
if !Leased {
|
||||
if why := c.BeforeLease(); why != "" {
|
||||
findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why))
|
||||
}
|
||||
}
|
||||
}
|
||||
laid, lay := delivery.LayChanges(in.Current, in.Changes)
|
||||
findings = append(findings, lay...)
|
||||
all := map[string]delivery.Placement{}
|
||||
for node, pl := range in.Current {
|
||||
all[node] = pl
|
||||
}
|
||||
out := map[string]*delivery.Placement{}
|
||||
for node, pl := range laid {
|
||||
if before, ok := in.Current[node]; ok && delivery.SamePlacement(before, pl) {
|
||||
continue
|
||||
}
|
||||
pl := pl
|
||||
all[node] = pl
|
||||
out[node] = &pl
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
if len(findings) == 0 && len(out) == 0 {
|
||||
findings = append(findings, "the changes leave every placement as it is")
|
||||
}
|
||||
held, recorded := in.Held, in.Recorded
|
||||
if held == nil {
|
||||
held = func(string) bool { return false }
|
||||
}
|
||||
if recorded == nil {
|
||||
recorded = func(string) bool { return false }
|
||||
}
|
||||
for _, r := range delivery.CheckPlacements(all, held, recorded) {
|
||||
findings = append(findings, r)
|
||||
}
|
||||
return p, out, findings
|
||||
}
|
||||
|
||||
// Write is one key a warrant writes (§5): the node, its key, the revision it is written at (0: it has none, so it is
|
||||
// created) and its value (nil: it is deleted).
|
||||
type Write struct {
|
||||
Node, Key string
|
||||
Base uint64
|
||||
Value *delivery.Placement
|
||||
}
|
||||
|
||||
// Writes is every key a proposal writes, by node order.
|
||||
func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) ([]Write, error) {
|
||||
var out []Write
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, Write{Node: node, Key: key, Base: p.Base[node], Value: laid[node]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ControllerCall is one call of the controller's seat verbs the holder makes after writing an approved proposal, until
|
||||
// the lease write (§8): assign, unassign, pin or unpin, with its arguments.
|
||||
type ControllerCall struct {
|
||||
Verb string
|
||||
Args map[string]any
|
||||
// Node and Module name the pair the call keeps in step (a pin names no module).
|
||||
Node, Module string
|
||||
}
|
||||
|
||||
// Key names the call, as the condition of its pair says it.
|
||||
func (c ControllerCall) Key() string {
|
||||
return fmt.Sprintf("mesh-controller.%s %s %s", c.Verb, c.Node, c.Module)
|
||||
}
|
||||
|
||||
// ControllerCalls is every call that brings the controller's assignments and pins to the placement after: assign for a
|
||||
// module placed on a node before not, unassign for one taken off, pin for a pin set or changed, unpin for one removed,
|
||||
// each naming the proposal and its warrant in its why. In node order, unassigns before assigns, so a module moved
|
||||
// between nodes is first taken off.
|
||||
func ControllerCalls(before map[string]delivery.Placement, after map[string]*delivery.Placement, proposal, warrant string) []ControllerCall {
|
||||
why := fmt.Sprintf("placement proposal %s, approved (%s)", proposal, warrant)
|
||||
nodes := make([]string, 0, len(after))
|
||||
for node := range after {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var off, on, pins []ControllerCall
|
||||
for _, node := range nodes {
|
||||
was, now := before[node], delivery.Placement{}
|
||||
if a := after[node]; a != nil {
|
||||
now = *a
|
||||
}
|
||||
had, has := map[string]bool{}, map[string]bool{}
|
||||
for _, m := range was.Modules {
|
||||
had[m.Module] = true
|
||||
}
|
||||
for _, m := range now.Modules {
|
||||
has[m.Module] = true
|
||||
if !had[m.Module] {
|
||||
on = append(on, ControllerCall{Verb: "assign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
|
||||
}
|
||||
}
|
||||
for _, m := range was.Modules {
|
||||
if !has[m.Module] {
|
||||
off = append(off, ControllerCall{Verb: "unassign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
|
||||
}
|
||||
}
|
||||
for _, provision := range sortedKeys(now.Pins) {
|
||||
to := now.Pins[provision]
|
||||
if old, ok := was.Pins[provision]; !ok || old != to {
|
||||
pins = append(pins, ControllerCall{Verb: "pin", Node: node, Module: provision,
|
||||
Args: map[string]any{"node": node, "provision": provision, "from": to.Node, "module": to.Module}})
|
||||
}
|
||||
}
|
||||
for _, provision := range sortedKeys(was.Pins) {
|
||||
if _, ok := now.Pins[provision]; !ok {
|
||||
pins = append(pins, ControllerCall{Verb: "unpin", Node: node, Module: provision, Args: map[string]any{"node": node, "provision": provision}})
|
||||
}
|
||||
}
|
||||
}
|
||||
out := append(append(off, on...), pins...)
|
||||
for i := range out {
|
||||
out[i].Args["why"] = why
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
|
||||
func PlacementAct(p delivery.PlacementProposal) asks.Act {
|
||||
return asks.Act{"placement": "write", "proposal": p.ID, "changes": delivery.ChangesDigest(p)}
|
||||
}
|
||||
|
||||
// PlacementAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
|
||||
// another asking, under an id of its own.
|
||||
func PlacementAsking(p delivery.PlacementProposal, at time.Time) int {
|
||||
return int(at.Unix())
|
||||
}
|
||||
|
||||
// PlacementAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
|
||||
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
|
||||
func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking int, now time.Time) (delivery.AssignerAsk, error) {
|
||||
binds, err := asks.ActDigest(PlacementAct(p))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
nodes := p.Nodes()
|
||||
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
|
||||
if p.Seed {
|
||||
headline = "Take placement over from the controller?"
|
||||
}
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
|
||||
}
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: headline,
|
||||
Explanation: brief,
|
||||
Options: []asks.Option{
|
||||
{ID: OptionYes, Label: "Approve", Does: "the placement of " + strings.Join(nodes, ", ") + " is written as proposed",
|
||||
Level: asks.Approve, Binds: binds},
|
||||
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(asks.ApproveLasts),
|
||||
OnExpiry: "nothing is written, and the proposal ends",
|
||||
About: key,
|
||||
}, Asking: asking}, nil
|
||||
}
|
||||
|
||||
// ExportOf reads the controller's answer to placement-export (§7): its verb answers what the command printed, the
|
||||
// JSON of every node's placement, in `output` (the controller's verbAnswer; `answer` when it carries it as data). A
|
||||
// refused or unreadable answer is an error naming it.
|
||||
func ExportOf(raw []byte) (map[string]delivery.Placement, error) {
|
||||
var a struct {
|
||||
Output string `json:"output"`
|
||||
OK *bool `json:"ok"`
|
||||
Answer json.RawMessage `json:"answer"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export answered no verb answer: %w", err)
|
||||
}
|
||||
if a.OK != nil && !*a.OK {
|
||||
return nil, fmt.Errorf("the controller's placement-export refused: %s", strings.TrimSpace(a.Output))
|
||||
}
|
||||
var byNode map[string]json.RawMessage
|
||||
if len(a.Answer) > 0 && string(a.Answer) != "null" {
|
||||
if err := json.Unmarshal(a.Answer, &byNode); err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export is no placement by node: %w", err)
|
||||
}
|
||||
} else if err := json.NewDecoder(strings.NewReader(a.Output)).Decode(&byNode); err != nil {
|
||||
// The command prints its JSON first; what its standard error said after it is left.
|
||||
return nil, fmt.Errorf("the controller's placement-export printed no placement by node: %w", err)
|
||||
}
|
||||
out := map[string]delivery.Placement{}
|
||||
for node, rawP := range byNode {
|
||||
p, err := delivery.DecodePlacement(rawP)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export of %s cannot be read: %w", node, err)
|
||||
}
|
||||
if p.Node != node {
|
||||
return nil, fmt.Errorf("the controller's placement-export names %s under %s", p.Node, node)
|
||||
}
|
||||
out[node] = p
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("the controller's placement-export names no node")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SeedChanges is the seed's changes: every node's placement of the export, whole, in node order.
|
||||
func SeedChanges(export map[string]delivery.Placement) []delivery.PlacementProposalChange {
|
||||
var out []delivery.PlacementProposalChange
|
||||
for _, node := range sortedKeys(export) {
|
||||
p := export[node]
|
||||
out = append(out, delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: node, Whole: &p})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
|
||||
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
|
||||
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
|
||||
rest := p
|
||||
rest.Changes, rest.Base = nil, map[string]uint64{}
|
||||
for _, c := range p.Changes {
|
||||
if _, done := written[c.Node]; !done {
|
||||
rest.Changes = append(rest.Changes, c)
|
||||
}
|
||||
}
|
||||
for node, base := range p.Base {
|
||||
if _, done := written[node]; !done {
|
||||
rest.Base[node] = base
|
||||
}
|
||||
}
|
||||
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", delivery.ProposalAnswering
|
||||
return rest, len(rest.Changes) > 0
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0567 §2, §5, §7, §8: what the seat's holder decides of a proposal: every refusal named at once, a valid one laid
|
||||
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
|
||||
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
|
||||
func TestThePlacementSeatDecides(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Decided: delivery.NodeDecided{Hub: true, Address: "10.0.0.1"},
|
||||
Modules: []delivery.PlacedModule{{Module: "store", Ports: map[string]int{"5432": 5432}}, {Module: "web"}}}
|
||||
laptop := delivery.Placement{V: delivery.Version, Node: "laptop", Decided: delivery.NodeDecided{Address: "10.0.0.2"}, Modules: []delivery.PlacedModule{{Module: "shell"}}}
|
||||
in := func(changes ...delivery.PlacementProposalChange) ProposeIn {
|
||||
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
|
||||
Current: map[string]delivery.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
|
||||
Held: func(m string) bool { return m != "ghost" },
|
||||
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
|
||||
}
|
||||
move := []delivery.PlacementProposalChange{{Op: delivery.OpTakeOff, Node: "anchor", Module: "web"}, {Op: delivery.OpPlace, Node: "laptop", Module: "web"}}
|
||||
p, laid, findings := Propose(in(move...))
|
||||
if len(findings) != 0 || p.State != delivery.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
|
||||
!slices.Equal(laid["laptop"].Assigned(), []string{"shell", "web"}) || !slices.Equal(laid["anchor"].Assigned(), []string{"store"}) {
|
||||
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
|
||||
}
|
||||
if _, err := delivery.EncodePlacementProposal(p); err != nil {
|
||||
t.Fatalf("the proposal cannot be written: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
in ProposeIn
|
||||
want string
|
||||
}{
|
||||
"a port before the lease": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}}), "lease write"},
|
||||
"a holding": {in(delivery.PlacementProposalChange{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "mesh-store", Scope: "mesh"}), "lease write"},
|
||||
"a binding": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}}), "lease write"},
|
||||
"a config place": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}}), "lease write"},
|
||||
"a config access": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}}), "lease write"},
|
||||
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
|
||||
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
|
||||
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
|
||||
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
|
||||
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
|
||||
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "adopt's alone"},
|
||||
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
|
||||
} {
|
||||
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
|
||||
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
// A module's config values may be proposed before the lease write (§13); once the lease is written every change is
|
||||
// the state's alone, and none waits.
|
||||
if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 {
|
||||
t.Errorf("config values before the lease write: %v", f)
|
||||
}
|
||||
Leased = true
|
||||
for _, c := range []delivery.PlacementProposalChange{
|
||||
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
|
||||
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
|
||||
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
|
||||
} {
|
||||
if _, _, f := Propose(in(c)); len(f) != 0 {
|
||||
t.Errorf("after the lease write %s is refused: %v", c.Op, f)
|
||||
}
|
||||
}
|
||||
Leased = false
|
||||
|
||||
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
|
||||
if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
|
||||
len(rest.Base) != 1 || rest.State != delivery.ProposalAnswering || rest.Must != nil {
|
||||
t.Errorf("the rest = %+v, %v", rest, ok)
|
||||
}
|
||||
if _, ok := Rest(p, laid); ok {
|
||||
t.Error("a proposal written whole leaves something to write")
|
||||
}
|
||||
|
||||
// The seat says it waits while one of its states is not bound, naming it, and clears once all are.
|
||||
for _, unbound := range [][]string{{delivery.PlacementPulsesBucket}, nil} {
|
||||
c := SeatWaitsCondition(unbound)
|
||||
if _, err := asks.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == asks.StateRaised) ||
|
||||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+delivery.PlacementPulsesBucket)) {
|
||||
t.Errorf("waiting for %v: %+v, %v", unbound, c, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126: the seed places the private network, which no repository holds; it is the mesh's own, and passes.
|
||||
wired := laptop
|
||||
wired.Modules = append(slices.Clone(laptop.Modules), delivery.PlacedModule{Module: "mesh-wireguard"})
|
||||
seed := in(SeedChanges(map[string]delivery.Placement{"laptop": wired})...)
|
||||
seed.Seed, seed.ExportDigest, seed.Current, seed.Revisions = true, "sha256:x", map[string]delivery.Placement{}, map[string]uint64{}
|
||||
seed.Held = func(m string) bool { return m != "mesh-wireguard" } // the indexer holds every module but it
|
||||
if _, _, f := Propose(seed); len(f) != 0 {
|
||||
t.Errorf("a seed placing the private network is refused: %v", f)
|
||||
}
|
||||
|
||||
writes, err := Writes(p, laid)
|
||||
if err != nil || len(writes) != 2 || writes[0].Key != "placement.anchor" || writes[0].Base != 4 || writes[1].Base != 9 {
|
||||
t.Fatalf("writes = %+v, %v", writes, err)
|
||||
}
|
||||
pinned := *laid["laptop"]
|
||||
pinned.Pins = map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
|
||||
calls := ControllerCalls(map[string]delivery.Placement{"anchor": anchor, "laptop": laptop},
|
||||
map[string]*delivery.Placement{"anchor": laid["anchor"], "laptop": &pinned}, "p-one", "w1@t")
|
||||
var said []string
|
||||
for _, c := range calls {
|
||||
said = append(said, c.Key())
|
||||
if !strings.Contains(c.Args["why"].(string), "p-one") {
|
||||
t.Errorf("%s does not name the proposal: %v", c.Key(), c.Args)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(said, []string{"mesh-controller.unassign anchor web", "mesh-controller.assign laptop web", "mesh-controller.pin laptop db"}) {
|
||||
t.Errorf("the controller's calls = %v", said)
|
||||
}
|
||||
|
||||
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
|
||||
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
}
|
||||
q := p
|
||||
q.Changes = append(slices.Clone(p.Changes), delivery.PlacementProposalChange{Op: delivery.OpUnpin, Node: "laptop", Provision: "x"})
|
||||
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
|
||||
t.Error("the ask's yes binds changes it was not asked about")
|
||||
}
|
||||
|
||||
export, _ := json.Marshal(map[string]any{"anchor": anchor, "laptop": laptop})
|
||||
for name, c := range map[string]struct {
|
||||
raw string
|
||||
want int
|
||||
}{
|
||||
"in output": {`{"ok":true,"output":` + quote(string(export)+"\n") + `}`, 2},
|
||||
"as answer": {`{"ok":true,"output":"","answer":` + string(export) + `}`, 2},
|
||||
"refused": {`{"ok":false,"output":"no store"}`, 0},
|
||||
"no node": {`{"ok":true,"output":"{}"}`, 0},
|
||||
"no version": {`{"ok":true,"output":"{\"anchor\":{\"node\":\"anchor\",\"modules\":[]}}"}`, 0},
|
||||
} {
|
||||
got, err := ExportOf([]byte(c.raw))
|
||||
if len(got) != c.want || (c.want == 0) != (err != nil) {
|
||||
t.Errorf("%s: %d nodes, %v", name, len(got), err)
|
||||
}
|
||||
}
|
||||
if seed := SeedChanges(map[string]delivery.Placement{"laptop": laptop, "anchor": anchor}); len(seed) != 2 || seed[0].Node != "anchor" || seed[0].Op != delivery.OpSeed {
|
||||
t.Errorf("the seed's changes = %+v", seed)
|
||||
}
|
||||
}
|
||||
|
||||
func quote(s string) string {
|
||||
raw, _ := json.Marshal(s)
|
||||
return string(raw)
|
||||
}
|
||||
@@ -2,11 +2,8 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -16,50 +13,13 @@ import (
|
||||
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
|
||||
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
|
||||
type Module struct {
|
||||
Name string
|
||||
FP string
|
||||
Built bool
|
||||
Held bool
|
||||
Granted bool
|
||||
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
|
||||
// that runs an older one names it (issue 1945). Nil: none.
|
||||
GrantedAt func(fp string) bool
|
||||
// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
|
||||
// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
|
||||
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
|
||||
GrantedOn func(node string) bool
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
Allowed func(node, digest string) bool
|
||||
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
|
||||
Released bool
|
||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
||||
Restored func(node string) bool
|
||||
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
|
||||
Kept bool
|
||||
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
|
||||
Now time.Time
|
||||
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
||||
Puts map[string]delivery.Put
|
||||
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
|
||||
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
|
||||
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
|
||||
Self func(node string) bool
|
||||
}
|
||||
|
||||
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
|
||||
|
||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if m.Composed == nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return m.Composed(node, fp)
|
||||
Name string
|
||||
FP string
|
||||
Built bool
|
||||
Held bool
|
||||
Granted bool
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
}
|
||||
|
||||
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
|
||||
@@ -91,43 +51,27 @@ func Canary(nodes []string, online map[string]bool) string {
|
||||
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
||||
// limit (ADR 0528 clause 3).
|
||||
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||
return holds(m, node, n, true)
|
||||
}
|
||||
|
||||
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
|
||||
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
|
||||
// composition.
|
||||
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
switch {
|
||||
case !m.Built:
|
||||
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
|
||||
case m.Held:
|
||||
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
|
||||
}
|
||||
if ordering {
|
||||
canary := Canary(m.Nodes, n.Online)
|
||||
switch {
|
||||
case canary == "":
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return false, delivery.RuleProviders, why
|
||||
}
|
||||
canary := Canary(m.Nodes, n.Online)
|
||||
switch {
|
||||
case canary == "":
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return false, delivery.RuleProviders, why
|
||||
}
|
||||
}
|
||||
if !m.Granted {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||
}
|
||||
if m.GrantedOn != nil && !m.GrantedOn(node) {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
|
||||
short(m.FP))
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||
}
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
@@ -173,90 +117,31 @@ type Outcome struct {
|
||||
Wants map[string]delivery.Want
|
||||
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
||||
Held *delivery.Held
|
||||
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
||||
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
||||
Lifted bool
|
||||
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
|
||||
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
|
||||
NodeFailed map[string]NodeFailure
|
||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||
Record []RecordNeed
|
||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||
// back: the operator is asked whether to restore each (ADR 0539 §19).
|
||||
Restore []string
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why.
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why, for the comparison.
|
||||
Rules map[string]Holding
|
||||
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
|
||||
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
|
||||
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
|
||||
// own composition and is never here.
|
||||
Neither map[string]Holding
|
||||
}
|
||||
|
||||
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
|
||||
// controller's judgement of the move and its words.
|
||||
type RecordNeed struct {
|
||||
Node string
|
||||
Digest string
|
||||
Judgement delivery.Judgement
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then
|
||||
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
|
||||
// running an old-form build, got no want and the pass said nothing).
|
||||
// Holding is a rule holding a node back, and why.
|
||||
type Holding struct {
|
||||
Rule int
|
||||
Words string
|
||||
Bus bool
|
||||
}
|
||||
|
||||
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
|
||||
func (h Holding) Reason() string {
|
||||
if h.Bus {
|
||||
return h.Words
|
||||
}
|
||||
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
|
||||
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
|
||||
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
|
||||
func BusBarWords(node, prev, fp string) string {
|
||||
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
|
||||
}
|
||||
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
|
||||
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
|
||||
// module holding data is not set back but asked about (Restore). A
|
||||
// module holding data is not set back (in release 5 it is asked about from release 6 on: wants are only compared). A
|
||||
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
||||
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
||||
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
|
||||
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
|
||||
held := m.Held
|
||||
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
|
||||
out.Lifted, held = true, false
|
||||
}
|
||||
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
||||
if !held && m.FP != "" {
|
||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || h.FP != m.FP || !failed(h) {
|
||||
continue
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
|
||||
out.Held = &delivery.Held{Node: node, Words: h.Detail}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
|
||||
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
|
||||
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
|
||||
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
|
||||
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
|
||||
// §2, ADR 0429 §6).
|
||||
if out.NodeFailed == nil {
|
||||
out.NodeFailed = map[string]NodeFailure{}
|
||||
}
|
||||
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
|
||||
continue
|
||||
}
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
}
|
||||
mm := m
|
||||
@@ -267,167 +152,32 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
before, had := wants[node]
|
||||
var next delivery.Want
|
||||
p, put := m.Puts[node]
|
||||
if !put && had && before.Put && before.Hold {
|
||||
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
|
||||
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
|
||||
// without hold is not kept this way: its put takes it further once heard.
|
||||
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
|
||||
Step: delivery.PutHolding}, true
|
||||
}
|
||||
if put {
|
||||
if want, owns := PutOwns(p); owns {
|
||||
if want != nil {
|
||||
w := *want
|
||||
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
|
||||
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
|
||||
w.Composed = ul.Composed
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
||||
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
||||
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
||||
if w, ok := m.userListRevision(node, before, had, n); ok {
|
||||
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
||||
out.Wants[node] = w
|
||||
}
|
||||
continue
|
||||
}
|
||||
ok, rule, words := Holds(mm, node, n)
|
||||
target := m.FP
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
|
||||
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
|
||||
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
|
||||
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
|
||||
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
|
||||
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
|
||||
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
|
||||
// order the move from it as from any want.
|
||||
switch _, composed := m.composition(node, have.FP); {
|
||||
case !ran:
|
||||
case have.FP == m.FP:
|
||||
ok, rule, words = holds(mm, node, n, false)
|
||||
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
|
||||
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
|
||||
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
|
||||
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
|
||||
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
|
||||
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
|
||||
ok, target = true, have.FP
|
||||
}
|
||||
}
|
||||
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
|
||||
n.Haves[node][m.Name].FP != m.FP {
|
||||
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
|
||||
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
|
||||
// it would be on neither list and its node-engine would give the module up (issue 1945).
|
||||
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
|
||||
}
|
||||
switch {
|
||||
switch ok, rule, words := Holds(mm, node, n); {
|
||||
case ok:
|
||||
d, _ := m.composition(node, target)
|
||||
next = delivery.Want{FP: target, Composed: d.Digest}
|
||||
at := m
|
||||
if target != m.FP {
|
||||
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
|
||||
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||
at.FP = target
|
||||
}
|
||||
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
// A removed want is no want: a first want after it goes through the record ask as a first want does.
|
||||
asked, hadLive := before, had
|
||||
if before.Removed {
|
||||
asked, hadLive = delivery.Want{}, false
|
||||
}
|
||||
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
|
||||
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
|
||||
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
|
||||
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
|
||||
out.Rules[node] = h
|
||||
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
|
||||
out.Neither[node] = h
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
next = delivery.Want{FP: m.FP}
|
||||
case held && had && before.FP == m.FP:
|
||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||
last := lastHealthy[node]
|
||||
switch {
|
||||
case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
|
||||
case keepsData(m.Manifest):
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
|
||||
out.Restore = append(out.Restore, node)
|
||||
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked from release 6", m.Name, short(m.FP))
|
||||
case last == "":
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
|
||||
default:
|
||||
d, there := m.composition(node, last)
|
||||
if !there {
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
|
||||
break
|
||||
}
|
||||
next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
|
||||
next = delivery.Want{FP: last, Newest: m.FP, Rule: rule, Words: words}
|
||||
}
|
||||
default:
|
||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||
if !had || before.Removed {
|
||||
// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
|
||||
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
||||
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
||||
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
||||
h, there := n.Haves[node][m.Name]
|
||||
switch {
|
||||
case there && h.Phase != delivery.PhaseRemoved && !held:
|
||||
out.Neither[node] = Holding{Rule: rule, Words: words}
|
||||
case had && (!there || h.Phase == delivery.PhaseRemoved):
|
||||
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
|
||||
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
|
||||
// would have its node agent remove the module and stop holding its composition's resources back.
|
||||
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
|
||||
if !delivery.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
|
||||
next = delivery.Want{FP: before.FP, Newest: m.FP, Rule: rule, Words: words}
|
||||
if before.FP == m.FP {
|
||||
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
|
||||
next = delivery.Want{FP: m.FP, Composed: before.Composed}
|
||||
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
|
||||
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
|
||||
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
|
||||
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
|
||||
at := m
|
||||
at.FP = before.FP
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
if !wait {
|
||||
next.Composed = d.Digest
|
||||
}
|
||||
next = delivery.Want{FP: m.FP}
|
||||
}
|
||||
}
|
||||
next, write := m.selfDerived(node, before, had, next, n, &out)
|
||||
if !write {
|
||||
continue
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
@@ -440,137 +190,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
return out
|
||||
}
|
||||
|
||||
// selfDerived is next with its Self derived from the placement now (Module.Self), whichever rule made it, and whether it is
|
||||
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
|
||||
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
|
||||
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
|
||||
// placement seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
|
||||
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
|
||||
// while its ask waits, the want written stays and the node's holding says why.
|
||||
func (m Module) selfDerived(node string, before delivery.Want, had bool, next delivery.Want, n Nodes, out *Outcome) (delivery.Want, bool) {
|
||||
switch {
|
||||
case next.Removed:
|
||||
return next, true
|
||||
case m.selfOn(node):
|
||||
next.Self, next.Composed = true, ""
|
||||
return next, true
|
||||
case !next.Self && next.Composed != "":
|
||||
return next, true
|
||||
}
|
||||
next.Self, next.Yes = false, ""
|
||||
d, there := m.composition(node, next.FP)
|
||||
if !there || d.Digest == "" {
|
||||
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
|
||||
"and the controller has not composed it at %s, so its want stays", m.Name, node, short(next.FP))}
|
||||
return next, false
|
||||
}
|
||||
at := m
|
||||
at.FP = next.FP
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
return next, false
|
||||
}
|
||||
next.Composed = d.Digest
|
||||
return next, true
|
||||
}
|
||||
|
||||
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
|
||||
func NoWantWords(rule int, words string) string {
|
||||
return fmt.Sprintf("no want: rule %d: %s", rule, words)
|
||||
}
|
||||
|
||||
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
||||
// moving). A failed health while it
|
||||
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
||||
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
|
||||
// the world's lastHealthy) already asks for phase running.
|
||||
func failed(h delivery.Have) bool {
|
||||
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
|
||||
}
|
||||
|
||||
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
|
||||
func healthyElsewhere(m Module, n Nodes, node string) []string {
|
||||
var out []string
|
||||
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
|
||||
// healthy now.
|
||||
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
|
||||
out = append(out, other)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
|
||||
// the hold and the set-back stand at the second failure (ADR 0557 §2).
|
||||
func failedElsewhere(m Module, n Nodes, node string) bool {
|
||||
for _, other := range m.Nodes {
|
||||
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
|
||||
func FailedAloneWords(module, fp string, failing []string) string {
|
||||
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
|
||||
strings.Join(failing, ", "))
|
||||
}
|
||||
|
||||
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
|
||||
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
|
||||
type NodeFailure struct {
|
||||
Others []string
|
||||
Words string
|
||||
}
|
||||
|
||||
// runs says whether a have is of a module the node runs: running, whatever its health.
|
||||
func runs(h delivery.Have) bool {
|
||||
return h.Phase == delivery.PhaseRunning
|
||||
}
|
||||
|
||||
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
|
||||
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
|
||||
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
|
||||
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
|
||||
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
|
||||
func allRunHealthy(m Module, n Nodes) bool {
|
||||
ran := 0
|
||||
for _, node := range m.Nodes {
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
|
||||
continue
|
||||
}
|
||||
if !healthy(h, m.FP) {
|
||||
return false
|
||||
}
|
||||
ran++
|
||||
}
|
||||
return ran > 0
|
||||
}
|
||||
|
||||
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
|
||||
// it, and the node's own words. The node is the held record's own field.
|
||||
func HeldWords(h delivery.Have, at time.Time) string {
|
||||
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
|
||||
if !at.IsZero() {
|
||||
words += ", seen at " + at.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if d := strings.TrimSpace(h.Detail); d != "" {
|
||||
return words + ": " + d
|
||||
}
|
||||
return words + ": the node gave no words"
|
||||
}
|
||||
|
||||
// LiftedWords say why a hold was lifted by itself.
|
||||
func LiftedWords(module, fp string) string {
|
||||
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
|
||||
}
|
||||
|
||||
// normal is a want without its version, as a decoded and a composed one are compared.
|
||||
func normal(w delivery.Want) delivery.Want {
|
||||
w.V = 0
|
||||
@@ -584,78 +203,3 @@ func short(fp string) string {
|
||||
}
|
||||
return part[:12]
|
||||
}
|
||||
|
||||
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
|
||||
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
|
||||
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
|
||||
// same fingerprint judged neither (a user list only, §7).
|
||||
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
|
||||
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
|
||||
return nil, false
|
||||
}
|
||||
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
|
||||
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
|
||||
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
|
||||
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
if _, barred := m.busBar(before, had, running); barred {
|
||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
if m.Allowed != nil && m.Allowed(node, d.Digest) {
|
||||
return nil, false
|
||||
}
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
}
|
||||
|
||||
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
|
||||
// readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
|
||||
prev = before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
||||
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||
if d.Policy != "" {
|
||||
return d.Policy == delivery.PolicyRecord
|
||||
}
|
||||
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
|
||||
return policy == catalogue.PolicyRecord
|
||||
}
|
||||
|
||||
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
|
||||
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
|
||||
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
|
||||
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
|
||||
if !catalogue.ProvidesBus(m.Manifest) {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
d, ok := m.composition(node, running.FP)
|
||||
if !ok || !d.UserListOnly {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
w := delivery.Want{FP: running.FP, Composed: d.Digest}
|
||||
if had && before.Composed == d.Digest {
|
||||
w.Backup, w.Words = before.Backup, before.Words
|
||||
}
|
||||
return w, true
|
||||
}
|
||||
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
|
||||
// put reads by too).
|
||||
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
|
||||
return g.NamesNode(node, module)
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func load(t *testing.T) catalogue.Manifest {
|
||||
@@ -25,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest {
|
||||
|
||||
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
|
||||
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
|
||||
// the states it declares (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||
// its three states, and holds a credential of its own.
|
||||
func TestManifest(t *testing.T) {
|
||||
m := load(t)
|
||||
if m.Module != "mesh-delivery-assigner" {
|
||||
@@ -33,9 +32,9 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
@@ -43,8 +42,7 @@ func TestManifest(t *testing.T) {
|
||||
for _, s := range m.State {
|
||||
states[s.Name] = s.History
|
||||
}
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20,
|
||||
"placement": 64, "proposals": 5, "candidates": 5, "pulses": 0}; !reflect.DeepEqual(states, want) {
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||
@@ -53,36 +51,11 @@ func TestManifest(t *testing.T) {
|
||||
if m.SecretsOwner != "mesh-delivery-assigner" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the
|
||||
// checker puts its checks.
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||
// 1622: the assigner puts its compose asks on the Go build seat, as the checker puts its checks.
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go", m.Uses)
|
||||
}
|
||||
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
|
||||
// module), says each put's step as the event put, and calls the controller's warranted and the bus node's backup holder.
|
||||
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its seven verbs; keeps the seat's events on its stream
|
||||
// placement for 30 days; calls the controller's export, its four verbs that keep its assignments in step and its
|
||||
// hand-act log; and reads mesh-config's resolved values to compose every node a proposal touches.
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) {
|
||||
t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs", m.Claims)
|
||||
}
|
||||
if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat))
|
||||
}
|
||||
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||
t.Errorf("the assigner serves tools %v, want its three put tools", m.Tools)
|
||||
}
|
||||
if len(m.Streams) != 1 || m.Streams[0].Name != delivery.PlacementStream || m.Streams[0].MaxAgeDays != delivery.PlacementStreamDays ||
|
||||
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
|
||||
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
|
||||
}
|
||||
if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.hand-act", "seat:mesh-controller.pin",
|
||||
"seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin",
|
||||
"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if !slices.Equal(m.Emits, []string{"put"}) {
|
||||
t.Errorf("emits = %v, want put", m.Emits)
|
||||
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
|
||||
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,82 +2,27 @@
|
||||
"module": "mesh-delivery-assigner",
|
||||
"version": "1",
|
||||
"slug": "assigner",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-placement",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"placement",
|
||||
"proposals",
|
||||
"propose",
|
||||
"withdraw",
|
||||
"again",
|
||||
"adopt",
|
||||
"hand-back"
|
||||
]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-config.resolved",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"node-engine.have"
|
||||
],
|
||||
"uses": [
|
||||
"node-build-go",
|
||||
"operator-channel"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.assign",
|
||||
"seat:mesh-controller.hand-act",
|
||||
"seat:mesh-controller.pin",
|
||||
"seat:mesh-controller.placement-export",
|
||||
"seat:mesh-controller.unassign",
|
||||
"seat:mesh-controller.unpin",
|
||||
"seat:mesh-controller.warranted",
|
||||
"seat:node-backup.backed-up",
|
||||
"seat:node-backup.now"
|
||||
],
|
||||
"emits": [
|
||||
"put"
|
||||
],
|
||||
"tools": [
|
||||
"assigner_put",
|
||||
"assigner_release",
|
||||
"assigner_puts"
|
||||
"node-build-go"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "placement", "history": 64 },
|
||||
{ "name": "proposals", "history": 5 },
|
||||
{ "name": "candidates", "history": 5 },
|
||||
{ "name": "pulses", "key-ttl": true },
|
||||
{ "name": "asks", "history": 5 },
|
||||
{ "name": "compositions", "history": 5 },
|
||||
{ "name": "differences", "history": 64 },
|
||||
{ "name": "puts", "history": 20 },
|
||||
{ "name": "wants", "history": 20 }
|
||||
],
|
||||
"streams": [
|
||||
{
|
||||
"name": "placement",
|
||||
"events": [
|
||||
"placement-proposed",
|
||||
"placement-judged",
|
||||
"placement-changed",
|
||||
"placement-withdrawn"
|
||||
],
|
||||
"max-age-days": 30
|
||||
}
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
@@ -123,10 +68,7 @@
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-assigner",
|
||||
"binary": "mesh-delivery-assigner",
|
||||
"loads": [
|
||||
"mesh-delivery-assigner"
|
||||
]
|
||||
"binary": "mesh-delivery-assigner"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,229 +0,0 @@
|
||||
// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
|
||||
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
|
||||
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
|
||||
// other program: it serves no tool (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// queueAsker writes a queues value and waits for the builder's answer to it.
|
||||
type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
||||
|
||||
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
||||
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: build.StuckTool,
|
||||
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
||||
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
||||
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
|
||||
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
||||
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
||||
{Name: build.AskAgainTool,
|
||||
Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
|
||||
"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
|
||||
"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
|
||||
"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := askAgain(key, value); err != nil {
|
||||
return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
|
||||
}
|
||||
return map[string]any{"key": key, "asked": true,
|
||||
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
|
||||
}},
|
||||
{Name: build.BuildAgainTool,
|
||||
Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
|
||||
"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
|
||||
"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
|
||||
"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
|
||||
"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
|
||||
"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := buildAgain(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Refused {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Failed != "" {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Revision == 0 && !r.Pending {
|
||||
return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
|
||||
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
|
||||
}},
|
||||
{Name: build.SeatQueueTool,
|
||||
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
|
||||
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
|
||||
"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
|
||||
"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
|
||||
"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
|
||||
"seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
|
||||
"limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
language := build.LanguageOf(str(args, "language"))
|
||||
if language == "" {
|
||||
language = build.LanguageOf(str(args, "seat"))
|
||||
}
|
||||
if language == "" {
|
||||
language = delivery.AskGo
|
||||
}
|
||||
limit, _ := args["limit"].(float64)
|
||||
key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return queue(key, value)
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
|
||||
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
||||
return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
|
||||
a, err := build.DecodeQueueAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
}
|
||||
|
||||
// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
|
||||
// bound for a value written after it that answered says is the builder's answer.
|
||||
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
|
||||
return func(key string, value []byte) (json.RawMessage, error) {
|
||||
heard := make(chan json.RawMessage, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
mu.Lock()
|
||||
mine := rev
|
||||
mu.Unlock()
|
||||
if c.Op != "put" || mine == 0 || c.Revision <= mine {
|
||||
return nil
|
||||
}
|
||||
if answered(c.Value) {
|
||||
select {
|
||||
case heard <- c.Value:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
// The watch may have heard the answer before the revision was known: read it once too.
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
|
||||
return e.Value, nil
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case v := <-heard:
|
||||
return v, nil
|
||||
case <-wait.Done():
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func read(s stdio.KeptState) build.StateRead {
|
||||
return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
return e.Value, time.Time{}, nil
|
||||
}, Written: func(key string) time.Time {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return written(s, key, e.Revision)
|
||||
}}
|
||||
}
|
||||
|
||||
// written is when the key's revision was written, from its history, or zero.
|
||||
func written(s stdio.KeptState, key string, revision uint64) time.Time {
|
||||
h, err := s.History(key)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, r := range h {
|
||||
if r.Revision == revision {
|
||||
return r.At
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func main() {
|
||||
steps := map[string]build.StateRead{}
|
||||
for _, bm := range build.StuckBuildModules {
|
||||
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||
}
|
||||
reasks := stdio.State(build.ReAsksState)
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
|
||||
Hearing: read(stdio.State(buildrun.HearingState))}
|
||||
put := func(key string, value []byte) error {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
|
||||
a, err := build.DecodeReAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, time.Now)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -66,11 +66,6 @@ func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyV
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return settled(ctx, w, each)
|
||||
}
|
||||
|
||||
// settled hands each entry the watch delivers before its end-of-initial marker to each, and answers the watch.
|
||||
func settled(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
||||
@@ -4,303 +4,73 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
|
||||
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
|
||||
// be answered (issue 1945). Never polled.
|
||||
// (issue 1910): read once, at the take-over of a key at start, never polled.
|
||||
type heldOnTheBus struct{ js jetstream.JetStream }
|
||||
|
||||
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
|
||||
// the ask's subject whose Nats-Msg-Id is the ask's id.
|
||||
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
|
||||
seen, err := h.seen(ctx, a, false)
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return seen.Step || seen.OnStream, nil
|
||||
}
|
||||
|
||||
// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "".
|
||||
func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) {
|
||||
seen, err := h.seen(ctx, a, true)
|
||||
pattern, err := delivery.AskStepWatch(a.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return seen.Why(), nil
|
||||
}
|
||||
|
||||
// maxOnSubject bounds the messages of one subject a read walks (each a read by sequence, next by subject), and maxSteps
|
||||
// the steps it reads whole to find the last ask taken: past them the read says it cannot tell, and the ask is waited for.
|
||||
const (
|
||||
maxOnSubject = 5000
|
||||
maxSteps = 200
|
||||
)
|
||||
|
||||
// seen reads what the bus holds of the ask; with later, the ask asked last that a build node took too (AskSeen.Last), when
|
||||
// the ask is on its stream.
|
||||
func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) {
|
||||
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
return false, err
|
||||
}
|
||||
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
pattern, err := delivery.AskStepWatch(a.ID)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
keys, err := steps.ListKeysFiltered(ctx, pattern)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
return false, err
|
||||
}
|
||||
for range keys.Keys() {
|
||||
keys.Stop()
|
||||
seen.Step = true // a node took it
|
||||
return seen, nil
|
||||
}
|
||||
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||
seen.Stream = name
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return seen, nil
|
||||
return true, nil // a node took it
|
||||
}
|
||||
subject, err := delivery.AskSubject(a.Language, a.Kind)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
var ahead []build.Queued
|
||||
err = onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||
if q.Ask == a.ID {
|
||||
seen.OnStream, seen.PutAt = true, q.PutAt
|
||||
return false
|
||||
}
|
||||
ahead = append(ahead, q)
|
||||
return true
|
||||
})
|
||||
if err != nil || !seen.OnStream || !later {
|
||||
return seen, err
|
||||
}
|
||||
taken, err := stepKeys(ctx, steps)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
for i := range ahead {
|
||||
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||
}
|
||||
seen.Ahead = ahead
|
||||
since := seen.At
|
||||
if since.IsZero() {
|
||||
since = seen.PutAt
|
||||
}
|
||||
for _, q := range ahead {
|
||||
if q.At.Before(since) {
|
||||
since = q.At
|
||||
}
|
||||
}
|
||||
seen.Last = lastTaken(ctx, steps, taken, a.Kind, since)
|
||||
return seen, nil
|
||||
}
|
||||
|
||||
// stream is the build seat's stream of language, its name and the subject of kind on it.
|
||||
func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) {
|
||||
subject, err := delivery.AskSubject(language, kind)
|
||||
if err != nil {
|
||||
return nil, "", "", err
|
||||
return false, err
|
||||
}
|
||||
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
|
||||
// messages by sequence and nothing wider (issue 1910).
|
||||
name, err := delivery.AskStream(language)
|
||||
name, err := delivery.AskStream(a.Language)
|
||||
if err != nil {
|
||||
return nil, "", "", err
|
||||
return false, err
|
||||
}
|
||||
stream, err := h.js.Stream(ctx, name)
|
||||
return stream, name, subject, err
|
||||
}
|
||||
|
||||
// onSubject hands each message of subject on stream to each, in order, until each answers false or the subject's
|
||||
// messages end. A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read
|
||||
// by sequence instead, each the next of the subject (one read per message, none for the sequences taken between), and
|
||||
// none consumed.
|
||||
func onSubject(ctx context.Context, stream jetstream.Stream, subject string, each func(build.Queued) bool) error {
|
||||
for seq, n := uint64(1), 0; ; n++ {
|
||||
if n >= maxOnSubject {
|
||||
return fmt.Errorf("%s holds more than %d messages before the one asked after: not read further", subject, maxOnSubject)
|
||||
}
|
||||
m, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(subject))
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q := build.Queued{Seq: m.Sequence, Ask: m.Header.Get(jetstream.MsgIDHeader), PutAt: m.Time}
|
||||
if _, kind, err := delivery.ParseAskSubject(m.Subject); err == nil {
|
||||
q.Kind = kind
|
||||
}
|
||||
if q.At = build.AskedAt(q.Ask); q.At.IsZero() {
|
||||
q.At = q.PutAt
|
||||
}
|
||||
if !each(q) {
|
||||
return nil
|
||||
}
|
||||
seq = m.Sequence + 1
|
||||
}
|
||||
}
|
||||
|
||||
// stepKeys are the build module's step keys by ask id: `<node>.<ask id>` (a node's own key, such as its pause, has no
|
||||
// ask id and is left out).
|
||||
func stepKeys(ctx context.Context, steps jetstream.KeyValue) (map[string][]string, error) {
|
||||
all, err := steps.ListKeys(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return map[string][]string{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer all.Stop()
|
||||
out := map[string][]string{}
|
||||
for k := range all.Keys() {
|
||||
if _, id, ok := strings.Cut(k, "."); ok && !build.AskedAt(id).IsZero() {
|
||||
out[id] = append(out[id], k)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// stepState is the state of the first of keys a node wrote, "" for none.
|
||||
func stepState(ctx context.Context, steps jetstream.KeyValue, keys []string) string {
|
||||
for _, k := range keys {
|
||||
e, err := steps.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if s, err := delivery.DecodeStep(e.Value()); err == nil {
|
||||
return s.State
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// lastTaken is the ask asked last after since that a build node took, of a kind whose taking proves kind passed
|
||||
// (build.Proves), read newest first and at most maxSteps of them; nil for none. A step's tool is "<kind> ask"
|
||||
// (mesh-sdk buildstep).
|
||||
func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][]string, kind string, since time.Time) *build.Taken {
|
||||
var ids []string
|
||||
for id := range taken {
|
||||
if build.AskedAt(id).After(since) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool { return build.AskedAt(ids[i]).After(build.AskedAt(ids[j])) })
|
||||
read := 0
|
||||
for _, id := range ids {
|
||||
for _, k := range taken[id] {
|
||||
if read >= maxSteps {
|
||||
return nil
|
||||
}
|
||||
read++
|
||||
e, err := steps.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stepKind, ok := strings.CutSuffix(s.Tool, " ask")
|
||||
if !ok || !build.Proves(stepKind, kind) {
|
||||
continue
|
||||
}
|
||||
return &build.Taken{Ask: id, Kind: stepKind, At: build.AskedAt(id), Node: s.Node, Started: s.Started}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Drop is build.Dropper's Drop: it removes the ask's message from its build seat's stream, found by its Nats-Msg-Id on its
|
||||
// subject, when the stream still holds it. The builder's account must be granted $JS.API.STREAM.MSG.DELETE.<stream>.
|
||||
func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error {
|
||||
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var seq uint64
|
||||
if err := onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||
if q.Ask == a.ID {
|
||||
seq = q.Seq
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); err != nil || seq == 0 {
|
||||
return err
|
||||
}
|
||||
if err := stream.DeleteMsg(ctx, seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return fmt.Errorf("deleting message %d of %s (is $JS.API.STREAM.MSG.DELETE.%s granted to the builder?): %w", seq, name, name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Queue is what the tool builder_seat_queue answers (build.SeatQueue): the messages of language's build seat's stream, of
|
||||
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
|
||||
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
|
||||
q := build.SeatQueue{Language: language, Note: build.QueueNote}
|
||||
module, err := delivery.BuildModuleOf(language)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
stream, name, _, err := h.stream(ctx, language, delivery.AskDelivery)
|
||||
q.Stream = name
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
q.Note = name + " does not exist"
|
||||
return q, nil
|
||||
}
|
||||
if err != nil {
|
||||
return q, err
|
||||
return false, err
|
||||
}
|
||||
// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
|
||||
// instead, each a direct read that consumes nothing.
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
return q, err
|
||||
return false, err
|
||||
}
|
||||
q.Messages, q.FirstSeq, q.LastSeq = info.State.Msgs, info.State.FirstSeq, info.State.LastSeq
|
||||
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
taken, err := stepKeys(ctx, steps)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
for _, kind := range delivery.AskKinds {
|
||||
subject, _ := delivery.AskSubject(language, kind)
|
||||
var ahead []build.Queued
|
||||
if err := onSubject(ctx, stream, subject, func(m build.Queued) bool {
|
||||
ahead = append(ahead, m)
|
||||
return len(ahead) < limit
|
||||
}); err != nil {
|
||||
return q, err
|
||||
for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
|
||||
m, err := stream.GetMsg(ctx, seq)
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
continue // taken and acknowledged, or removed
|
||||
}
|
||||
if len(ahead) == 0 {
|
||||
continue
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for i := range ahead {
|
||||
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||
if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
|
||||
return true, nil
|
||||
}
|
||||
seen := build.AskSeen{Kind: kind, Ahead: ahead, Last: lastTaken(ctx, steps, taken, kind, ahead[0].At)}
|
||||
q.Kinds = append(q.Kinds, build.NewKindQueue(kind, seen, time.Now()))
|
||||
}
|
||||
return q, nil
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -24,9 +24,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
@@ -165,29 +162,11 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
|
||||
const trunkHead = "3333333333333333333333333333333333333333"
|
||||
|
||||
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
|
||||
func (b *bed) trunk(t *testing.T, head string) uint64 {
|
||||
t.Helper()
|
||||
data, err := delivery.EncodeSource(delivery.Source{V: delivery.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rev, err := b.source.Put(b.ctx, delivery.SourceMainKey, data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rev
|
||||
}
|
||||
|
||||
type bed struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
modules jetstream.KeyValue
|
||||
candidates jetstream.KeyValue
|
||||
source jetstream.KeyValue
|
||||
builds jetstream.KeyValue
|
||||
tools map[string]jetstream.KeyValue
|
||||
instances jetstream.KeyValue
|
||||
@@ -264,7 +243,6 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
}
|
||||
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
|
||||
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
|
||||
b.source = mk(delivery.SourceBucket, delivery.SourceHistory)
|
||||
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
||||
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
||||
b.instances = mk(instancesBucket, 1)
|
||||
@@ -346,7 +324,7 @@ func (b *bed) archive(t *testing.T, name, path, at string) string {
|
||||
func (b *bed) start(t *testing.T, node string, env map[string]string) {
|
||||
t.Helper()
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
|
||||
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -535,9 +513,7 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
|
||||
|
||||
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
|
||||
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
|
||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source
|
||||
// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its
|
||||
// fingerprint first appeared at).
|
||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk commit (issue 1684).
|
||||
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
|
||||
b := startBed(t)
|
||||
other := "src2:" + strings.Repeat("e", 64)
|
||||
@@ -554,7 +530,7 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"},
|
||||
Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -574,12 +550,12 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
||||
defer b.fake.mu.Unlock()
|
||||
found := false
|
||||
for _, f := range b.fake.fetches {
|
||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead {
|
||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == commit {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches)
|
||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk commit", b.fake.fetches)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -627,7 +603,7 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
|
||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"imaged"},
|
||||
Fps: map[string]delivery.CandidateModule{"imaged": {FP2: fp, Manifest: digest}}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -825,34 +801,6 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
|
||||
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
|
||||
}
|
||||
}
|
||||
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
|
||||
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
|
||||
held := heldOnTheBus{b.js}
|
||||
goAsk := delivery.StepAsk{ID: v.Asks[delivery.AskGo], Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||
stale := delivery.StepAsk{ID: "mvo53jcm70b5f7", Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskDelivery)
|
||||
if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queued := func() []build.QueuedState {
|
||||
q, err := held.Queue(b.ctx, delivery.AskGo, 10)
|
||||
if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != delivery.AskDelivery {
|
||||
t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err)
|
||||
}
|
||||
return q.Kinds[0].Messages
|
||||
}
|
||||
if m := queued(); len(m) != 2 || m[0].Ask != goAsk.ID || !m[0].Waits || m[1].Ask != stale.ID {
|
||||
t.Fatalf("the Go delivery queue = %+v, want the key's ask waiting, then the stale one", m)
|
||||
}
|
||||
if why, err := held.Gone(b.ctx, goAsk); why != "" || err != nil {
|
||||
t.Errorf("Gone(the waiting ask) = %q, %v; want it answerable", why, err)
|
||||
}
|
||||
if err := held.Drop(b.ctx, stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := queued(); len(m) != 1 || m[0].Ask != goAsk.ID {
|
||||
t.Fatalf("after the drop the Go delivery queue = %+v, want the key's ask alone", m)
|
||||
}
|
||||
b.stop()
|
||||
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
|
||||
b.fake.open()
|
||||
|
||||
@@ -6,20 +6,22 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"io"
|
||||
"log"
|
||||
"maps"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -45,32 +47,103 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
|
||||
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func readCredential(path string) (credential, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return credential{}, err
|
||||
}
|
||||
var c credential
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if c.URL == "" || c.User == "" {
|
||||
return credential{}, errors.New("the broker credential names no bus address or user")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinned accepts exactly the certificate with this SHA-256 and no other.
|
||||
func pinned(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-builder: ", log.LstdFlags)
|
||||
cred, err := process.ReadBrokerAccount(getenv)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
|
||||
}
|
||||
cred, err := readCredential(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, js, err := cred.Dial()
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build")
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
builds, err := bucket(delivery.BuilderBuildsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds")
|
||||
modules, err := bucket(delivery.ModulesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := bucket(delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidates, err := bucket(delivery.CandidatesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's.
|
||||
candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf)
|
||||
bound := defaultStepBound
|
||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
||||
if bound, err = time.ParseDuration(v); err != nil {
|
||||
@@ -81,56 +154,35 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
|
||||
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
// state hearing, which builder_stuck says (dead_letters).
|
||||
dead := &buildrun.DeadLetters{}
|
||||
// The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it
|
||||
// is bound when the bus says it was created, and the latest hearing is written then.
|
||||
hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
|
||||
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
||||
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
account, err := process.ReadAccount(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer calls.Close()
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
CheckAfter: bound,
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
Toolchains: tools,
|
||||
Manifests: manifestReader{manifests},
|
||||
Logf: logger.Printf,
|
||||
Stored: storedVia(calls.Call),
|
||||
}
|
||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
|
||||
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
|
||||
errs := make(chan error, 6)
|
||||
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
|
||||
trunks.start()
|
||||
b.Trunk = trunks.head
|
||||
b.Said = func(e build.AskedAgain) {
|
||||
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
}
|
||||
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
|
||||
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
|
||||
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
|
||||
// is asked since R4g, ADR 0524 §7).
|
||||
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); {
|
||||
case kv == nil:
|
||||
switch kv, err := js.KeyValue(ctx, instancesBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
|
||||
default:
|
||||
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
|
||||
b.Answered = says.Answered
|
||||
@@ -160,12 +212,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// module.
|
||||
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
|
||||
stream := toolchainStreams[module]
|
||||
cons, err := cred.EventConsumer(ctx, js, stream)
|
||||
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the builder starts again: %v", cred.Node, cred.Module, stream, module, err)
|
||||
continue
|
||||
}
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { watch.Reported(m) }, logger.Printf)
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -174,12 +227,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
|
||||
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
|
||||
// so; it still asks held at its start and at every module value.
|
||||
switch cons, err := cred.EventConsumer(ctx, js, eventsStream); {
|
||||
switch cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module); {
|
||||
case err != nil:
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s is not heard until the builder starts again: %v",
|
||||
cred.Node, cred.Module, eventsStream, delivery.ArtifactsGoneEvent, err)
|
||||
default:
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) }, logger.Printf)
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -208,37 +262,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
b.ForgetCandidate(e.Key())
|
||||
}
|
||||
}
|
||||
if candidates != nil {
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||
}
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
|
||||
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||
}
|
||||
})
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||
}
|
||||
})
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
errs := make(chan error, 2)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
b.Wait()
|
||||
@@ -251,50 +282,3 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
|
||||
// delivery.SourceHistory of them: what a candidate's base revision names.
|
||||
type sourceHeads struct {
|
||||
mu sync.Mutex
|
||||
byRev map[uint64]map[string]string
|
||||
}
|
||||
|
||||
// put takes one revision in and answers whether it is new.
|
||||
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
|
||||
if !opOf(e) {
|
||||
return false
|
||||
}
|
||||
src, err := delivery.DecodeSource(e.Value())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.byRev == nil {
|
||||
s.byRev = map[uint64]map[string]string{}
|
||||
}
|
||||
if _, had := s.byRev[e.Revision()]; had {
|
||||
return false
|
||||
}
|
||||
s.byRev[e.Revision()] = src.Heads
|
||||
for len(s.byRev) > delivery.SourceHistory {
|
||||
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
|
||||
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
heads, ok := s.byRev[rev]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
|
||||
}
|
||||
for r, head := range heads {
|
||||
if delivery.SameRepository(r, repository) {
|
||||
return head, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
|
||||
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
|
||||
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
|
||||
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
|
||||
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
|
||||
// and opened again at the next event that needs it where there is one.
|
||||
|
||||
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
|
||||
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
|
||||
// and a candidate's preview waits, saying why.
|
||||
type trunkSource struct {
|
||||
ctx context.Context
|
||||
js bucketOpener
|
||||
heads *sourceHeads
|
||||
moved func(context.Context)
|
||||
errs chan<- error
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
opened bool
|
||||
}
|
||||
|
||||
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
|
||||
func (t *trunkSource) ensure() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.opened {
|
||||
return nil
|
||||
}
|
||||
kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err)
|
||||
}
|
||||
w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err)
|
||||
}
|
||||
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
}
|
||||
t.opened = true
|
||||
go func() {
|
||||
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if t.heads.put(e) && t.moved != nil {
|
||||
t.moved(t.ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// start opens the source at start; one that cannot be is said, and the builder goes on.
|
||||
func (t *trunkSource) start() {
|
||||
if err := t.ensure(); err != nil {
|
||||
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
|
||||
}
|
||||
}
|
||||
|
||||
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
|
||||
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||
if err := t.ensure(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return t.heads.head(ctx, rev, repository)
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// queueReader is the part of the builder a queues value reads through: heldOnTheBus.
|
||||
type queueReader interface {
|
||||
Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error)
|
||||
}
|
||||
|
||||
// queueRead answers one value of the queues state (the tool builder_seat_queue, issue 1945 note 62): a value not yet
|
||||
// answered is read now, from the stream the builder's account may read, and written back by compare-and-set at its
|
||||
// revision with what was read or why it could not be. A value answered already (this builder's own write, or another
|
||||
// instance's) or deleted is left.
|
||||
func queueRead(ctx context.Context, r queueReader, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
a, err := build.DecodeQueueAsk(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not a queue read, left: %v", build.QueuesState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if !a.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
q, err := r.Queue(read, e.Key(), max(1, min(a.Limit, build.MaxQueueLimit)))
|
||||
cancel()
|
||||
if err != nil {
|
||||
a.Error = err.Error()
|
||||
}
|
||||
a.Queue, a.Answered = &q, time.Now().UTC()
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the read could not be written: %v", build.QueuesState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// askAgainer is the part of the builder a reasks value asks through.
|
||||
type askAgainer interface {
|
||||
AskAgain(key, why string) []string
|
||||
BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
|
||||
}
|
||||
|
||||
// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
|
||||
// longer for the builder's answer.
|
||||
const buildAgainBound = 15 * time.Second
|
||||
|
||||
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
|
||||
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
|
||||
// answered already (this builder's own write) or deleted is left.
|
||||
func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
r, err := build.DecodeReAsk(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if r.Answer != "" {
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeBuild {
|
||||
// The answer comes when the claim and the asks are written, so it is made beside the watch, not in it.
|
||||
go builtAgain(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
asked := b.AskAgain(e.Key(), r.Why)
|
||||
if len(asked) == 0 {
|
||||
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
|
||||
} else {
|
||||
r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names"
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
|
||||
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
|
||||
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Account, e build.AskedAgain) error {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(cred.EventSubject(build.AskedAgainEvent))
|
||||
id := "asked-again-" + e.New
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// builtAgain answers a reasks value of mode build (the tool builder_build_again, issue 1126 note 72): the failed key is
|
||||
// built again as a new revision, and the value is written back with the revision and the asks put, or why nothing was done.
|
||||
func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
|
||||
res := b.BuildAgain(ctx, r.Module, r.FP, r.Why, r.By, buildAgainBound)
|
||||
switch {
|
||||
case res.Refused != "":
|
||||
r.Answer, r.Refused = res.Refused, true
|
||||
case res.Failed != "":
|
||||
r.Answer, r.Revision, r.Failed = fmt.Sprintf("the key is written building as revision %d, and the build failed before any ask was put: %s", res.Revision, res.Failed), res.Revision, res.Failed
|
||||
default:
|
||||
r.Pending = res.Revision == 0
|
||||
r.Answer, r.Revision, r.Asks = "built again: the key is written building as revision "+fmt.Sprint(res.Revision)+" and its asks are put", res.Revision, res.Asks
|
||||
if res.Pending != "" {
|
||||
r.Answer = res.Pending
|
||||
if res.Revision != 0 {
|
||||
r.Answer = "built again: the key is written building as revision " + fmt.Sprint(res.Revision) + "; " + res.Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: build again asked by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
@@ -28,11 +28,8 @@ import (
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
@@ -41,14 +38,14 @@ type toolchainGate struct {
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
standing map[string]bool // <node>.<module> with an open assignment
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
@@ -70,31 +67,20 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
open := !deleted && openAssignment(value)
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
if g.standing[key] == open {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
g.standing[key] = true
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
@@ -125,58 +111,37 @@ func (g *toolchainGate) Settled() bool {
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||
// otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
if !g.left[module+"/"+key] {
|
||||
g.left[module+"/"+key] = true
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||
delivery.ToolchainsBucket(module), key, module, node)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||
func openAssignment(value []byte) bool {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
return false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
return a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
@@ -190,19 +155,17 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -3,19 +3,10 @@ package main
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
@@ -53,10 +44,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
@@ -72,20 +63,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
@@ -97,132 +88,25 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
set := &buildrun.ToolchainSet{}
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||
gate.Settled()
|
||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
key, value := gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an assigned node's report was not passed on")
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,13 +4,11 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -64,14 +62,11 @@ func (w *toolchainWatch) ensure(module string) error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||
return nil
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,344 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
|
||||
// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
|
||||
// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
|
||||
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
|
||||
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
|
||||
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
|
||||
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
|
||||
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
|
||||
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
|
||||
const DefaultAskings = 3
|
||||
|
||||
// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
|
||||
const AskedAgainEvent = "build.asked-again"
|
||||
|
||||
// AskedAgain is the body of AskedAgainEvent.
|
||||
type AskedAgain struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
BuildModule string `json:"build_module"`
|
||||
Language string `json:"language"`
|
||||
Old string `json:"old"`
|
||||
New string `json:"new"`
|
||||
Asking int `json:"asking"`
|
||||
Askings int `json:"askings"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
|
||||
// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
|
||||
// delivery counts or ack floors (granted to the delivery view alone: $JS.API.CONSUMER.INFO.<stream>.<worker>), so a
|
||||
// message given up on is told by what the workers did after it: a build node took an ask asked later, of the same kind or
|
||||
// of one pulled after it (issue 1945 note 62: seven messages SEAT_NODE_BUILD_GO had given up on sat ahead of a Go ask for
|
||||
// five hours, each counted as waiting, so the ask was never asked again).
|
||||
type AskSeen struct {
|
||||
Stream string
|
||||
// Kind is the ask's kind, At when it was asked (its id's time).
|
||||
Kind string
|
||||
At time.Time
|
||||
// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
|
||||
Step bool
|
||||
// OnStream is whether the stream still holds the ask's message; PutAt is when it was put.
|
||||
OnStream bool
|
||||
PutAt time.Time
|
||||
// Ahead are the messages of its subject before it on the stream.
|
||||
Ahead []Queued
|
||||
// Last is the ask asked last that a build node took (wrote a step for) of the ask's kind or of a kind pulled after
|
||||
// it, when one was taken: every message of the kind asked before it was then delivered already, or given up on.
|
||||
Last *Taken
|
||||
}
|
||||
|
||||
// Queued is one message of a build seat's stream: its sequence, its ask's id and kind, when the ask was asked (its id's
|
||||
// time, the put time when the id says none) and put, and the state of the step a build node wrote for it ("" for none).
|
||||
type Queued struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Ask string `json:"ask"`
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"asked"`
|
||||
PutAt time.Time `json:"put"`
|
||||
Step string `json:"step,omitempty"`
|
||||
}
|
||||
|
||||
// Taken is an ask a build node took: its id, kind and when it was asked, and the step's node and start.
|
||||
type Taken struct {
|
||||
Ask string `json:"ask"`
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"asked"`
|
||||
Node string `json:"node"`
|
||||
Started time.Time `json:"started"`
|
||||
}
|
||||
|
||||
// Proves is whether a taken ask of kind taken shows that a message of kind waiting asked before it is no longer
|
||||
// delivered: a holder pulls the kinds in delivery.AskKinds' order, so taking one of the same kind or of a later one means
|
||||
// its worker had none of waiting's kind left to give.
|
||||
func Proves(taken, waiting string) bool {
|
||||
t, w := slices.Index(delivery.AskKinds, taken), slices.Index(delivery.AskKinds, waiting)
|
||||
return t >= 0 && w >= 0 && t >= w
|
||||
}
|
||||
|
||||
// Waits is whether a message ahead still waits to be delivered: no build node wrote a step for it, and no ask of its
|
||||
// kind asked after it was taken. A message its worker gave up on (delivered its most times) stays on a work queue until
|
||||
// its age, and is not counted.
|
||||
func (s AskSeen) Waits(q Queued) bool {
|
||||
return q.Step == "" && !s.passed(q.At)
|
||||
}
|
||||
|
||||
func (s AskSeen) passed(at time.Time) bool {
|
||||
return s.Last != nil && Proves(s.Last.Kind, s.Kind) && s.Last.At.After(at)
|
||||
}
|
||||
|
||||
// Waiting is how many messages ahead still wait to be delivered (Waits).
|
||||
func (s AskSeen) Waiting() int {
|
||||
n := 0
|
||||
for _, q := range s.Ahead {
|
||||
if s.Waits(q) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind still
|
||||
// deliverable, or no build node took a later ask of its kind (the nodes are busy or paused: a backlog, never a fault,
|
||||
// ADR 0525 §9).
|
||||
func (s AskSeen) Why() string {
|
||||
switch {
|
||||
case s.Step:
|
||||
return ""
|
||||
case !s.OnStream:
|
||||
return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
|
||||
case s.Waiting() > 0 || !s.passed(s.at()):
|
||||
return ""
|
||||
default:
|
||||
behind := ""
|
||||
if n := len(s.Ahead); n > 0 {
|
||||
behind = fmt.Sprintf(" behind %d given up on or taken", n)
|
||||
}
|
||||
return fmt.Sprintf("it waits on %s since %s%s while %s took the %s ask %s asked later at %s: its stream no longer delivers it (given up after its most deliveries)",
|
||||
s.Stream, s.PutAt.UTC().Format(time.RFC3339), behind, s.Last.Node, s.Last.Kind, s.Last.Ask, s.Last.At.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
|
||||
func (s AskSeen) at() time.Time {
|
||||
if s.At.IsZero() {
|
||||
return s.PutAt
|
||||
}
|
||||
return s.At
|
||||
}
|
||||
|
||||
// AskedAt is when an ask id was made (buildstep.NewID: its milliseconds in base 36 before six hex digits), or zero.
|
||||
func AskedAt(id string) time.Time {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil || ms <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.UnixMilli(ms)
|
||||
}
|
||||
|
||||
// Dropper removes an ask's message from its build seat's stream (Builder.Gone may be one): the old message of an ask asked
|
||||
// again, so a message given up on does not sit ahead of every later ask of its kind until its age (issue 1945 note 62).
|
||||
type Dropper interface {
|
||||
Drop(ctx context.Context, a delivery.StepAsk) error
|
||||
}
|
||||
|
||||
// drop removes the old ask's message when Gone can; a refusal is said and changes nothing else: the new ask is put.
|
||||
func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
|
||||
d, ok := b.Gone.(Dropper)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
prior := ask
|
||||
prior.ID = old
|
||||
if err := d.Drop(ctx, prior); err != nil {
|
||||
b.logf("the message of the ask %s asked again as %s stays on its stream: %v", old, ask.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) askings() int {
|
||||
if b.Askings > 0 {
|
||||
return b.Askings
|
||||
}
|
||||
return DefaultAskings
|
||||
}
|
||||
|
||||
var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)
|
||||
|
||||
// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
|
||||
func askingOf(step, buildModule string) int {
|
||||
for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
|
||||
if m[1] == buildModule {
|
||||
if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
|
||||
// languages whose lane waits, none when no lane of this builder waits on the key.
|
||||
func (b *Builder) AskAgain(key, why string) []string {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
var asked []string
|
||||
for language, c := range b.waiting[key] {
|
||||
select {
|
||||
case c <- why:
|
||||
default: // one is already asked
|
||||
}
|
||||
asked = append(asked, language)
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// Waiting are the build keys whose lanes wait for a step now, each with its languages.
|
||||
func (b *Builder) Waiting() map[string][]string {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
out := map[string][]string{}
|
||||
for key, ls := range b.waiting {
|
||||
for l := range ls {
|
||||
out[key] = append(out[key], l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *Builder) listen(key, language string) chan string {
|
||||
b.init()
|
||||
c := make(chan string, 1)
|
||||
b.mu.Lock()
|
||||
if b.waiting[key] == nil {
|
||||
b.waiting[key] = map[string]chan string{}
|
||||
}
|
||||
b.waiting[key][language] = c
|
||||
b.mu.Unlock()
|
||||
return c
|
||||
}
|
||||
|
||||
func (b *Builder) unlisten(key, language string, c chan string) {
|
||||
b.mu.Lock()
|
||||
if b.waiting[key][language] == c {
|
||||
delete(b.waiting[key], language)
|
||||
if len(b.waiting[key]) == 0 {
|
||||
delete(b.waiting, key)
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
var (
|
||||
errCheck = errors.New("the check bound passed")
|
||||
errAgain = errors.New("asked again")
|
||||
)
|
||||
|
||||
type byHand struct{ why string }
|
||||
|
||||
func (h byHand) Error() string { return "asked again by hand: " + h.why }
|
||||
|
||||
// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
|
||||
// asked it again (AskAgain); it answers the step and the ask that has it.
|
||||
func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
|
||||
key, _ := delivery.BuilderBuildKey(module, fp)
|
||||
again := b.listen(key, ask.Language)
|
||||
defer b.unlisten(key, ask.Language, again)
|
||||
asking := askingOf(asked.step, l.module)
|
||||
for {
|
||||
step, why, err := b.waitOnce(ctx, ask, again)
|
||||
switch {
|
||||
case errors.Is(err, errCheck):
|
||||
continue
|
||||
case errors.Is(err, errAgain):
|
||||
if asking >= b.askings() {
|
||||
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
|
||||
}
|
||||
asking++
|
||||
old := ask.ID
|
||||
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
|
||||
return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
return step, ask, err
|
||||
}
|
||||
b.drop(ctx, ask, old)
|
||||
b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
|
||||
if b.Said != nil {
|
||||
b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
|
||||
Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
|
||||
}
|
||||
default:
|
||||
return step, ask, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
|
||||
// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
|
||||
func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
|
||||
hand, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
go func() {
|
||||
select {
|
||||
case why := <-again:
|
||||
cancel(byHand{why})
|
||||
case <-hand.Done():
|
||||
}
|
||||
}()
|
||||
wait := hand
|
||||
if b.Gone != nil && b.CheckAfter > 0 {
|
||||
var stop context.CancelFunc
|
||||
wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if err == nil && buildrun.IsLost(step) {
|
||||
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
|
||||
// words of why, not at the check bound.
|
||||
return step, step.Reason, errAgain
|
||||
}
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return step, "", err
|
||||
}
|
||||
var h byHand
|
||||
if errors.As(context.Cause(hand), &h) {
|
||||
return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
|
||||
}
|
||||
if errors.Is(context.Cause(wait), errCheck) {
|
||||
why, gerr := b.Gone.Gone(ctx, ask)
|
||||
if gerr != nil {
|
||||
b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
|
||||
return step, "", errCheck
|
||||
}
|
||||
if why == "" {
|
||||
return step, "", errCheck
|
||||
}
|
||||
return step, why, errAgain
|
||||
}
|
||||
return step, "", err
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
|
||||
// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a
|
||||
// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed
|
||||
// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key
|
||||
// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool
|
||||
// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps
|
||||
// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused.
|
||||
|
||||
// BuildAgainResult is the builder's answer to a request to build a key again.
|
||||
type BuildAgainResult struct {
|
||||
// Refused is why nothing was done, "" when the key was written building as a new revision.
|
||||
Refused string
|
||||
// Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound.
|
||||
Revision uint64
|
||||
Asks map[string]string
|
||||
// Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal.
|
||||
Pending string
|
||||
// Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a
|
||||
// toolchain not agreed): the new revision was written, and nothing was asked.
|
||||
Failed string
|
||||
}
|
||||
|
||||
// handWatch follows one request to build a key again until its asks are put.
|
||||
type handWatch struct {
|
||||
rev uint64
|
||||
lanes int
|
||||
asks map[string]string
|
||||
done chan struct{}
|
||||
shut bool
|
||||
// ended is set when the look ended before the asks were all put.
|
||||
ended bool
|
||||
}
|
||||
|
||||
// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
|
||||
func (w *handWatch) settle() {
|
||||
if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes {
|
||||
w.shut = true
|
||||
close(w.done)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) hand(key string, f func(w *handWatch)) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if w := b.hands[key]; w != nil {
|
||||
f(w)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) handClaimed(key string, rev uint64) {
|
||||
b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handLanes(key string, n int) {
|
||||
b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handAsk(key, language, id string) {
|
||||
b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() })
|
||||
}
|
||||
|
||||
// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
|
||||
func (b *Builder) handEnded(key string) {
|
||||
b.hand(key, func(w *handWatch) {
|
||||
if !w.shut {
|
||||
w.shut, w.ended = true, true
|
||||
close(w.done)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
|
||||
// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
|
||||
// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
|
||||
// ctx is the builder's own context: the build it starts lives on it.
|
||||
func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
|
||||
b.init()
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: err.Error()}
|
||||
}
|
||||
value, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
|
||||
}
|
||||
if !found {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)}
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
|
||||
}
|
||||
switch {
|
||||
case cur.State == delivery.BuildDone:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
|
||||
case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
|
||||
case cur.State != delivery.BuildFailed:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)}
|
||||
case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix):
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)}
|
||||
}
|
||||
// One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint,
|
||||
// and the watch and the look are registered before anything else can take the key.
|
||||
w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})}
|
||||
b.mu.Lock()
|
||||
m, known := b.known[module]
|
||||
switch {
|
||||
case !known || m.FP2 != fp:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)}
|
||||
case m.Manifest == "":
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)}
|
||||
case b.inflight[key] || b.hands[key] != nil:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)}
|
||||
}
|
||||
b.hands[key] = w
|
||||
b.inflight[key] = true
|
||||
b.mu.Unlock()
|
||||
defer func() {
|
||||
b.mu.Lock()
|
||||
if b.hands[key] == w {
|
||||
delete(b.hands, key)
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}()
|
||||
b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why)
|
||||
b.startOnce(ctx, key, module, m)
|
||||
wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8)
|
||||
defer stop()
|
||||
select {
|
||||
case <-w.done:
|
||||
case <-wait.Done():
|
||||
}
|
||||
b.mu.Lock()
|
||||
res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
|
||||
ended, shut, lanes := w.ended, w.shut, w.lanes
|
||||
b.mu.Unlock()
|
||||
switch {
|
||||
case res.Revision == 0 && shut:
|
||||
res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
|
||||
case res.Revision == 0:
|
||||
res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
|
||||
case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
|
||||
// The look ended before every ask was put: say how the key stands, not that it was asked.
|
||||
if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found {
|
||||
if cur, err := delivery.DecodeBuilderBuild(v); err == nil {
|
||||
switch cur.State {
|
||||
case delivery.BuildFailed:
|
||||
res.Failed = cur.Error
|
||||
case delivery.BuildDone:
|
||||
default:
|
||||
res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step)
|
||||
}
|
||||
}
|
||||
}
|
||||
case !shut:
|
||||
res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func boolInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -73,17 +73,9 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
|
||||
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
|
||||
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
|
||||
}
|
||||
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
|
||||
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
|
||||
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
|
||||
base, err := b.trunkHead(ctx, c.Base, p.Repository)
|
||||
if err != nil {
|
||||
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.FP2, err)
|
||||
continue
|
||||
}
|
||||
b.mu.Lock()
|
||||
if _, onTrunk := b.trunkFP[key]; !onTrunk {
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base}
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
b.consider(ctx, name, m)
|
||||
@@ -114,18 +106,3 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]del
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
|
||||
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||
if b.Trunk == nil {
|
||||
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
|
||||
}
|
||||
head, err := b.Trunk(ctx, rev, repository)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
|
||||
}
|
||||
if head == "" {
|
||||
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
|
||||
}
|
||||
return head, nil
|
||||
}
|
||||
|
||||
@@ -2,13 +2,9 @@ package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -44,74 +40,11 @@ func candidateValue(t *testing.T, group string, number int, head string) []byte
|
||||
}
|
||||
|
||||
const (
|
||||
// trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the
|
||||
// commit its fingerprint first appeared at.
|
||||
trunkHead = "3333333333333333333333333333333333333333"
|
||||
headX = "1111111111111111111111111111111111111111"
|
||||
headY = "2222222222222222222222222222222222222222"
|
||||
headX = "1111111111111111111111111111111111111111"
|
||||
headY = "2222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer
|
||||
// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test).
|
||||
func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c delivery.Candidate, raw []byte) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w struct {
|
||||
Source struct {
|
||||
Rev uint64 `json:"rev"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
} `json:"source"`
|
||||
Candidate json.RawMessage `json:"candidate"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, err = delivery.DecodeCandidate(w.Candidate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return w.Source.Rev, w.Source.Heads, c, w.Candidate
|
||||
}
|
||||
|
||||
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
|
||||
// The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module
|
||||
// value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change).
|
||||
{
|
||||
rev, heads, c, raw := indexerCandidate(t)
|
||||
p := c.Pulls[0]
|
||||
cm := p.Fps["avahi"]
|
||||
ir := newRig(t, goBundle)
|
||||
ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) {
|
||||
if at != rev {
|
||||
return "", fmt.Errorf("source revision %d is not heard", at)
|
||||
}
|
||||
return heads[repository], nil
|
||||
}
|
||||
const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
|
||||
value, err := delivery.EncodeModule(delivery.Module{Repository: p.Repository, Commit: valueCommit, FP2: "src2:" + hex64('e'),
|
||||
Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: agreedToolchains})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ir.b.Module(context.Background(), "avahi", value)
|
||||
ir.b.Wait()
|
||||
ir.bm.mu.Lock()
|
||||
ir.bm.calls = nil
|
||||
ir.bm.mu.Unlock()
|
||||
ir.b.Candidate(context.Background(), "feat%2Fx", raw)
|
||||
ir.b.Wait()
|
||||
fetches := ir.bm.callsOf("build_go_fetch")
|
||||
if len(fetches) != 1 {
|
||||
t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches))
|
||||
}
|
||||
f := fetches[0].body.(map[string]any)
|
||||
if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head.Head {
|
||||
t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev)
|
||||
}
|
||||
}
|
||||
r := newRig(t, goBundle)
|
||||
trunkModule(t, r)
|
||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
||||
@@ -125,8 +58,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe
|
||||
t.Fatalf("%d fetches, want one", len(fetches))
|
||||
}
|
||||
f := fetches[0].body.(map[string]any)
|
||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" {
|
||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f)
|
||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" {
|
||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk commit to merge onto", f)
|
||||
}
|
||||
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
|
||||
t.Errorf("build args = %+v", b)
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The tool builder_seat_queue reads a build seat's stream as the builder reads it** (novox/hq issue 1945 note 62; the
|
||||
// instruction "tools are how a module is spoken to"): the messages waiting on SEAT_NODE_BUILD_<LANGUAGE>, bounded, each
|
||||
// with its ask, its age, the state of the step a build node wrote for it, and whether the builder counts it as still
|
||||
// waiting (AskSeen.Waits) or as given up on. Only the builder's own account may read the stream (issue 1910): the tools
|
||||
// bundle writes the read into the builder's state queues, keyed by the language, which the builder watches and answers
|
||||
// there, as builder_ask_again does through reasks.
|
||||
const (
|
||||
SeatQueueTool = "builder_seat_queue"
|
||||
// QueuesState is the builder's state of queue reads: the bucket mesh-delivery-builder_queues, keyed by the language.
|
||||
QueuesState = "queues"
|
||||
QueuesBucket = delivery.BuilderModule + "_" + QueuesState
|
||||
// DefaultQueueLimit and MaxQueueLimit bound the messages read of each kind.
|
||||
DefaultQueueLimit = 50
|
||||
MaxQueueLimit = 500
|
||||
)
|
||||
|
||||
// QueueNote says what the read cannot show.
|
||||
const QueueNote = "the builder may not read the seat workers' delivery counts or ack floors ($JS.API.CONSUMER.INFO.<stream>.<worker> is " +
|
||||
"granted to the delivery view alone): a message counts as given up on when no node wrote its step and a node took an ask of its kind " +
|
||||
"asked after it; nats.nats_streams shows the workers' pending and waiting pulls"
|
||||
|
||||
// QueueAsk is a value of the queues state: a read asked (limit, who, when) and, once the builder read the stream, its answer.
|
||||
type QueueAsk struct {
|
||||
V int `json:"v"`
|
||||
Limit int `json:"limit"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Queue *SeatQueue `json:"queue,omitempty"`
|
||||
}
|
||||
|
||||
// SeatQueue is what the builder read of one build seat's stream.
|
||||
type SeatQueue struct {
|
||||
Language string `json:"language"`
|
||||
Stream string `json:"stream"`
|
||||
Messages uint64 `json:"messages"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
Kinds []KindQueue `json:"kinds"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// KindQueue is one kind's messages on the stream, oldest first: how many the builder counts as waiting, the ask asked
|
||||
// last a build node took of the kind (or of one pulled after it), and each message.
|
||||
type KindQueue struct {
|
||||
Kind string `json:"kind"`
|
||||
Read int `json:"read"`
|
||||
Waiting int `json:"waiting"`
|
||||
GivenUp int `json:"given_up_or_taken"`
|
||||
Last *Taken `json:"last_taken,omitempty"`
|
||||
Messages []QueuedState `json:"messages"`
|
||||
}
|
||||
|
||||
// QueuedState is one message as the tool shows it.
|
||||
type QueuedState struct {
|
||||
Queued
|
||||
AgeSeconds int64 `json:"age_s"`
|
||||
Waits bool `json:"waits"`
|
||||
}
|
||||
|
||||
// NewKindQueue is seen's messages ahead as one kind's queue at now.
|
||||
func NewKindQueue(kind string, seen AskSeen, now time.Time) KindQueue {
|
||||
k := KindQueue{Kind: kind, Read: len(seen.Ahead), Last: seen.Last}
|
||||
for _, q := range seen.Ahead {
|
||||
w := seen.Waits(q)
|
||||
if w {
|
||||
k.Waiting++
|
||||
} else {
|
||||
k.GivenUp++
|
||||
}
|
||||
k.Messages = append(k.Messages, QueuedState{Queued: q, AgeSeconds: int64(now.Sub(q.PutAt) / time.Second), Waits: w})
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// NewQueueAsk is the value builder_seat_queue writes for a language: its key and the value.
|
||||
func NewQueueAsk(language string, limit int, by string, at time.Time) (string, []byte, error) {
|
||||
if _, err := delivery.AskStream(language); err != nil {
|
||||
return "", nil, fmt.Errorf("language: %w", err)
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = DefaultQueueLimit
|
||||
}
|
||||
data, err := json.Marshal(QueueAsk{V: 1, Limit: min(limit, MaxQueueLimit), By: by, At: at.UTC()})
|
||||
return language, data, err
|
||||
}
|
||||
|
||||
// DecodeQueueAsk reads a queues value.
|
||||
func DecodeQueueAsk(data []byte) (QueueAsk, error) {
|
||||
var a QueueAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a queues value of version %d, not 1", a.V)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// LanguageOf is the language a tool's seat or language argument names: go, node or image, or node-build-<language>.
|
||||
func LanguageOf(arg string) string {
|
||||
return strings.TrimPrefix(strings.TrimSpace(arg), "node-build-")
|
||||
}
|
||||
@@ -379,9 +379,8 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A
|
||||
t.Fatal("no fetch")
|
||||
}
|
||||
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
|
||||
// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
|
||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
|
||||
t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
|
||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" {
|
||||
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -53,21 +53,6 @@ type Builder struct {
|
||||
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
|
||||
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
||||
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
|
||||
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
|
||||
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
|
||||
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
|
||||
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
|
||||
Gone interface {
|
||||
Gone(ctx context.Context, a delivery.StepAsk) (string, error)
|
||||
}
|
||||
CheckAfter time.Duration
|
||||
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
|
||||
// with why; DefaultAskings when 0.
|
||||
Askings int
|
||||
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
|
||||
Said func(AskedAgain)
|
||||
|
||||
once sync.Once
|
||||
sem chan struct{}
|
||||
@@ -87,10 +72,6 @@ type Builder struct {
|
||||
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
||||
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
||||
held map[string]bool
|
||||
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
|
||||
waiting map[string]map[string]chan string
|
||||
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
|
||||
hands map[string]*handWatch
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -107,8 +88,7 @@ func (b *Builder) init() {
|
||||
b.sem = make(chan struct{}, limit)
|
||||
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||
b.hands = map[string]*handWatch{}
|
||||
b.published, b.held = map[string]bool{}, map[string]bool{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -289,13 +269,6 @@ func (b *Builder) Reconsider(ctx context.Context) {
|
||||
for _, k := range sortedKeys(known) {
|
||||
b.consider(ctx, k, known[k])
|
||||
}
|
||||
b.TrunkMoved(ctx)
|
||||
}
|
||||
|
||||
// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the
|
||||
// revision it was previewed onto had no trunk head to be merged onto.
|
||||
func (b *Builder) TrunkMoved(ctx context.Context) {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
cands := make([]delivery.Candidate, 0, len(b.cands))
|
||||
for _, c := range b.cands {
|
||||
@@ -375,35 +348,9 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module
|
||||
}()
|
||||
}
|
||||
|
||||
// startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and
|
||||
// a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does.
|
||||
func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery.Module) {
|
||||
b.wg.Add(1)
|
||||
go func() {
|
||||
defer b.wg.Done()
|
||||
b.sem <- struct{}{}
|
||||
defer func() { <-b.sem }()
|
||||
for {
|
||||
b.try(ctx, module, m)
|
||||
b.mu.Lock()
|
||||
next, more := b.again[key]
|
||||
delete(b.again, key)
|
||||
if !more || ctx.Err() != nil {
|
||||
delete(b.inflight, key)
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.mu.Unlock()
|
||||
m = next // a look the builder queued meanwhile is not lost (consider's rule); it is not a second by-hand build
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
|
||||
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
|
||||
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
defer b.handEnded(key)
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
nothing := read && planErr == nil && len(plan.Tasks) == 0
|
||||
if planErr == nil && !nothing {
|
||||
@@ -414,6 +361,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
return
|
||||
}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
b.mu.Lock()
|
||||
trunk := b.trunkFP[key]
|
||||
first := !b.tried[key]
|
||||
@@ -428,7 +376,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
if cl == nil {
|
||||
return
|
||||
}
|
||||
b.handClaimed(key, cl.Rev)
|
||||
if cl.Unasked != nil {
|
||||
b.logf("%v", cl.Unasked)
|
||||
}
|
||||
@@ -564,9 +511,6 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
|
||||
lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
|
||||
}
|
||||
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
|
||||
b.handLanes(key, len(lanes))
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, l := range lanes {
|
||||
wg.Add(1)
|
||||
@@ -719,14 +663,12 @@ type askNotes struct {
|
||||
b *Builder
|
||||
cl *Claim
|
||||
resumed map[string]string
|
||||
// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
|
||||
step string
|
||||
}
|
||||
|
||||
func (b *Builder) notes(cl *Claim) *askNotes {
|
||||
n := &askNotes{b: b, cl: cl}
|
||||
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
|
||||
n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step
|
||||
n.resumed = maps.Clone(cl.Build.Asks)
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -749,11 +691,7 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
|
||||
v.Asks = map[string]string{}
|
||||
}
|
||||
v.Asks[language] = id
|
||||
if err := n.b.Claimer.Finish(ctx, n.cl, v); err != nil {
|
||||
return err
|
||||
}
|
||||
n.b.handAsk(n.cl.Key, language, id)
|
||||
return nil
|
||||
return n.b.Claimer.Finish(ctx, n.cl, v)
|
||||
}
|
||||
|
||||
// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the
|
||||
@@ -828,7 +766,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
return
|
||||
}
|
||||
}
|
||||
step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2)
|
||||
step, err := b.Asks.Wait(ctx, ask)
|
||||
if err != nil {
|
||||
l.err = fmt.Errorf("%s: %w", l.module, err)
|
||||
return
|
||||
|
||||
@@ -6,10 +6,8 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -178,7 +176,6 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
|
||||
r.b = &Builder{
|
||||
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
|
||||
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
|
||||
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
|
||||
}
|
||||
var err error
|
||||
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
|
||||
@@ -618,413 +615,27 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
|
||||
}
|
||||
}
|
||||
|
||||
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
|
||||
// drop from their stream (Dropper).
|
||||
type goneAsks struct {
|
||||
mu sync.Mutex
|
||||
n int
|
||||
why func(a delivery.StepAsk) string
|
||||
dropped []string
|
||||
}
|
||||
|
||||
func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.dropped = append(g.dropped, a.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
|
||||
g.mu.Lock()
|
||||
g.n++
|
||||
g.mu.Unlock()
|
||||
return g.why(a), nil
|
||||
}
|
||||
|
||||
func (g *goneAsks) asked() int {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.n
|
||||
}
|
||||
|
||||
// waitForAsks waits until n asks were put.
|
||||
func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
|
||||
}
|
||||
}
|
||||
return r.asks.Asks()
|
||||
}
|
||||
|
||||
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
|
||||
// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up)
|
||||
// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
|
||||
// again; the key is done when a node takes it.
|
||||
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
t.Run("waits in its stream: never asked again", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||
gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }}
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
waitForAsks(t, r, 1)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no ask was put")
|
||||
}
|
||||
if gone.asked() == 0 {
|
||||
t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
|
||||
}
|
||||
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
}
|
||||
})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
askings int
|
||||
// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
|
||||
step string
|
||||
byHand bool
|
||||
always bool // Gone says every ask cannot be answered, not only the first
|
||||
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
|
||||
puts int
|
||||
done bool
|
||||
asking int // the asking the event says, 0 when none is said
|
||||
// again, when set, is a request to build the key again by hand (BuildAgain, issue 1126 note 72) of the key as
|
||||
// it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it
|
||||
// is refused, "" when it is built again as a new revision and its ask id answered.
|
||||
again bool
|
||||
key *delivery.BuilderBuild
|
||||
known string
|
||||
refuse string
|
||||
// edit changes the module the builder has read, busy marks the key looked at (1) or asked by hand (2), failed is a part of
|
||||
// the error the key then holds when its build ends before its asks are put.
|
||||
edit func(m *delivery.Module)
|
||||
busy int
|
||||
failed string
|
||||
// during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards.
|
||||
during bool
|
||||
}{
|
||||
{name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a key looked at by the builder now is refused, not built twice", again: true, refuse: "is being looked at", busy: 1,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a key already asked by hand is refused, its watch kept", again: true, refuse: "is being looked at", busy: 2,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *delivery.Module) { m.Manifest = "" },
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing",
|
||||
edit: func(m *delivery.Module) {
|
||||
m.Toolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}
|
||||
},
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a build that fails before its asks says the key failed, not that it was asked", again: true, failed: "cannot be read",
|
||||
edit: func(m *delivery.Module) { m.Manifest = mdigest('b') },
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a failed key is written building as a new revision and asks the seat", again: true,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}},
|
||||
{name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
|
||||
{name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}},
|
||||
{name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}},
|
||||
{name: "build again: a key with no build is refused", again: true, refuse: "has no build key"},
|
||||
{name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64),
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
|
||||
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
|
||||
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
|
||||
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
|
||||
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) { // 1945
|
||||
r := newRig(t, goBundle)
|
||||
if c.again {
|
||||
var rev uint64
|
||||
if c.key != nil {
|
||||
c.key.Manifest = manifestD
|
||||
rev = putBuild(t, r.st, r.key, *c.key)
|
||||
}
|
||||
m, err := delivery.DecodeModule(moduleValue(t, "modules/m"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.known != "" {
|
||||
m.FP2 = c.known
|
||||
}
|
||||
if c.edit != nil {
|
||||
c.edit(&m)
|
||||
}
|
||||
r.b.init()
|
||||
var stored atomic.Int64
|
||||
if c.during {
|
||||
r.asks.Answer = nil
|
||||
r.b.Stored = func(context.Context, []string) ([]string, error) { stored.Add(1); return nil, nil }
|
||||
}
|
||||
switch c.busy {
|
||||
case 1:
|
||||
r.b.inflight[r.key] = true
|
||||
case 2:
|
||||
r.b.hands[r.key] = &handWatch{}
|
||||
}
|
||||
r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked
|
||||
log := &logged{}
|
||||
r.b.Logf = log.logf
|
||||
res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second)
|
||||
if c.during {
|
||||
// The trunk's modules value arrives while the by-hand build waits for its ask: it is queued, and run after it.
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || res.Revision <= rev || stored.Load() != 1 {
|
||||
t.Fatalf("key %+v, answer %+v, %d asks of the store; want built from the new revision and the queued look run once after it", v, res, stored.Load())
|
||||
}
|
||||
return
|
||||
}
|
||||
r.b.Wait()
|
||||
after := r.st.revs[r.key]
|
||||
if c.refuse != "" {
|
||||
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || len(r.asks.Asks()) != 0 || after != rev {
|
||||
t.Fatalf("refused %q (revision %d, %d asks, key at revision %d was %d); want refused with %q and nothing written or asked", res.Refused, res.Revision, len(r.asks.Asks()), after, rev, c.refuse)
|
||||
}
|
||||
return
|
||||
}
|
||||
if c.failed != "" {
|
||||
if !strings.Contains(res.Failed, c.failed) || res.Revision <= rev || len(res.Asks) != 0 || len(r.asks.Asks()) != 0 {
|
||||
t.Fatalf("answer %+v after %d asks; want a new revision (was %d), no ask and the key's failure %q", res, len(r.asks.Asks()), rev, c.failed)
|
||||
}
|
||||
return
|
||||
}
|
||||
asks := r.asks.Asks()
|
||||
if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID {
|
||||
t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision {
|
||||
t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision)
|
||||
}
|
||||
if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") {
|
||||
t.Errorf("the why is not logged: %v", log.lines)
|
||||
}
|
||||
return
|
||||
}
|
||||
r.asks.Answer = nil
|
||||
r.b.Askings = c.askings
|
||||
var said []AskedAgain
|
||||
var mu sync.Mutex
|
||||
r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
|
||||
first := ""
|
||||
if c.step != "" {
|
||||
first = "mvl0frv18d1960"
|
||||
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||
Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
|
||||
r.st.vals[r.key] = data
|
||||
}
|
||||
gone := &goneAsks{}
|
||||
dead := &buildrun.DeadLetters{}
|
||||
if c.dead {
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
|
||||
}
|
||||
if !c.byHand && !c.dead {
|
||||
gone.why = func(a delivery.StepAsk) string {
|
||||
if c.always || a.ID == first || first == "" {
|
||||
return "it waits first in line while a node took a later ask"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
if c.dead {
|
||||
dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
|
||||
At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
|
||||
}
|
||||
if c.byHand {
|
||||
waitForAsks(t, r, 1)
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no lane of the key waits")
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.done {
|
||||
asks := waitForAsks(t, r, c.puts)
|
||||
if first == "" && !c.byHand {
|
||||
first = asks[0].ID
|
||||
}
|
||||
last := asks[len(asks)-1]
|
||||
if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
|
||||
t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
|
||||
}
|
||||
r.asks.End(r.answer(last))
|
||||
}
|
||||
r.b.Wait()
|
||||
v := valueOf(t, r.st, r.key)
|
||||
if n := len(r.asks.Asks()); n != c.puts {
|
||||
t.Errorf("%d asks put, want %d", n, c.puts)
|
||||
}
|
||||
if c.done && v.State != delivery.BuildDone {
|
||||
t.Errorf("the key = %+v, want done from the ask asked again", v)
|
||||
}
|
||||
if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
|
||||
t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
|
||||
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
|
||||
}
|
||||
gone.mu.Unlock()
|
||||
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
|
||||
t.Errorf("said %+v, want once at asking %d", said, c.asking)
|
||||
}
|
||||
if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
|
||||
t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
|
||||
put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||
taken := func(kind string, at time.Time) *Taken {
|
||||
return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at}
|
||||
}
|
||||
later := taken(delivery.AskDelivery, put.Add(time.Minute))
|
||||
// ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none).
|
||||
ahead := func(n int, step string) []Queued {
|
||||
var qs []Queued
|
||||
for i := range n {
|
||||
at := put.Add(-time.Duration(n-i) * time.Minute)
|
||||
qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step})
|
||||
}
|
||||
return qs
|
||||
}
|
||||
seen := func(qs []Queued, last *Taken) AskSeen {
|
||||
return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
seen AskSeen
|
||||
want string // a part of the why, "" when it can be answered
|
||||
waiting int
|
||||
}{
|
||||
{"a node took it", AskSeen{Stream: "S", Step: true}, "", 0},
|
||||
{"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0},
|
||||
{"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0},
|
||||
{"first in line, nothing taken since", seen(nil, nil), "", 0},
|
||||
{"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||
{"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2},
|
||||
{"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||
// Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours.
|
||||
{"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0},
|
||||
{"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0},
|
||||
{"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0},
|
||||
{"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3},
|
||||
} {
|
||||
got := c.seen.Why()
|
||||
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting {
|
||||
t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting)
|
||||
}
|
||||
}
|
||||
if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() {
|
||||
t.Errorf("AskedAt does not read an ask id's time")
|
||||
}
|
||||
})
|
||||
t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
|
||||
at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||
state := func(vals map[string]string) StateRead {
|
||||
return StateRead{Keys: func() ([]string, error) {
|
||||
var ks []string
|
||||
for k := range vals {
|
||||
ks = append(ks, k)
|
||||
}
|
||||
return ks, nil
|
||||
}, Get: func(k string) (json.RawMessage, time.Time, error) {
|
||||
if v, ok := vals[k]; ok {
|
||||
return json.RawMessage(v), at, nil
|
||||
}
|
||||
return nil, time.Time{}, nil
|
||||
}}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey("m", fp1)
|
||||
done, _ := delivery.BuilderBuildKey("d", fp1)
|
||||
building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||
Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
|
||||
finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
|
||||
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
|
||||
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
|
||||
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
out, err := r.Stuck("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
|
||||
if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
|
||||
t.Errorf("dead_letters = %v, want not heard, saying why", dl)
|
||||
}
|
||||
stuck := out["stuck"].([]Stuck)
|
||||
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
|
||||
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
|
||||
}
|
||||
a := stuck[0].Asks
|
||||
if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
|
||||
a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
|
||||
t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
|
||||
}
|
||||
|
||||
// 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its
|
||||
// own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked
|
||||
// only for the rows kept.
|
||||
var gets, written, inFlight, maxFlight atomic.Int64
|
||||
vals := map[string]string{}
|
||||
for i := 0; i < 3000; i++ {
|
||||
k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i))
|
||||
vals[k] = string(finished)
|
||||
if (i/100)%10 == 0 {
|
||||
vals[k] = string(building)
|
||||
}
|
||||
}
|
||||
slow := state(vals)
|
||||
get := slow.Get
|
||||
slow.Get = func(k string) (json.RawMessage, time.Time, error) {
|
||||
gets.Add(1)
|
||||
n := inFlight.Add(1)
|
||||
for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() {
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
inFlight.Add(-1)
|
||||
raw, _, err := get(k)
|
||||
return raw, time.Time{}, err
|
||||
}
|
||||
slow.Written = func(string) time.Time { written.Add(1); return at }
|
||||
for _, row := range []struct {
|
||||
module string
|
||||
stuck int
|
||||
gets int64
|
||||
cut bool
|
||||
}{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} {
|
||||
gets.Store(0)
|
||||
written.Store(0)
|
||||
out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, cut := out["cut"]
|
||||
if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets {
|
||||
t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut)
|
||||
}
|
||||
}
|
||||
if maxFlight.Load() < 2 {
|
||||
t.Errorf("the values were read one at a time")
|
||||
}
|
||||
})
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||
}
|
||||
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
}
|
||||
}
|
||||
|
||||
// heldAsks says an ask is held when its id is among held.
|
||||
|
||||
@@ -1,338 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
|
||||
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
|
||||
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
|
||||
// builder's state reasks, which the builder watches and answers there.
|
||||
const (
|
||||
StuckTool = "builder_stuck"
|
||||
AskAgainTool = "builder_ask_again"
|
||||
// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
|
||||
BuildAgainTool = "builder_build_again"
|
||||
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
|
||||
ReAsksState = "reasks"
|
||||
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
|
||||
)
|
||||
|
||||
// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
|
||||
// read it, its answer.
|
||||
type ReAsk struct {
|
||||
V int `json:"v"`
|
||||
Why string `json:"why"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
// Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again.
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
// Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new
|
||||
// revision (building) and the ask ids it put by language.
|
||||
Refused bool `json:"refused,omitempty"`
|
||||
// Pending says the builder answered before the key was claimed or all its asks were put: neither a refusal nor done.
|
||||
// Failed is the error of a key written building that failed before its asks were put.
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
Failed string `json:"failed,omitempty"`
|
||||
Revision uint64 `json:"revision,omitempty"`
|
||||
Asks map[string]string `json:"asks,omitempty"`
|
||||
}
|
||||
|
||||
// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
|
||||
const ModeBuild = "build"
|
||||
|
||||
// DecodeReAsk reads a reasks value.
|
||||
func DecodeReAsk(data []byte) (ReAsk, error) {
|
||||
var r ReAsk
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return r, err
|
||||
}
|
||||
if r.V != 1 {
|
||||
return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
|
||||
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the build is asked again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value.
|
||||
func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the key is built again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
|
||||
type StateRead struct {
|
||||
Keys func() ([]string, error)
|
||||
Get func(key string) (json.RawMessage, time.Time, error)
|
||||
// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
|
||||
// only for a row they keep, and only when Get did not say.
|
||||
Written func(key string) time.Time
|
||||
}
|
||||
|
||||
// StuckAsk is one ask a key waits on.
|
||||
type StuckAsk struct {
|
||||
Language string `json:"language"`
|
||||
BuildModule string `json:"build_module"`
|
||||
ID string `json:"id"`
|
||||
Asking int `json:"asking"`
|
||||
// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
|
||||
// state on its node since its start.
|
||||
Asked time.Time `json:"asked,omitzero"`
|
||||
Step string `json:"step"`
|
||||
}
|
||||
|
||||
// Stuck is one build key that waits on an ask.
|
||||
type Stuck struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
State string `json:"state"`
|
||||
Step string `json:"step"`
|
||||
By string `json:"by"`
|
||||
Since time.Time `json:"since,omitzero"`
|
||||
Age string `json:"age,omitempty"`
|
||||
Asks []StuckAsk `json:"asks"`
|
||||
ReAsk *ReAsk `json:"reask,omitempty"`
|
||||
Unasked bool `json:"unasked,omitempty"`
|
||||
NoneStep int `json:"asks_with_no_step"`
|
||||
}
|
||||
|
||||
// StuckReader reads what builder_stuck answers.
|
||||
type StuckReader struct {
|
||||
Builds StateRead
|
||||
ReAsks StateRead
|
||||
// Steps are the build modules' steps states, by build module.
|
||||
Steps map[string]StateRead
|
||||
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
|
||||
Hearing StateRead
|
||||
Now func() time.Time
|
||||
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
|
||||
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
|
||||
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
|
||||
const (
|
||||
ReadWorkers = 16
|
||||
MaxReads = 1000
|
||||
)
|
||||
|
||||
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
|
||||
func parallel(n int, read func(i int) error) error {
|
||||
var (
|
||||
next atomic.Int64
|
||||
failed atomic.Bool
|
||||
once sync.Once
|
||||
first error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for w := 0; w < min(ReadWorkers, n); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for !failed.Load() {
|
||||
i := int(next.Add(1) - 1)
|
||||
if i >= n {
|
||||
return
|
||||
}
|
||||
if err := read(i); err != nil {
|
||||
once.Do(func() { first = err })
|
||||
failed.Store(true)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return first
|
||||
}
|
||||
|
||||
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
|
||||
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
|
||||
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
|
||||
func (r StuckReader) deadLetters() map[string]any {
|
||||
if r.Hearing.Get == nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
|
||||
}
|
||||
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
|
||||
if err != nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
|
||||
}
|
||||
a := buildrun.HearingHealth(raw)
|
||||
return map[string]any{"heard": a.Healthy, "why": a.Why}
|
||||
}
|
||||
|
||||
func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
started := time.Now()
|
||||
reads := 0
|
||||
defer func() {
|
||||
if r.Logf != nil {
|
||||
r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
}()
|
||||
keys, err := r.Builds.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now
|
||||
if r.Now != nil {
|
||||
now = r.Now
|
||||
}
|
||||
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
|
||||
stepKeys := map[string]map[string]string{}
|
||||
for bm, s := range r.Steps {
|
||||
ks, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
|
||||
}
|
||||
byID := make(map[string]string, len(ks))
|
||||
for _, k := range ks {
|
||||
if _, id, err := delivery.ParseStepKey(k); err == nil {
|
||||
byID[id] = k
|
||||
}
|
||||
}
|
||||
stepKeys[bm] = byID
|
||||
}
|
||||
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
|
||||
// parallel, at most MaxReads of them.
|
||||
type parsed struct{ key, mod, fp string }
|
||||
var matching []parsed
|
||||
for _, key := range keys {
|
||||
mod, fp, err := delivery.ParseBuilderBuildKey(key)
|
||||
if err != nil || (module != "" && mod != module) {
|
||||
continue
|
||||
}
|
||||
matching = append(matching, parsed{key, mod, fp})
|
||||
}
|
||||
sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
|
||||
cut := len(matching) > MaxReads
|
||||
total := len(matching)
|
||||
if cut {
|
||||
matching = matching[:MaxReads]
|
||||
}
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
ats := make([]time.Time, len(matching))
|
||||
if err := parallel(len(matching), func(i int) error {
|
||||
raw, at, err := r.Builds.Get(matching[i].key)
|
||||
if err == nil {
|
||||
values[i], ats[i] = raw, at
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reads += len(matching)
|
||||
out := []Stuck{}
|
||||
for i, m := range matching {
|
||||
key, mod, fp := m.key, m.mod, m.fp
|
||||
raw, at := values[i], ats[i]
|
||||
if raw == nil {
|
||||
continue
|
||||
}
|
||||
v, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
|
||||
continue
|
||||
}
|
||||
if at.IsZero() && r.Builds.Written != nil {
|
||||
at = r.Builds.Written(key)
|
||||
}
|
||||
s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
|
||||
if !at.IsZero() {
|
||||
s.Age = now().Sub(at).Round(time.Second).String()
|
||||
}
|
||||
for language, id := range v.Asks {
|
||||
bm, _ := delivery.BuildModuleOf(language)
|
||||
a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
|
||||
if t, ok := idTime(id); ok {
|
||||
a.Asked = t
|
||||
}
|
||||
if st, ok := r.Steps[bm]; ok {
|
||||
if k, ok := stepKeys[bm][id]; ok {
|
||||
reads++
|
||||
if raw, _, err := st.Get(k); err == nil && raw != nil {
|
||||
if step, err := delivery.DecodeStep(raw); err == nil {
|
||||
a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if a.Step == "none" {
|
||||
s.NoneStep++
|
||||
}
|
||||
s.Asks = append(s.Asks, a)
|
||||
}
|
||||
sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
|
||||
s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
|
||||
if r.ReAsks.Get != nil {
|
||||
reads++
|
||||
if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
|
||||
if ra, err := DecodeReAsk(raw); err == nil {
|
||||
s.ReAsk = &ra
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if !out[i].Since.Equal(out[j].Since) {
|
||||
return out[i].Since.Before(out[j].Since)
|
||||
}
|
||||
return out[i].Key < out[j].Key
|
||||
})
|
||||
answer := map[string]any{"stuck": out, "count": len(out),
|
||||
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
|
||||
answer["dead_letters"] = r.deadLetters()
|
||||
if cut {
|
||||
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// StuckBuildModules are the build modules whose steps builder_stuck reads.
|
||||
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
|
||||
|
||||
// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
|
||||
func idTime(id string) (time.Time, bool) {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil || ms <= 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.UnixMilli(ms).UTC(), true
|
||||
}
|
||||
@@ -23,21 +23,18 @@ type state struct {
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
@@ -87,13 +84,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||
@@ -115,14 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.SecretsOwner != "mesh-delivery-builder" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
|
||||
if want := []string{"builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
t.Errorf("emits = %v, want %v", m.Emits, want)
|
||||
}
|
||||
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the builder has none", k)
|
||||
}
|
||||
@@ -137,7 +127,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
procs = append(procs, r)
|
||||
}
|
||||
if r["type"] == "tool" || r["type"] == "seat" {
|
||||
t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r)
|
||||
t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
|
||||
}
|
||||
}
|
||||
if len(procs) != 1 {
|
||||
@@ -159,13 +149,8 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
}
|
||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
|
||||
t.Errorf("tools artifact = %+v", a) // 1945
|
||||
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
|
||||
t.Errorf("the tools' source %s: %v", a.From, err)
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
|
||||
|
||||
@@ -2,15 +2,6 @@
|
||||
"module": "mesh-delivery-builder",
|
||||
"version": "1",
|
||||
"slug": "builder",
|
||||
"tools": [
|
||||
"builder_ask_again",
|
||||
"builder_build_again",
|
||||
"builder_seat_queue",
|
||||
"builder_stuck"
|
||||
],
|
||||
"emits": [
|
||||
"build.asked-again"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported",
|
||||
@@ -21,10 +12,7 @@
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 },
|
||||
{ "name": "hearing", "history": 1 },
|
||||
{ "name": "instances", "history": 1 },
|
||||
{ "name": "queues", "history": 1 },
|
||||
{ "name": "reasks", "history": 1 }
|
||||
{ "name": "instances", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
@@ -33,7 +21,6 @@
|
||||
"mesh-build-node.steps",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules"
|
||||
@@ -90,17 +77,6 @@
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-builder",
|
||||
"binary": "mesh-delivery-builder"
|
||||
},
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-builder-tools",
|
||||
"binary": "mesh-delivery-builder-tools",
|
||||
"loads": [
|
||||
"mesh-delivery-builder-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -7,45 +7,16 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// reads counts the values this call read from the bus: the runtime launches the tools for one call, so the count is the call's.
|
||||
var reads atomic.Int64
|
||||
|
||||
// cutAnswer says in the answer that it was cut, and how to narrow it.
|
||||
func cutAnswer(answer map[string]any, cut string) map[string]any {
|
||||
if cut != "" {
|
||||
answer["cut"], answer["note"] = true, cut
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// logged runs every tool with one line to stderr per call: the tool, the values it read and the time it took.
|
||||
func logged(tools []stdio.Tool) []stdio.Tool {
|
||||
for i := range tools {
|
||||
name, run := tools[i].Name, tools[i].Run
|
||||
tools[i].Run = func(args map[string]any) (any, error) {
|
||||
reads.Store(0)
|
||||
start := time.Now()
|
||||
out, err := run(args)
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %s read %d values in %s (error: %v)\n", name, reads.Load(), time.Since(start).Round(time.Millisecond), err)
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
return tools
|
||||
}
|
||||
|
||||
func read(s stdio.KeptState) check.StateRead {
|
||||
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
|
||||
reads.Add(1)
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
@@ -54,36 +25,10 @@ func read(s stdio.KeptState) check.StateRead {
|
||||
}}
|
||||
}
|
||||
|
||||
// runsOf says which commit the mesh runs of a module from the indexer's modules state, as the checker's own record of it does
|
||||
// (the commit the module's record was computed at); "" when the record is not readable.
|
||||
type runsOf struct{ modules check.StateRead }
|
||||
|
||||
func (r runsOf) CommitOf(module string) string {
|
||||
if r.modules.Get == nil {
|
||||
return ""
|
||||
}
|
||||
raw, err := r.modules.Get(module)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return ""
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return m.Commit
|
||||
}
|
||||
|
||||
// stateStore is a state the tools write: the tries.
|
||||
type stateStore struct {
|
||||
check.StateRead
|
||||
create func(key string, value []byte) error
|
||||
entry func(key string) (json.RawMessage, uint64, error)
|
||||
update func(key string, value []byte, revision uint64) error
|
||||
}
|
||||
|
||||
func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
|
||||
func (s stateStore) Update(key string, value []byte, revision uint64) error {
|
||||
return s.update(key, value, revision)
|
||||
}
|
||||
|
||||
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
|
||||
@@ -93,21 +38,19 @@ func (s stateStore) Create(key string, value []byte) error { return s.create(k
|
||||
// Trying is what the try's tools work on: the tries, and the gate's trunk heads.
|
||||
type Trying struct {
|
||||
Tries check.TryStore
|
||||
Ender check.TryEnder
|
||||
Trunks func() (map[string]string, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them.
|
||||
// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
|
||||
func TryTools(tr Trying) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{
|
||||
{Name: check.TryTool,
|
||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
|
||||
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
@@ -135,8 +78,7 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}},
|
||||
{Name: check.TryReadTool,
|
||||
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
|
||||
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
@@ -145,28 +87,6 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}
|
||||
return check.ReadTry(tr.Tries, str(args, "id"))
|
||||
}},
|
||||
{Name: check.TryEndTool,
|
||||
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
|
||||
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
|
||||
"takes it within the step bound. A try that ended already is refused.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if tr.Ender == nil {
|
||||
return nil, fmt.Errorf("the tries state cannot be written here")
|
||||
}
|
||||
now := time.Now
|
||||
if tr.Now != nil {
|
||||
now = tr.Now
|
||||
}
|
||||
t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,8 +100,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
group, _ := args["group"].(string)
|
||||
vs, cut, err := r.Verdicts(group)
|
||||
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
|
||||
vs, err := r.Verdicts(group)
|
||||
return map[string]any{"verdicts": vs}, err
|
||||
}},
|
||||
{Name: check.PullReadTool,
|
||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
||||
@@ -202,8 +122,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
level, _ := args["level"].(string)
|
||||
digest, _ := args["digest"].(string)
|
||||
kept, cut, err := r.Kept(level, digest)
|
||||
return cutAnswer(map[string]any{"kept": kept}, cut), err
|
||||
kept, err := r.Kept(level, digest)
|
||||
return map[string]any{"kept": kept}, err
|
||||
}},
|
||||
{Name: check.PreviewsReadTool,
|
||||
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
|
||||
@@ -218,18 +138,15 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
if head != "" && len(head) < 7 {
|
||||
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
|
||||
}
|
||||
ps, bad, cut, err := r.Previews(repository, head)
|
||||
return cutAnswer(map[string]any{"previews": ps, "unreadable": bad}, cut), err
|
||||
ps, bad, err := r.Previews(repository, head)
|
||||
return map[string]any{"previews": ps, "unreadable": bad}, err
|
||||
}},
|
||||
{Name: check.StatusReadTool,
|
||||
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
|
||||
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
||||
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
||||
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
|
||||
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
|
||||
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
|
||||
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||
}
|
||||
@@ -248,9 +165,7 @@ func main() {
|
||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
PreviewState: read(stdio.State("previews")),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
|
||||
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState))}
|
||||
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
|
||||
tries := stdio.State(check.TriesState)
|
||||
source := stdio.State("mesh-delivery-gate.source")
|
||||
trunks := func() (map[string]string, error) {
|
||||
@@ -262,22 +177,14 @@ func main() {
|
||||
return s.Heads, err
|
||||
}
|
||||
r.Trunks = trunks
|
||||
store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}, entry: func(key string) (json.RawMessage, uint64, error) {
|
||||
reads.Add(1)
|
||||
e, err := tries.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}, update: func(key string, value []byte, revision uint64) error {
|
||||
_, err := tries.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
|
||||
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
|
||||
trying := Trying{
|
||||
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}},
|
||||
Trunks: trunks,
|
||||
}
|
||||
if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -53,19 +53,13 @@ func (m *memStore) Create(key string, value []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
|
||||
func (m *memStore) Update(key string, value []byte, revision uint64) error {
|
||||
m.vals[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
|
||||
store := &memStore{vals: map[string]json.RawMessage{}}
|
||||
tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" {
|
||||
if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
|
||||
t.Fatalf("tools = %+v", tools)
|
||||
}
|
||||
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||
@@ -80,14 +74,6 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
|
||||
t.Errorf("read = %s, %v", raw, err)
|
||||
}
|
||||
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||
t.Errorf("end = %v, %v", ended, err)
|
||||
}
|
||||
if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
|
||||
t.Error("an ended try was ended again")
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
|
||||
err.Error() != "no level of novox/hq is checked by the checker" {
|
||||
t.Errorf("hq: %v", err)
|
||||
|
||||
@@ -121,12 +121,5 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return parseManifest(kept.JSON)
|
||||
}
|
||||
|
||||
// parseManifest reads a manifest the indexer kept as the indexer and the assigner read it (issues 1751, 1906): a config value that
|
||||
// falls back to a predefined key is judged by the merge gate in the catalogue, against the predefined.json at its root that this
|
||||
// process never holds, so it is not unreadable here (novox/hq ADR 0348 §7).
|
||||
func parseManifest(raw string) (catalogue.Manifest, error) {
|
||||
return catalogue.ParseManifestForInputs([]byte(raw))
|
||||
return catalogue.ParseManifest([]byte(kept.JSON))
|
||||
}
|
||||
|
||||
@@ -24,8 +24,6 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -81,8 +79,6 @@ type fakeBuildModules struct {
|
||||
js jetstream.JetStream
|
||||
ctx context.Context
|
||||
conflict atomic.Bool
|
||||
// hold is a commit whose fetch never answers: its ask stays taken, as one lost in a bus reload (issue 734).
|
||||
hold atomic.Value
|
||||
mu sync.Mutex
|
||||
calls map[string]int // by `<module>.<tool>`
|
||||
testTool *testing.T
|
||||
@@ -102,9 +98,6 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
|
||||
var v any
|
||||
switch {
|
||||
case strings.HasSuffix(tool, "_fetch"):
|
||||
if h, _ := f.hold.Load().(string); h != "" && args["commit"] == h {
|
||||
<-f.ctx.Done()
|
||||
}
|
||||
name := "go-fetch-clean"
|
||||
if f.conflict.Load() {
|
||||
name = "go-fetch-conflict"
|
||||
@@ -305,10 +298,6 @@ func startBed(t *testing.T) *bed {
|
||||
}
|
||||
b.tools[module] = kv
|
||||
}
|
||||
// What the controller keeps of the messages a consumer gave up on (issue 734).
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildrun.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The checker's stream of events, which the indexer and the forge bridge hear.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -316,8 +305,7 @@ func startBed(t *testing.T) *bed {
|
||||
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
|
||||
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
|
||||
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
|
||||
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
|
||||
delivery.TurnsBucket: delivery.TurnsHistory} {
|
||||
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -424,12 +412,11 @@ func pull(repository string, number int, head string) delivery.CandidatePull {
|
||||
func (b *bed) start(t *testing.T, env map[string]string) {
|
||||
t.Helper()
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: "n1", Module: "mesh-delivery-checker", User: "n1.mesh-delivery-checker", Password: "pw"})
|
||||
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: "n1", Module: "mesh-delivery-checker", User: "n1.mesh-delivery-checker", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The settle window of a group's heads (check.DefaultSettle) is cut short: each bed's candidate is written whole.
|
||||
e := map[string]string{"MESH_BROKER_FILE": broker, "MESH_CHECKER_SETTLE": "10ms"}
|
||||
e := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
for k, v := range env {
|
||||
e[k] = v
|
||||
}
|
||||
@@ -520,12 +507,9 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
|
||||
}
|
||||
}
|
||||
|
||||
// TestACheckOrACarryOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every
|
||||
// package taken from the first's kept verdicts: no test step runs and no second kept key appears. Then, through the real bus
|
||||
// (novox/hq ADR 0548 §3, §5): a third candidate is not claimed while the gate's merge turn on its repository stands, and is
|
||||
// claimed when the turn's key is deleted; and a move of the trunk carries the first verdict on the recheck stream by its
|
||||
// unchanged digests, running no test, the verdict naming the trunk it was carried from.
|
||||
func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2, ADR 0548 §3, §5
|
||||
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
|
||||
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
|
||||
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
|
||||
b := startBed(t)
|
||||
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||
b.start(t, nil)
|
||||
@@ -542,42 +526,6 @@ func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 128
|
||||
if got := keptCount(t, b); got != kept {
|
||||
t.Errorf("%d kept keys, was %d", got, kept)
|
||||
}
|
||||
|
||||
// §3: the gate's turn on the repository holds a new check off; its delete claims it.
|
||||
at := time.Now().UTC()
|
||||
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tk, _ := delivery.TurnsKey(repo)
|
||||
if _, err := b.buckets[delivery.TurnsBucket].Put(b.ctx, tk, turn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond) // the checker hears the turn
|
||||
third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333"))
|
||||
time.Sleep(time.Second)
|
||||
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err)
|
||||
}
|
||||
if err := b.buckets[delivery.TurnsBucket].Delete(b.ctx, tk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.verdict(t, third, ended)
|
||||
|
||||
// §5: the trunk moves; the first verdict is carried by its digests, nothing runs.
|
||||
tests = b.fake.count("mesh-build-go.build_go_test")
|
||||
const moved = "abababababababababababababababababababab"
|
||||
b.setTrunk(t, map[string]string{repo: moved})
|
||||
c := b.verdict(t, first, func(v delivery.Verdict) bool { return ended(v) && v.Trunk[repo] == moved })
|
||||
if c.State != delivery.VerdictPass || c.CarriedFrom[repo] != trunk || c.Levels[delivery.LevelUnit].Run != 0 || c.Levels[delivery.LevelIntegration].Run != 0 {
|
||||
t.Fatalf("the carried verdict = %+v, want a pass carried from %s with no test run", c, trunk[:12])
|
||||
}
|
||||
if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
|
||||
t.Errorf("build_go_test called %d times on the carry, was %d: a carry of unchanged digests runs nothing", got, tests)
|
||||
}
|
||||
if o := b.owed(t, 7, head1, "success"); !strings.HasPrefix(o.Text, "pass 111111111111 on abababababab | carried from aaaaaaaaaaaa") {
|
||||
t.Errorf("the carried status says %q", o.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func keptCount(t *testing.T, b *bed) int {
|
||||
@@ -728,49 +676,6 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
|
||||
t.Errorf("a try wrote %s: %v", name, keys)
|
||||
}
|
||||
}
|
||||
// Issue 734: a try whose ask the build seat gives up on (the controller keeps its copy in DEAD_LETTERS with the ask's
|
||||
// message id) ends as error saying so, and its head asked again is a new try.
|
||||
b.fake.hold.Store(head2)
|
||||
lost, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps, err := b.js.KeyValue(b.ctx, delivery.StepsBucket("mesh-build-go"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
askID := ""
|
||||
for deadline := time.Now().Add(20 * time.Second); askID == "" && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
keys, _ := steps.Keys(b.ctx)
|
||||
for _, k := range keys {
|
||||
if e, err := steps.Get(b.ctx, k); err == nil && strings.Contains(string(e.Value()), `"running"`) {
|
||||
askID = strings.TrimPrefix(k, "n1.")
|
||||
}
|
||||
}
|
||||
}
|
||||
if askID == "" {
|
||||
t.Fatal("the held try's ask was never taken")
|
||||
}
|
||||
dead := &nats.Msg{Subject: "mesh.events.dead.SEAT_NODE_BUILD_GO.SEAT_NODE_BUILD_GO_CHECK_worker", Header: nats.Header{}}
|
||||
dead.Header.Set(buildrun.DeadHeaderMsgID, askID)
|
||||
dead.Header.Set(buildrun.DeadHeaderDeliveries, "5")
|
||||
dead.Header.Set(buildrun.DeadHeaderStream, "SEAT_NODE_BUILD_GO")
|
||||
dead.Header.Set(buildrun.DeadHeaderGaveUp, time.Now().UTC().Format(time.RFC3339Nano))
|
||||
if _, err := b.js.PublishMsg(b.ctx, dead); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
raw, _ := store.Get(lost)
|
||||
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
|
||||
break
|
||||
}
|
||||
}
|
||||
if last.State != check.TryError || !strings.Contains(last.Said, buildrun.LostWords+" after 5 deliveries") {
|
||||
t.Fatalf("the try whose ask was given up = %+v, want error saying so", last)
|
||||
}
|
||||
if again, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now()); err != nil || again == lost {
|
||||
t.Errorf("its head asked again answered %q, %v, want a new try", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among
|
||||
@@ -797,7 +702,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
|
||||
break
|
||||
}
|
||||
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
|
||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") || len(said.States) != 6 {
|
||||
t.Fatalf("the checker said %+v, want its six states with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
|
||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
|
||||
t.Fatalf("the checker said %+v, want its five states with kept, owed, previews and verdicts readable (1929/AC4; 1887 A5b)", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,16 +2,13 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -29,46 +26,11 @@ type judgeDeps struct {
|
||||
builds check.BuildStates
|
||||
// compositions is the assigner's bucket of compositions; nil leaves every composition not run.
|
||||
compositions check.CompositionReader
|
||||
// nodes names every node with a record; nil hands no names.
|
||||
nodes *nodeNames
|
||||
logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// nodeNames reads every node's name from the node-engine's have, one key `<node>.node` per node with a record (novox/hq
|
||||
// ADR 0567 §12), at each ask: a node enrolled since is named in the next check.
|
||||
type nodeNames struct{ kv jetstream.KeyValue }
|
||||
|
||||
// openNodeNames opens the node-engine's have for the nodes' names alone: it is not watched, so no check waits on a node.
|
||||
func openNodeNames(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) *nodeNames {
|
||||
return &nodeNames{kv: process.OptionalBucket(ctx, js, delivery.HaveBucket, "no check is asked until it is: the names check needs the nodes' names", logf)}
|
||||
}
|
||||
|
||||
func (n nodeNames) Names(ctx context.Context) ([]string, error) {
|
||||
if n.kv == nil {
|
||||
return nil, fmt.Errorf("the bucket %s could not be opened at the checker's start", delivery.HaveBucket)
|
||||
}
|
||||
lister, err := n.kv.ListKeysFiltered(ctx, "*."+delivery.HaveNodeWord)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for key := range lister.Keys() {
|
||||
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
|
||||
out = append(out, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
func newJudge(d judgeDeps) check.Judge {
|
||||
var nodes interface {
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
}
|
||||
if d.nodes != nil {
|
||||
nodes = d.nodes
|
||||
}
|
||||
return &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||
return &check.Examiner{Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf}
|
||||
}
|
||||
|
||||
@@ -131,7 +93,7 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
|
||||
if mk, err := delivery.ManifestsKey(m.Manifest); err == nil {
|
||||
if raw, _, found, err := r.manifests.Get(ctx, mk); err == nil && found {
|
||||
if kept, err := delivery.DecodeManifest(raw); err == nil {
|
||||
if man, err := parseManifest(kept.JSON); err == nil && man.Build != nil {
|
||||
if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil && man.Build != nil {
|
||||
for _, a := range man.Build.Artifacts {
|
||||
if a.Language != "" {
|
||||
set[a.Language] = true
|
||||
@@ -221,8 +183,7 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu
|
||||
return b, err == nil, err
|
||||
}
|
||||
|
||||
// CommitOf is the commit the indexer's record of the module names: the commit its fingerprint first appeared at, so the source
|
||||
// of the build the mesh runs (a later merge that leaves the module alone writes no record, the operator 2026-10-25).
|
||||
// CommitOf is the commit the indexer's record of the module was computed at: the commit of its repository the mesh runs.
|
||||
func (r *repoModules) CommitOf(module string) string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -113,31 +112,3 @@ func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *t
|
||||
t.Errorf("another repository: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest whose config value falls back to a predefined key (ADR 0348 §7) is read, not logged as unreadable for the
|
||||
// catalogue's predefined.json this process never holds: the indexer and the assigner read it with
|
||||
// catalogue.ParseManifestForInputs (issues 1751, 1906). xterm's module.json from mesh-catalog at 835bf22b.
|
||||
func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest := string(raw)
|
||||
data, err := delivery.EncodeManifest(delivery.Manifest{Module: "xterm", JSON: manifest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := memManifests{delivery.ManifestDigest([]byte(manifest)): data}
|
||||
var said []string
|
||||
r := &repoModules{manifests: store, logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
|
||||
putModule(t, r, "xterm", "novox/mesh-catalog", manifest)
|
||||
r.BuildModules("novox/mesh-catalog")
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, "cannot be read") {
|
||||
t.Errorf("the manifest was not read: %s", s)
|
||||
}
|
||||
}
|
||||
if _, err := parseManifest(manifest); err != nil {
|
||||
t.Errorf("manifestReader: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,8 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
|
||||
if !equal(manifest, announced) {
|
||||
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
|
||||
}
|
||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1); tries by the delivery view (build_usage).
|
||||
if want := []string{"kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) {
|
||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1).
|
||||
if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
|
||||
t.Errorf("readable %v, want %v", readable, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// lazyTurns is the gate's merge turns as the checker reads them (novox/hq ADR 0548 §3), opened when the gate's bucket is
|
||||
// there: before each claim it is opened again until it is, so a checker that started before the gate made the bucket holds
|
||||
// checks from the gate's first turn on, with no timer and no restart. Until then it holds nothing, and says so once, loudly,
|
||||
// and once again when it opens.
|
||||
type lazyTurns struct {
|
||||
turns *check.Turns
|
||||
open func() error
|
||||
logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
opened bool
|
||||
said bool
|
||||
}
|
||||
|
||||
// ensure opens the bucket once it is there; a failure is said once.
|
||||
func (l *lazyTurns) ensure() {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.opened || l.open == nil {
|
||||
return
|
||||
}
|
||||
if err := l.open(); err != nil {
|
||||
if !l.said {
|
||||
l.said = true
|
||||
l.logf("NO MERGE TURN HOLDS A CHECK: the gate's bucket %s cannot be read (%v); every check is claimed as if no approved merge waited, "+
|
||||
"and the bucket is opened again before each claim until it is there", delivery.TurnsBucket, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
l.opened = true
|
||||
if l.said {
|
||||
l.logf("the gate's merge turns are read from %s from now on: an approved merge's turn holds checks off its repositories again", delivery.TurnsBucket)
|
||||
}
|
||||
}
|
||||
|
||||
// Holding opens the bucket when it is not yet, then answers what the turns hold.
|
||||
func (l *lazyTurns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
|
||||
l.ensure()
|
||||
return l.turns.Holding(repositories, heads)
|
||||
}
|
||||
@@ -7,7 +7,12 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -31,7 +36,7 @@ import (
|
||||
// eventSubject is the subject of an event of the checker's stream checks (the bus names the stream CHECKS): the indexer's
|
||||
// watch of the previews bucket and the forge bridge's watch of the owed bucket are woken by them.
|
||||
func eventSubject(event string) string {
|
||||
return process.ModuleStreamSubject(delivery.CheckerModule, event)
|
||||
return "mesh.stream." + delivery.CheckerModule + ".event." + event
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -48,6 +53,50 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
|
||||
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func readCredential(path string) (credential, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return credential{}, err
|
||||
}
|
||||
var c credential
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if c.URL == "" || c.User == "" {
|
||||
return credential{}, errors.New("the broker credential names no bus address or user")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinned accepts exactly the certificate with this SHA-256 and no other.
|
||||
func pinned(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// durationEnv reads a duration setting from the environment, def when it is not set.
|
||||
func durationEnv(getenv func(string) string, name string, def time.Duration) (time.Duration, error) {
|
||||
v := strings.TrimSpace(getenv(name))
|
||||
@@ -63,43 +112,67 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
|
||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate; tries by the delivery view.
|
||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
|
||||
var keptStates = []announce.State{
|
||||
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
|
||||
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
|
||||
// Whether DEAD_LETTERS is heard (issue 734): read by the checker's own status tool alone.
|
||||
{Name: "hearing"},
|
||||
{Name: "kept", Readable: true},
|
||||
{Name: "owed", Readable: true},
|
||||
{Name: "previews", Readable: true},
|
||||
// The tries are read by the delivery view, which tells a try's asks from a candidate's by the asks each names (build_usage).
|
||||
{Name: "tries", Readable: true},
|
||||
{Name: "tries"},
|
||||
{Name: "verdicts", Readable: true},
|
||||
}
|
||||
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-checker: ", log.LstdFlags)
|
||||
cred, err := process.ReadBrokerAccount(getenv)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
|
||||
}
|
||||
cred, err := readCredential(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, js, err := cred.Dial()
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
// Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository:
|
||||
// a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945).
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
buckets := map[string]jetstream.KeyValue{}
|
||||
for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket,
|
||||
delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} {
|
||||
if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil {
|
||||
if buckets[name], err = bucket(name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the
|
||||
// buckets are read and written on the connection above.
|
||||
caller, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
account, err := process.ReadAccount(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
caller, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -118,10 +191,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
settleWindow, err := durationEnv(getenv, "MESH_CHECKER_SETTLE", check.DefaultSettle)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
link := strings.TrimSpace(getenv("MESH_CHECKER_LINK"))
|
||||
if link == "" {
|
||||
link = defaultLink
|
||||
@@ -129,37 +198,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
tools := &buildrun.ToolchainSet{}
|
||||
presence := &buildrun.PresenceSet{}
|
||||
trunks := &check.Trunks{}
|
||||
turns := &check.Turns{}
|
||||
lazy := &lazyTurns{turns: turns, logf: logger.Printf}
|
||||
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
|
||||
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the step its asker waits for.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
// state hearing, which checker_status says (dead_letters).
|
||||
dead := &buildrun.DeadLetters{}
|
||||
// The state may not be on the bus yet when a delivery started the checker (the grants part creates it, issue 1790): it
|
||||
// is bound when the bus says it was created, and the latest hearing is written then.
|
||||
hearingKV := process.AwaitBucket(ctx, js, conn, check.HearingBucket, "checker_status cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, []string{delivery.AskGo, delivery.AskNode}, dead,
|
||||
logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||
waiter := buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}
|
||||
ch := &check.Checker{
|
||||
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
|
||||
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
|
||||
Trunks: trunks,
|
||||
Turns: lazy,
|
||||
Bound: checkBound,
|
||||
Link: link,
|
||||
Logf: logger.Printf,
|
||||
Settle: settleWindow,
|
||||
}
|
||||
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
|
||||
// says nothing of the builds, until the checker starts again.
|
||||
var builds check.BuildStates
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil {
|
||||
switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err)
|
||||
default:
|
||||
builds = buildReader{kvStore{kv}}
|
||||
}
|
||||
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
|
||||
@@ -180,7 +238,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return cw, nil
|
||||
}}
|
||||
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
|
||||
announce := func(ctx context.Context, event, key string) {
|
||||
body, _ := json.Marshal(map[string]string{"key": key})
|
||||
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
||||
@@ -188,13 +246,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
|
||||
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
|
||||
ch.Judge = newJudge(judgeDeps{
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
||||
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||
announce: announce, compositions: compositions,
|
||||
nodes: openNodeNames(ctx, js, logger.Printf),
|
||||
})
|
||||
ch.Announce = announce
|
||||
tryer.Judge = ch.Judge
|
||||
@@ -217,12 +273,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// module.
|
||||
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
|
||||
stream := toolchainStreams[module]
|
||||
cons, err := cred.EventConsumer(ctx, js, stream)
|
||||
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the checker starts again: %v", cred.Node, cred.Module, stream, module, err)
|
||||
continue
|
||||
}
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { watch.Reported(m) }, logger.Printf)
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -233,75 +290,55 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no
|
||||
// longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6,
|
||||
// issue 1912), so the node-engine's have is not watched here.
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil {
|
||||
if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.PresenceBucket, err, "no node counts as online until the checker starts again")
|
||||
} else {
|
||||
go func() {
|
||||
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
|
||||
}
|
||||
})
|
||||
}()
|
||||
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
|
||||
default:
|
||||
pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
// The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there
|
||||
// yet no trunk is known, so nothing is checked.
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil {
|
||||
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
default:
|
||||
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) {
|
||||
if e.Key() == delivery.SourceMainKey {
|
||||
trunks.Put(e.Value(), !opOf(e))
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again")
|
||||
} else {
|
||||
go func() {
|
||||
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
|
||||
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
}
|
||||
// The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the
|
||||
// delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start.
|
||||
// Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there
|
||||
// (lazyTurns), so a gate delivered after the checker started holds checks from its first turn on.
|
||||
lazy.open = func() error {
|
||||
kv, err := js.KeyValue(ctx, delivery.TurnsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { turns.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) {
|
||||
if turns.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
ch.Reconsider(ctx) // a turn went or changed: a check it held is claimed now
|
||||
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
|
||||
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
|
||||
}
|
||||
})
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
lazy.ensure()
|
||||
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
|
||||
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) {
|
||||
before := repos.CommitOf(check.NodeEngineModule)
|
||||
repos.Put(ctx, e.Key(), e.Value(), !opOf(e))
|
||||
if after := repos.CommitOf(check.NodeEngineModule); after != before {
|
||||
ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6)
|
||||
}
|
||||
})
|
||||
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
||||
}()
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
|
||||
@@ -7,8 +7,7 @@ import (
|
||||
)
|
||||
|
||||
// 1912: with checks as asks on the build seats' streams, the stream redelivers a lost node's ask (ADR 0525 §6), so the
|
||||
// checker program watches no node-engine have to start again a check that failed for want of a node. It reads the have's
|
||||
// keys only for the nodes' names at each ask (judge.go, novox/hq ADR 0567 §12).
|
||||
// checker program opens no node-engine have to start again a check that failed for want of a node.
|
||||
func TestTheCheckerWatchesNoHave(t *testing.T) {
|
||||
raw, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
|
||||
@@ -28,11 +28,8 @@ import (
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
@@ -41,14 +38,14 @@ type toolchainGate struct {
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
standing map[string]bool // <node>.<module> with an open assignment
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
@@ -70,31 +67,20 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
open := !deleted && openAssignment(value)
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
if g.standing[key] == open {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
g.standing[key] = true
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
@@ -125,58 +111,37 @@ func (g *toolchainGate) Settled() bool {
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||
// otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
if !g.left[module+"/"+key] {
|
||||
g.left[module+"/"+key] = true
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||
delivery.ToolchainsBucket(module), key, module, node)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||
func openAssignment(value []byte) bool {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
return false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
return a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
@@ -190,19 +155,17 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -3,19 +3,10 @@ package main
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
@@ -53,10 +44,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
@@ -72,20 +63,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
@@ -97,122 +88,25 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
set := &buildrun.ToolchainSet{}
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||
gate.Settled()
|
||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
key, value := gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an assigned node's report was not passed on")
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
}
|
||||
|
||||
-236
@@ -1,236 +0,0 @@
|
||||
{
|
||||
"module": "xterm",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"requires": [
|
||||
"x11-display"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-terminal-emulator",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"open"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"xterm_font",
|
||||
"xterm_colours"
|
||||
],
|
||||
"config": {
|
||||
"face": {
|
||||
"type": "string",
|
||||
"description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).",
|
||||
"pattern": "[A-Za-z0-9 ._-]{1,64}",
|
||||
"default": "JetBrainsMono Nerd Font"
|
||||
},
|
||||
"size": {
|
||||
"type": "number",
|
||||
"description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
|
||||
"minimum": 6,
|
||||
"maximum": 72,
|
||||
"predefined": "font-size",
|
||||
"default": 12
|
||||
},
|
||||
"save-lines": {
|
||||
"type": "number",
|
||||
"description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.",
|
||||
"minimum": 0,
|
||||
"maximum": 1000000,
|
||||
"default": 10000
|
||||
},
|
||||
"term-name": {
|
||||
"type": "string",
|
||||
"description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.",
|
||||
"pattern": "[A-Za-z0-9._-]{1,64}",
|
||||
"default": "xterm-256color"
|
||||
},
|
||||
"render-font": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.",
|
||||
"default": true
|
||||
},
|
||||
"bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.",
|
||||
"default": false
|
||||
},
|
||||
"always-bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.",
|
||||
"default": false
|
||||
},
|
||||
"allow-bold-fonts": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.",
|
||||
"default": true
|
||||
},
|
||||
"background": {
|
||||
"type": "string",
|
||||
"description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"foreground": {
|
||||
"type": "string",
|
||||
"description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a8a8a8"
|
||||
},
|
||||
"color0": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"color1": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#9e1828"
|
||||
},
|
||||
"color2": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#008800"
|
||||
},
|
||||
"color3": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#d2bb4b"
|
||||
},
|
||||
"color4": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#414171"
|
||||
},
|
||||
"color5": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#963c59"
|
||||
},
|
||||
"color6": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#418179"
|
||||
},
|
||||
"color7": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bebebe"
|
||||
},
|
||||
"color8": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#666666"
|
||||
},
|
||||
"color9": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bc5766"
|
||||
},
|
||||
"color10": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#61a171"
|
||||
},
|
||||
"color11": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#e7db52"
|
||||
},
|
||||
"color12": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#5085af"
|
||||
},
|
||||
"color13": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a97a99"
|
||||
},
|
||||
"color14": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#6ba4a4"
|
||||
},
|
||||
"color15": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#ffffff"
|
||||
},
|
||||
"copy-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> C"
|
||||
},
|
||||
"paste-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> V"
|
||||
}
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "terminal",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "TERMINAL"
|
||||
},
|
||||
"template": "files/env-terminal"
|
||||
},
|
||||
{
|
||||
"collection": "shell:xresources:normal",
|
||||
"name": "xresources-normal",
|
||||
"template": "files/xresources-normal"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "xterm"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/xterm-tools",
|
||||
"binary": "xterm-tools",
|
||||
"loads": [
|
||||
"xterm-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -65,14 +63,11 @@ func (w *toolchainWatch) ensure(module string) error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||
return nil
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d h1:oJREw0ggKqJWJ5DEISSL0i2ZC5XqdXAidxo9NJ74TCc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,355 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// One check per delivery (novox/hq ADR 0548): the checker's part. A merge turn holds new checks off its repository (§3);
|
||||
// a trunk or sibling move never ends a running check and carries its verdict once it ended (§1, §5, §6); only a head git
|
||||
// refused gets a whole check after a move (§7); the line of a carried verdict says so (§8).
|
||||
|
||||
// turnOf is a turn's value on a repository for a delivery of the heads hash.
|
||||
func turnOf(t testing.TB, repo, group, heads string) []byte {
|
||||
t.Helper()
|
||||
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
|
||||
data, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: group, Heads: heads, Approved: at, Since: at}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func turnKey(t testing.TB, repo string) string {
|
||||
t.Helper()
|
||||
k, err := delivery.TurnsKey(repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// waitFor polls a condition the checker's goroutines make true, failing after two seconds.
|
||||
func waitFor(t testing.TB, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(2 * time.Second); !ok(); time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%s did not happen", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pullWithFP is a pull request touching the module m at the fingerprint `src2:` and 64 times the hex digit d.
|
||||
func pullWithFP(repo string, number int, head string, d string) delivery.CandidatePull {
|
||||
p := pullOf(repo, number, head)
|
||||
p.Modules = []string{"m"}
|
||||
p.Fps = map[string]delivery.CandidateModule{"m": {FP2: "src2:" + strings.Repeat(d, 64)}}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestOneCheckPerDelivery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for name, run := range map[string]func(t *testing.T){
|
||||
// §3: no claim while a turn names one of its repositories; a claim on the turn's delete event.
|
||||
"a turn on its repository holds a new check off until the turn goes": func(t *testing.T) {
|
||||
j := passJudge()
|
||||
r := newRig(t, j)
|
||||
turns := &Turns{}
|
||||
r.ch.Turns = turns
|
||||
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if j.count() != 0 || len(r.verdicts.vals) != 0 {
|
||||
t.Fatalf("judged %d times with a turn on %s, verdicts %v", j.count(), repoA, r.verdicts.vals)
|
||||
}
|
||||
if held := r.ch.Held()[keyOf(t, "feat/x", cand)]; !strings.Contains(held, repoA) || !strings.Contains(held, "fix/other") {
|
||||
t.Errorf("the hold is said as %q, want the turn's repository and group", held)
|
||||
}
|
||||
if !turns.Put(turnKey(t, repoA), nil, true) {
|
||||
t.Fatal("the turn's delete changed nothing")
|
||||
}
|
||||
r.ch.Reconsider(ctx) // what the watch of turns does on the delete
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
|
||||
t.Errorf("judged %d times after the turn went, want once", j.count())
|
||||
}
|
||||
},
|
||||
"a turn on another repository, or of its own heads, holds nothing": func(t *testing.T) {
|
||||
j := passJudge()
|
||||
r := newRig(t, j)
|
||||
turns := &Turns{}
|
||||
r.ch.Turns = turns
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
c, _ := delivery.DecodeCandidate(cand)
|
||||
turns.Put(turnKey(t, repoSDK), turnOf(t, repoSDK, "fix/other", "0123456789ab"), false)
|
||||
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "feat/x", GateHeads(c)), false)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 {
|
||||
t.Errorf("judged %d times, want once: the turns hold another repository and its own merge", j.count())
|
||||
}
|
||||
},
|
||||
// §3, the race: a check claimed before the gate's turn is judged on the commit it read, never restarted.
|
||||
"a check claimed before a turn was written runs to its verdict": func(t *testing.T) {
|
||||
release, started := make(chan struct{}), make(chan struct{})
|
||||
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
|
||||
close(started)
|
||||
<-release
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
turns := &Turns{}
|
||||
r.ch.Turns = turns
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
<-started
|
||||
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
|
||||
r.ch.Reconsider(ctx)
|
||||
close(release)
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
|
||||
t.Errorf("judged %d times; the running check is not ended by the turn", j.count())
|
||||
}
|
||||
},
|
||||
// §1, §5: a trunk move during a running check calls the judge once, writes its verdict, keeps the status pending (a pass
|
||||
// on an old trunk is never shown), then asks one carry, on the recheck stream, and no check.
|
||||
"a trunk move during a check ends nothing and asks one carry after it": func(t *testing.T) {
|
||||
var r *rig
|
||||
var kinds []string
|
||||
var from []map[string]string
|
||||
judged := make(chan struct{})
|
||||
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
|
||||
if call == 1 {
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
||||
r.ch.TrunkMoved(ctx) // as the watch of the source says it
|
||||
close(judged)
|
||||
} else if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||
t.Errorf("the status during the carry is %q, want pending: the pass on the old trunk is never shown", s)
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
<-judged
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if j.count() != 2 || kinds[0] != KindNew || kinds[1] != KindTrunk || from[0] != nil || from[1][repoA] != trunkA {
|
||||
t.Fatalf("judged %d times as %v carried from %v; want a whole check, then one carry from %s", j.count(), kinds, from, trunkA[:12])
|
||||
}
|
||||
if v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
|
||||
t.Errorf("the verdict is %s on %v carried from %v; want a pass on the new trunk carried from the old", v.State, v.Trunk, v.CarriedFrom)
|
||||
}
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "success" {
|
||||
t.Errorf("the status after the carry is %q, want success", s)
|
||||
}
|
||||
if r.verdicts.creates[keyOf(t, "feat/x", cand)] != 1 {
|
||||
t.Error("the verdict key was made again: a carry rewrites it")
|
||||
}
|
||||
},
|
||||
// §5: the composition is kept when no fingerprint of a module it composed changed, and runs again otherwise.
|
||||
"a carry keeps a passed composition while the fingerprints stay the same": func(t *testing.T) {
|
||||
var kept []bool
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
|
||||
kept = append(kept, c.Composition != nil)
|
||||
v := passing()
|
||||
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "v0.1.14-x", FPSet: c.FPSet}
|
||||
return Outcome{Verdict: v}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "a"))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx) // the indexer's fingerprints of the candidate stay the same
|
||||
r.ch.Wait()
|
||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "b"))) // a fingerprint changed with the move
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if len(kept) != 3 || kept[0] || !kept[1] || kept[2] {
|
||||
t.Errorf("the composition was kept in the checks %v; want composed, kept on the carry, composed again once a fingerprint changed", kept)
|
||||
}
|
||||
},
|
||||
// §6: a sibling's move puts the verdict out of date, its status pending saying it is carried, and the carry's verdict
|
||||
// names the sibling's old commit; the status follows the carry's verdict. Which packages run is the digest's (only
|
||||
// those that name MESH_CHECK_BESIDE change with a sibling), the build module's part.
|
||||
"a sibling's move is carried and the status follows the carry": func(t *testing.T) {
|
||||
var r *rig
|
||||
var from map[string]string
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||
if call == 2 {
|
||||
from = c.CarriedFrom
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||
t.Errorf("the status during the sibling's carry is %q, want pending", s)
|
||||
}
|
||||
}
|
||||
return Outcome{Verdict: passing(), Beside: map[string]string{catalogueRepository: r.trunks.Heads()[catalogueRepository]}}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS})
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if b := r.verdict(t, keyOf(t, "feat/x", cand)).Beside; b[catalogueRepository] != trunkS {
|
||||
t.Fatalf("the verdict names the siblings %v, want the catalogue at %s", b, trunkS[:12])
|
||||
}
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS2})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if j.count() != 2 || from[catalogueRepository] != trunkS || v.CarriedFrom[catalogueRepository] != trunkS || v.Beside[catalogueRepository] != trunkS2 {
|
||||
t.Errorf("judged %d times, carried from %v, verdict beside %v carried from %v; want one carry over the catalogue's move", j.count(), from, v.Beside, v.CarriedFrom)
|
||||
}
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "success" {
|
||||
t.Errorf("the status after the carry is %q, want success", s)
|
||||
}
|
||||
},
|
||||
// §7: a carry whose merge conflicts fails naming the files and asks no check; a later move onto which the head merges
|
||||
// cleanly asks a whole check, not a carry.
|
||||
"a carry git refuses fails naming the files, and the next move asks a whole check": func(t *testing.T) {
|
||||
var kinds []string
|
||||
var from []map[string]string
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
|
||||
if call == 2 {
|
||||
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "ace",
|
||||
Words: repoA + "#7 " + ConflictWords + "go.mod"}}}, nil
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
r.ch.Reconsider(ctx) // anything else that looks again asks nothing of a conflict
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, keyOf(t, "feat/x", cand)); j.count() != 2 || !IsConflictFailure(v) || !strings.Contains(v.Failed.Words, "go.mod") {
|
||||
t.Fatalf("judged %d times, verdict %+v; want the carry failed naming go.mod and nothing asked after it", j.count(), v.Failed)
|
||||
}
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if j.count() != 3 || kinds[1] != KindTrunk || from[1] == nil || kinds[2] != KindNew || from[2] != nil {
|
||||
t.Errorf("checks %v carried from %v; want a whole check, a carry, then a whole check after the conflict", kinds, from)
|
||||
}
|
||||
},
|
||||
// §8, §6: the line of a carried verdict names the head on the new trunk commit first, as the gate and the merge tool
|
||||
// read a line, then where it was carried from and what the carry ran and kept; while it is carried it is pending and says so.
|
||||
"the line of a carry says where it was carried from": func(t *testing.T) {
|
||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||
v := passVerdict()
|
||||
v.Trunk = map[string]string{repoA: trunkA}
|
||||
v.Beside = map[string]string{catalogueRepository: trunkS2}
|
||||
v.CarriedFrom = map[string]string{repoA: trunkA2, catalogueRepository: trunkS}
|
||||
line := Line(v, cs, p, nil, nil, nil)
|
||||
if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa | carried from abababababab, mesh-catalog 555555555555 | unit 12 run") {
|
||||
t.Errorf("the carried line = %q", line)
|
||||
}
|
||||
if head, trunk := checkWordsOf(line); head != head1[:12] || trunk != trunkA[:12] {
|
||||
t.Errorf("the line names %s on %s first, want the head on the new trunk", head, trunk)
|
||||
}
|
||||
cs.CarriedFrom = map[string]string{repoA: trunkA2}
|
||||
pending := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil)
|
||||
if !strings.HasPrefix(pending, "being carried to 111111111111 on aaaaaaaaaaaa | from abababababab") {
|
||||
t.Errorf("the pending line of a carry = %q", pending)
|
||||
}
|
||||
},
|
||||
// §3 (review round 1): two approved merges in one turn whose trunk moved each get their carry; neither holds the other's.
|
||||
"two merges waiting in one turn each get their carry": func(t *testing.T) {
|
||||
var r *rig
|
||||
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
|
||||
r = newRig(t, j)
|
||||
candA, candC := candidateOf(t, "feat/a", pullOf(repoA, 7, head1)), candidateOf(t, "feat/c", pullOf(repoA, 8, head2))
|
||||
r.ch.Candidate(ctx, "feat%2Fa", candA)
|
||||
r.ch.Candidate(ctx, "feat%2Fc", candC)
|
||||
r.ch.Wait()
|
||||
a, _ := delivery.DecodeCandidate(candA)
|
||||
c, _ := delivery.DecodeCandidate(candC)
|
||||
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
|
||||
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repoA, Waiting: []delivery.TurnWait{
|
||||
{Group: "feat/a", Heads: GateHeads(a), Approved: at, Since: at}, {Group: "feat/c", Heads: GateHeads(c), Approved: at.Add(time.Minute), Since: at}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
turns := &Turns{}
|
||||
turns.Put(turnKey(t, repoA), turn, false)
|
||||
r.ch.Turns = turns
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // a third merge moves the trunk under both
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
for _, cand := range [][]byte{candA, candC} {
|
||||
group := map[bool]string{true: "feat/a", false: "feat/c"}[string(cand) == string(candA)]
|
||||
if v := r.verdict(t, keyOf(t, group, cand)); v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
|
||||
t.Errorf("%s: verdict on %v carried from %v; want its carry claimed and done", group, v.Trunk, v.CarriedFrom)
|
||||
}
|
||||
}
|
||||
},
|
||||
// §6 (review round 1): the node-engine read at the commit the mesh runs is named, and a new running commit is carried.
|
||||
"a new running node-engine is carried": func(t *testing.T) {
|
||||
runs := runsAt{nodeEngineModule: trunkS}
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
|
||||
return Outcome{Verdict: passing(), Beside: map[string]string{nodeEngineRepo: runs[nodeEngineModule]}}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
r.ch.Runs = runs
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.ch.TrunkMoved(ctx) // nothing moved: nothing is carried
|
||||
r.ch.Wait()
|
||||
runs[nodeEngineModule] = trunkS2
|
||||
r.ch.TrunkMoved(ctx) // what the indexer's watch does when mesh-host's running commit changes
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if j.count() != 2 || v.CarriedFrom[nodeEngineRepo] != trunkS || v.Beside[nodeEngineRepo] != trunkS2 {
|
||||
t.Errorf("judged %d times, verdict beside %v carried from %v; want one carry over the node-engine's new running commit", j.count(), v.Beside, v.CarriedFrom)
|
||||
}
|
||||
},
|
||||
// §5: a carry that fails names the move it failed on.
|
||||
"a carry that fails names the move": func(t *testing.T) {
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||
if call == 1 {
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}
|
||||
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if v.State != delivery.VerdictFail || v.CarriedFrom != nil || !strings.HasPrefix(v.Failed.Words, "want 4") ||
|
||||
!strings.Contains(v.Failed.Words, repoA+" "+trunkA[:12]+" to "+trunkA2[:12]) {
|
||||
t.Errorf("the failed carry is %s carried from %v with words %q; want the test's words, then the move", v.State, v.CarriedFrom, v.Failed.Words)
|
||||
}
|
||||
},
|
||||
} {
|
||||
t.Run(name, run)
|
||||
}
|
||||
}
|
||||
|
||||
// checkWordsOf is the head and trunk a line names first, as the gate reads it (mesh-delivery-gate onestatus.go checkLine).
|
||||
func checkWordsOf(line string) (string, string) {
|
||||
for _, part := range strings.Split(line, " | ") {
|
||||
f := strings.Fields(part)
|
||||
for i := 0; i+2 < len(f); i++ {
|
||||
if f[i+1] == "on" && len(f[i]) == 12 && len(f[i+2]) == 12 {
|
||||
return f[i], f[i+2]
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -97,19 +96,9 @@ func notJudged(v delivery.Verdict) bool {
|
||||
return v.Composition != nil && v.Composition.State == delivery.CompositionNotRun && v.Composition.Words == delivery.AssignerNotRunning
|
||||
}
|
||||
|
||||
// unanswered says whether a verdict failed only on its composition, and that only because nodes the change places did
|
||||
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (a composition is read
|
||||
// only into a passing verdict), so a later composition of the same heads is taken in on its event, with no new check
|
||||
// (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
|
||||
func unanswered(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && v.Composition != nil && v.Composition.State == delivery.CompositionFail &&
|
||||
strings.Contains(v.Composition.Words, delivery.CompositionUnanswered) && v.Failed.Words == v.Composition.Words
|
||||
}
|
||||
|
||||
// Composed takes in a composition the assigner wrote after the check's verdict said it was not judged because the
|
||||
// assigner was not running (the assigner started since), or after it failed only because placed nodes had not answered
|
||||
// (unanswered): the verdict and its status are written again with it. A composition of heads no candidate holds now, or
|
||||
// of a verdict that judged its composition otherwise, is left.
|
||||
// assigner was not running (the assigner started since): the verdict and its status are written again with it. A
|
||||
// composition of heads no candidate holds now, or of a verdict that judged its composition, is left.
|
||||
func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdict) {
|
||||
ch.init()
|
||||
if comp.State == delivery.CompositionWaits {
|
||||
@@ -134,21 +123,10 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
|
||||
return
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(cur)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case v.State == delivery.VerdictPass && notJudged(v):
|
||||
case unanswered(v) && comp.Words != v.Composition.Words:
|
||||
// The nodes answered since (or another failure stands now): the verdict is its tests' pass with this composition.
|
||||
v.State, v.Failed = delivery.VerdictPass, delivery.FailedFirst{}
|
||||
default:
|
||||
if err != nil || v.State != delivery.VerdictPass || !notJudged(v) {
|
||||
return
|
||||
}
|
||||
withComposition(&v, comp)
|
||||
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
||||
v.Composition.FPSet = cs.FPSet
|
||||
}
|
||||
data, err := delivery.EncodeVerdict(v)
|
||||
if err != nil {
|
||||
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
|
||||
|
||||
@@ -113,27 +113,6 @@ func TestTheAssignerStartingBringsTheComposition(t *testing.T) {
|
||||
t.Errorf("the status's line says %q", o.Text)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0559 §5: a verdict that failed only because a placed node had not answered takes the same heads' later
|
||||
// composition in on its event, a pass making it pass with no new check; a composition failing the same way is left.
|
||||
ch, cs := checkerWithNotJudged(t)
|
||||
silent := delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionFail, SDK: "x", Machine: "g14",
|
||||
Words: "g14 (offline) " + delivery.CompositionUnanswered}
|
||||
ch.Composed(context.Background(), silent)
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
comp delivery.CompositionVerdict
|
||||
state string
|
||||
}{
|
||||
{"the same failure again", silent, delivery.VerdictFail},
|
||||
{"the node's answer", delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionPass, SDK: "x"}, delivery.VerdictPass},
|
||||
} {
|
||||
ch.Composed(context.Background(), row.comp)
|
||||
raw, _, _, _ := ch.Verdicts.Get(context.Background(), cs.Key)
|
||||
if v, err := delivery.DecodeVerdict(raw); err != nil || v.State != row.state || v.Composition == nil || v.Composition.State != row.comp.State {
|
||||
t.Errorf("%s: the verdict is %+v, %v", row.name, v, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkerWithNotJudged is a checker holding a candidate whose passing verdict said its composition was not judged.
|
||||
|
||||
@@ -12,18 +12,8 @@ type BesideRef struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
// atTrunk says the sibling was read at its trunk's head, so a move of that trunk puts the verdict out of date and is
|
||||
// carried (novox/hq ADR 0548 §6). Never sent: the node-engine at the commit the mesh runs, mesh-sdk at a pin and a
|
||||
// group's member at its head do not move with a trunk.
|
||||
atTrunk bool
|
||||
// runs says the sibling was read at the commit the mesh runs (the node-engine, the indexer's record of mesh-host): a new
|
||||
// running commit puts the verdict out of date and is carried (ADR 0548 §6). Never sent.
|
||||
runs bool
|
||||
}
|
||||
|
||||
// NodeEngineModule is the module whose running commit the node-engine sibling is read at.
|
||||
const NodeEngineModule = nodeEngineModule
|
||||
|
||||
const (
|
||||
catalogueRepository = "novox/mesh-catalog"
|
||||
nodeEngineRepo = "novox/mesh-host"
|
||||
@@ -61,21 +51,18 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
switch {
|
||||
case repo == catalogueRepository:
|
||||
if head := heads[repo]; head != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: c.member(repo) == nil})
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head})
|
||||
}
|
||||
case repo == nodeEngineRepo:
|
||||
commit := ""
|
||||
if j.Runs != nil {
|
||||
commit = j.Runs.CommitOf(nodeEngineModule)
|
||||
}
|
||||
atTrunk := false
|
||||
if commit == "" {
|
||||
commit, atTrunk = heads[repo], true
|
||||
commit = heads[repo]
|
||||
}
|
||||
if commit != "" {
|
||||
// Named in the verdict either way (ADR 0548 §6, issue 568's own example): at the commit the mesh runs, a new
|
||||
// running commit puts the verdict out of date; at the trunk, a trunk move does (Checker.besideNow).
|
||||
out = append(out, BesideRef{Repository: repo, Commit: commit, atTrunk: atTrunk, runs: !atTrunk})
|
||||
out = append(out, BesideRef{Repository: repo, Commit: commit})
|
||||
}
|
||||
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
|
||||
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
|
||||
@@ -95,7 +82,7 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
if member := c.member(repo); member != nil {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
|
||||
} else if head := heads[repo]; head != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true})
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head})
|
||||
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
|
||||
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
|
||||
// checker that cannot place it says so, where placing none left that test to skip.
|
||||
@@ -121,13 +108,3 @@ func refOf(number int) string {
|
||||
}
|
||||
return "refs/pull/" + strconv.Itoa(number) + "/head"
|
||||
}
|
||||
|
||||
// besideAtTrunk are the siblings of refs read at their trunk's head that the case does not merge, by repository: what a
|
||||
// verdict names as Beside (ADR 0548 §6).
|
||||
func besideAtTrunk(c *Case, refs []BesideRef, into map[string]string) {
|
||||
for _, r := range refs {
|
||||
if _, merged := c.Trunk[r.Repository]; (r.atTrunk || r.runs) && !merged {
|
||||
into[r.Repository] = r.Commit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A change that adds or modifies a compiled program fails (issue 1577): it is build output, `go build` in a module's folder
|
||||
// writes one beside go.mod, and `git add -A` takes it. The build module's fetch step, which holds the merged clone, reads the
|
||||
// first bytes of each changed file and reports the programs (mesh-sdk buildstep.MergeInto); the checker judges the answer.
|
||||
|
||||
// NoProgramsReported is the warning of a build module whose fetch answered no programs field: an older module, which cannot
|
||||
// tell, so the check does not refuse a program it did not look for.
|
||||
const NoProgramsReported = "the build module did not report programs: a compiled program in the change is not refused"
|
||||
|
||||
// checkBinariesOf fails a pull request whose change adds or modifies a compiled program, naming each file. It judges the
|
||||
// first fetch answer of the pull request, whichever build module gave it; a build module that reports no programs leaves a
|
||||
// warning on the check and passes it.
|
||||
func (st *state) checkBinariesOf(t *target) verdictPtr {
|
||||
key := fmt.Sprintf("%s#%d", t.pull.Repository, t.pull.Number)
|
||||
if t.fetched.Merge == "" {
|
||||
return nil // no merge was made: nothing was changed to judge
|
||||
}
|
||||
if t.fetched.Programs == nil {
|
||||
st.c.note(fmt.Sprintf("%s: %s (%s)", key, NoProgramsReported, t.module))
|
||||
st.c.warn(key, fmt.Sprintf("a compiled program is not refused: %s reported no programs", t.module))
|
||||
st.j.logf("%s: %s (%s)", key, NoProgramsReported, t.module)
|
||||
return nil
|
||||
}
|
||||
var found []string
|
||||
for _, p := range *t.fetched.Programs {
|
||||
if p.Commit != "" {
|
||||
found = append(found, fmt.Sprintf("%s: a compiled program added in %s and removed later: it stays in history; squash the branch (issue 1577)", p.Path, short12(p.Commit)))
|
||||
continue
|
||||
}
|
||||
found = append(found, p.Path+": a compiled program is build output: remove it and add it to .gitignore (issue 1577)")
|
||||
}
|
||||
if len(found) > 0 {
|
||||
return failed(t.node, fmt.Sprintf("%s: %s", key, strings.Join(found, "; ")))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func TestAChangeWhoseFetchReportsProgramsFailsNamingEachFile(t *testing.T) { // 1577
|
||||
prog := func(p ...string) *[]ProgramFile {
|
||||
out := []ProgramFile{}
|
||||
for _, f := range p {
|
||||
path, commit, _ := strings.Cut(f, "@") // `path@commit`: added by a commit of the branch and removed later
|
||||
out = append(out, ProgramFile{Path: path, Kind: "ELF", Commit: commit})
|
||||
}
|
||||
return &out
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
programs *[]ProgramFile
|
||||
fail []string // paths named; none passes
|
||||
warned bool // the build module reported no programs field
|
||||
}{
|
||||
{"none found", prog(), nil, false},
|
||||
{"one program", prog("modules/m/m"), []string{"modules/m/m"}, false},
|
||||
{"two programs", prog("a/x", "b/y"), []string{"a/x", "b/y"}, false},
|
||||
{"added and removed in the branch", prog("gone/prog@0123456789abcdef0123"), []string{"gone/prog: a compiled program added in 0123456789ab and removed later: it stays in history; squash the branch (issue 1577)"}, false},
|
||||
{"an older build module reports nothing", nil, nil, true},
|
||||
} {
|
||||
b := newBench(t)
|
||||
b.fetch.Merge, b.fetch.Programs = "mmmm", c.programs
|
||||
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||
v := b.judge(t, cs).Verdict
|
||||
if got := slices.ContainsFunc(cs.Notes(), func(n string) bool { return strings.Contains(n, NoProgramsReported) }); got != c.warned {
|
||||
t.Errorf("%s: warned = %v, want %v (%v)", c.name, got, c.warned, cs.Notes())
|
||||
}
|
||||
line := Line(v, cs, cs.Pulls[0], nil, nil, nil)
|
||||
if got := strings.Contains(line, "a compiled program is not refused: mesh-build-go reported no programs"); got != c.warned {
|
||||
t.Errorf("%s: the status line holds the warning = %v, want %v: %s", c.name, got, c.warned, line)
|
||||
}
|
||||
if len(c.fail) == 0 {
|
||||
if v.State != delivery.VerdictPass {
|
||||
t.Errorf("%s: verdict = %+v", c.name, v)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if v.State != delivery.VerdictFail {
|
||||
t.Errorf("%s: verdict = %+v, want a failure", c.name, v)
|
||||
continue
|
||||
}
|
||||
for _, p := range c.fail {
|
||||
want := p
|
||||
if !strings.Contains(p, " added in ") {
|
||||
want = p + ": a compiled program is build output: remove it and add it to .gitignore (issue 1577)"
|
||||
}
|
||||
if !strings.Contains(v.Failed.Words, want) {
|
||||
t.Errorf("%s: words %q do not name %s", c.name, v.Failed.Words, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ReadWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
|
||||
// runtime to the bus, so reading a bucket of thousands of keys one after the other outlasts the 30 s a tool call is given
|
||||
// (issue 1827, as the delivery view's issue 1945). It is a copy of the view's: a few lines are not worth a group delivery
|
||||
// through mesh-sdk (rules-are-weighed).
|
||||
const ReadWorkers = 16
|
||||
|
||||
// MaxReads bounds the values one call of a read tool reads; past it the answer says it was cut and how to narrow it.
|
||||
const MaxReads = 1000
|
||||
|
||||
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error; once one failed no
|
||||
// further read starts.
|
||||
func parallel(n int, read func(i int) error) error {
|
||||
var (
|
||||
next atomic.Int64
|
||||
failed atomic.Bool
|
||||
once sync.Once
|
||||
first error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for w := 0; w < min(ReadWorkers, n); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for !failed.Load() {
|
||||
i := int(next.Add(1) - 1)
|
||||
if i >= n {
|
||||
return
|
||||
}
|
||||
if err := read(i); err != nil {
|
||||
once.Do(func() { first = err })
|
||||
failed.Store(true)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return first
|
||||
}
|
||||
|
||||
// capKeys keeps the first MaxReads keys and answers a note when it cut the rest, "" otherwise.
|
||||
func capKeys(keys []string, narrow string) ([]string, string) {
|
||||
if len(keys) <= MaxReads {
|
||||
return keys, ""
|
||||
}
|
||||
return keys[:MaxReads], fmt.Sprintf("read %d of %d matching keys and stopped; %s", MaxReads, len(keys), narrow)
|
||||
}
|
||||
|
||||
// newestFirst sorts step keys (<node>.<id>) by id, newest first: an id is the time in base 36, then six hex digits, so the
|
||||
// longer id is the later and equal lengths compare as text. A key that is no step key goes last.
|
||||
func newestFirst(keys []string) {
|
||||
id := func(k string) (string, bool) {
|
||||
_, id, err := delivery.ParseStepKey(k)
|
||||
return id, err == nil
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
a, aok := id(keys[i])
|
||||
b, bok := id(keys[j])
|
||||
switch {
|
||||
case aok != bok:
|
||||
return aok
|
||||
case len(a) != len(b):
|
||||
return len(a) > len(b)
|
||||
case a != b:
|
||||
return a > b
|
||||
}
|
||||
return keys[i] > keys[j]
|
||||
})
|
||||
}
|
||||
@@ -43,43 +43,12 @@ type Case struct {
|
||||
Try *TryOptions
|
||||
|
||||
notesMu sync.Mutex
|
||||
// warnings are, by pull request, what part of its check did not run (Case.warn).
|
||||
warnings map[string]string
|
||||
notes []string
|
||||
// asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks).
|
||||
asks []string
|
||||
// Watch, when set, hears each ask the check puts, as it is put (step nil), and each value its step key takes while
|
||||
// the check waits for it: the node that took it, the phase it says, its end. A try writes them into its record, so a
|
||||
// running try names its asks and where each stands (issue 1890). Called from the check's goroutines; it must not block
|
||||
// for long.
|
||||
Watch func(askID string, step *delivery.Step)
|
||||
notes []string
|
||||
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
||||
Progress *Progress
|
||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
|
||||
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
|
||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
|
||||
// seats' recheck stream, every other check on check (ADR 0536 §4).
|
||||
Kind string
|
||||
// CarriedFrom are, for the carry of a verdict that passed, the trunk and sibling commits it was judged on (ADR 0548 §5);
|
||||
// nil for a whole check.
|
||||
CarriedFrom map[string]string
|
||||
// FPSet is the candidate's fingerprint set (Candidate.FPSet): a composition names the set it judged.
|
||||
FPSet string
|
||||
// Composition is, for a carry, the composition that passed on the same fingerprint set: it is kept and not asked again,
|
||||
// since no fingerprint of a module it composed changed (ADR 0548 §5). Nil composes.
|
||||
Composition *delivery.Composition
|
||||
}
|
||||
|
||||
// Asked records the id of an ask the check put.
|
||||
func (c *Case) Asked(id string) {
|
||||
c.notesMu.Lock()
|
||||
c.asks = append(c.asks, id)
|
||||
c.notesMu.Unlock()
|
||||
}
|
||||
|
||||
// Asks are the ids of the asks the check put, in order.
|
||||
func (c *Case) Asks() []string {
|
||||
c.notesMu.Lock()
|
||||
defer c.notesMu.Unlock()
|
||||
return append([]string(nil), c.asks...)
|
||||
}
|
||||
|
||||
// Progress is the step a check is at, written by the check and read when its bound passes.
|
||||
@@ -111,11 +80,11 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}, FPSet: c.FPSet}
|
||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
|
||||
for _, p := range c.Pulls {
|
||||
switch head, known := trunk[p.Repository]; {
|
||||
case !known || head == "":
|
||||
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: %s", p.Repository, p.Number, UntrunkedWordsOf([]string{p.Repository})))
|
||||
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
|
||||
default:
|
||||
cs.Pulls = append(cs.Pulls, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Trunk: head, Modules: p.Modules, Fps: p.Fps})
|
||||
cs.Trunk[p.Repository] = head
|
||||
@@ -133,42 +102,6 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
||||
return cs
|
||||
}
|
||||
|
||||
// UntrunkedWords open the words of a verdict, and of a status, of a pull request that cannot be judged because the gate's
|
||||
// source.main names no trunk head of its repository (issue 1422): never a failure of the change, and judged whole once the
|
||||
// trunk head is known. It is said loudly, never left silently "new".
|
||||
const UntrunkedWords = "cannot be judged: the trunk head of "
|
||||
|
||||
// UntrunkedWordsOf are those words for the repositories whose trunk head is not known, in order, each named once.
|
||||
func UntrunkedWordsOf(repos []string) string {
|
||||
rs := slices.Clone(repos)
|
||||
slices.Sort(rs)
|
||||
rs = slices.Compact(rs)
|
||||
return UntrunkedWords + strings.Join(rs, ", ") + " is not known: the gate's source.main names none, so there is no commit to merge it onto"
|
||||
}
|
||||
|
||||
// IsUntrunked says whether a verdict says its heads cannot be judged for want of a trunk head (UntrunkedWords).
|
||||
func IsUntrunked(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, UntrunkedWords)
|
||||
}
|
||||
|
||||
// leftOf are the pull requests of a candidate whose repository has no trunk head known, with Trunk empty, in repository and
|
||||
// number order; nil when every repository's is known.
|
||||
func leftOf(c delivery.Candidate, trunk map[string]string) []Pull {
|
||||
var out []Pull
|
||||
for _, p := range c.Pulls {
|
||||
if trunk[p.Repository] == "" {
|
||||
out = append(out, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Modules: p.Modules, Fps: p.Fps})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Repository != out[j].Repository {
|
||||
return out[i].Repository < out[j].Repository
|
||||
}
|
||||
return out[i].Number < out[j].Number
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// GateHeads is the hash the gate matches the members an Approve covers by (mesh-catalog mesh-delivery-gate onestatus.go
|
||||
// headsOf): delivery.HeadsHash over every pull request of the candidate, hq's included, as
|
||||
// `<owner/name lowercase>#<number>=<full head sha lowercase>`. A lone pull request is a group of one. Empty for a candidate
|
||||
@@ -229,23 +162,6 @@ func (c *Case) note(s string) {
|
||||
}
|
||||
}
|
||||
|
||||
// warn says of a pull request (pullKey) that a part of its check did not run, in words for its status line (issue 1577).
|
||||
func (c *Case) warn(key, words string) {
|
||||
c.notesMu.Lock()
|
||||
defer c.notesMu.Unlock()
|
||||
if c.warnings == nil {
|
||||
c.warnings = map[string]string{}
|
||||
}
|
||||
c.warnings[key] = words
|
||||
}
|
||||
|
||||
// Warning is what warn said of a pull request, or "".
|
||||
func (c *Case) Warning(key string) string {
|
||||
c.notesMu.Lock()
|
||||
defer c.notesMu.Unlock()
|
||||
return c.warnings[key]
|
||||
}
|
||||
|
||||
// Notes are the words noted on the case, in the order they were noted.
|
||||
func (c *Case) Notes() []string {
|
||||
c.notesMu.Lock()
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -43,9 +42,6 @@ type Outcome struct {
|
||||
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
|
||||
// that changes none is not named.
|
||||
Judged map[string]string
|
||||
// Beside are the siblings the check read at their trunk's head, by repository (ADR 0548 §6): the verdict names them, and
|
||||
// a move of one is carried.
|
||||
Beside map[string]string
|
||||
}
|
||||
|
||||
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
|
||||
@@ -69,14 +65,6 @@ type Checker struct {
|
||||
Owed KV
|
||||
Judge Judge
|
||||
Trunks interface{ Heads() map[string]string }
|
||||
// Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its
|
||||
// running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk.
|
||||
Runs interface{ CommitOf(module string) string }
|
||||
// Turns are the gate's merge turns (ADR 0548 §3): no check is claimed of a repository a turn names, but the check of the
|
||||
// waiting delivery's own heads; nil holds none.
|
||||
Turns interface {
|
||||
Holding(repositories []string, heads string) (string, delivery.TurnWait, bool)
|
||||
}
|
||||
// Bound is how long one check may run; none when 0 (DefaultBound).
|
||||
Bound time.Duration
|
||||
// Link is the base of the address a status points at; the verdict key follows.
|
||||
@@ -85,10 +73,6 @@ type Checker struct {
|
||||
Announce Announcer
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
// Settle is how long a group's heads must stay the same after the candidate last changed them before its check starts
|
||||
// (issue 1890): the members of a group are announced one after another, and a check of each partial set is thrown
|
||||
// away at the next. None when 0; DefaultSettle in the module.
|
||||
Settle time.Duration
|
||||
|
||||
once sync.Once
|
||||
wg sync.WaitGroup
|
||||
@@ -99,78 +83,15 @@ type Checker struct {
|
||||
tried map[string]bool // by verdict key: looked at since this process started
|
||||
epoch uint64 // changes of the nodes seen since this process started
|
||||
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
|
||||
held map[string]string // by verdict key: the turn that holds its check off, as last said
|
||||
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
|
||||
settling map[string]*window // by candidates key: the window its heads settle in
|
||||
}
|
||||
|
||||
// liveCheck is a check running in this process: the candidates key of its group and the end of its run.
|
||||
type liveCheck struct {
|
||||
group string
|
||||
end context.CancelCauseFunc
|
||||
}
|
||||
|
||||
// window is a group's settle window: aborted with errUnsettled when its heads change again or the group goes, and with
|
||||
// errSettled to end it at once.
|
||||
type window struct{ abort context.CancelCauseFunc }
|
||||
|
||||
var (
|
||||
errUnsettled = errors.New("the group's heads changed again")
|
||||
errSettled = errors.New("the group's heads settled")
|
||||
)
|
||||
|
||||
// settleNow ends the settle window of every group at once, as if Settle had passed.
|
||||
func (ch *Checker) settleNow() {
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
for _, w := range ch.settling {
|
||||
w.abort(errSettled)
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultSettle is the module's Settle (issue 1890). On 2026-10-26 the three members of a group were announced within 2.8
|
||||
// seconds of each other (20:31:52.3, :53.7, :55.1 UTC), a session opening or pushing them one API call after another;
|
||||
// fifteen seconds holds five times that, and costs a check of 10 to 18 minutes under 2.5 % of its time. A member that comes
|
||||
// later still is no waste beyond its asks already taken: the check of the heads it replaced is ended at once.
|
||||
const DefaultSettle = 15 * time.Second
|
||||
|
||||
// EndedWords open the words of a verdict the checker ended without judging it, its heads no candidate's any more (issue
|
||||
// 1890): never a failure of the change, and judged whole if a candidate holds those heads again.
|
||||
const EndedWords = "ended: "
|
||||
|
||||
// IsEnded says whether a verdict was ended without being judged (EndedWords).
|
||||
func IsEnded(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, EndedWords)
|
||||
}
|
||||
|
||||
// superseded is the cause a running check is ended with: the heads that replaced its own, or none when its group went.
|
||||
type superseded struct{ by string }
|
||||
|
||||
func (s superseded) Error() string {
|
||||
if s.by == "" {
|
||||
return EndedWords + "no candidate holds these heads any more: the group was closed or merged"
|
||||
}
|
||||
return EndedWords + "superseded by " + s.by
|
||||
}
|
||||
|
||||
func (ch *Checker) init() {
|
||||
ch.once.Do(func() {
|
||||
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
|
||||
ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{}
|
||||
ch.nodesFail = map[string]uint64{}
|
||||
})
|
||||
}
|
||||
|
||||
// endOthers ends every check of the group running in this process whose verdict key is not keep: no candidate holds its
|
||||
// heads any more (issue 1890). Called with ch.mu held.
|
||||
func (ch *Checker) endOthers(group, keep, by string) {
|
||||
for k, l := range ch.live {
|
||||
if l.group == group && k != keep {
|
||||
l.end(superseded{by: by})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ch *Checker) logf(format string, args ...any) {
|
||||
if ch.Logf != nil {
|
||||
ch.Logf(format, args...)
|
||||
@@ -203,62 +124,17 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
|
||||
ch.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
|
||||
return
|
||||
}
|
||||
vkey, _ := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
ch.mu.Lock()
|
||||
prev, had := ch.known[key]
|
||||
ch.known[key] = c
|
||||
// A check of heads this candidate no longer holds is ended at once (issue 1890): a member's new head, a member joining
|
||||
// or leaving.
|
||||
ch.endOthers(key, vkey, GateHeads(c))
|
||||
moved := !had || prev.HeadsHash != c.HeadsHash
|
||||
if ch.Settle > 0 && (moved || ch.settling[key] != nil) {
|
||||
// The group's heads changed: its check waits until they have been the same for Settle, so a group whose members
|
||||
// are announced one after another is checked once, whole. A later change starts the window again. The wait is a
|
||||
// context's bound, no timer (ADR 0437 §8).
|
||||
if w := ch.settling[key]; w != nil {
|
||||
w.abort(errUnsettled)
|
||||
}
|
||||
base, abort := context.WithCancelCause(ctx)
|
||||
w := &window{abort: abort}
|
||||
ch.settling[key] = w
|
||||
ch.wg.Add(1)
|
||||
go func() {
|
||||
defer ch.wg.Done()
|
||||
wait, done := context.WithTimeout(base, ch.Settle)
|
||||
defer done()
|
||||
<-wait.Done()
|
||||
settled := errors.Is(wait.Err(), context.DeadlineExceeded) || errors.Is(context.Cause(wait), errSettled)
|
||||
ch.mu.Lock()
|
||||
if !settled || ch.settling[key] != w {
|
||||
ch.mu.Unlock()
|
||||
return // changed again, forgotten, or the checker stops
|
||||
}
|
||||
delete(ch.settling, key)
|
||||
now, ok := ch.known[key]
|
||||
ch.mu.Unlock()
|
||||
abort(nil)
|
||||
if ok {
|
||||
ch.consider(ctx, key, now)
|
||||
}
|
||||
}()
|
||||
ch.mu.Unlock()
|
||||
return
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
ch.consider(ctx, key, c)
|
||||
}
|
||||
|
||||
// Forget takes a candidate out (its key was deleted): a check of its heads still running is ended, as no candidate holds
|
||||
// them any more.
|
||||
// Forget takes a candidate out (its key was deleted).
|
||||
func (ch *Checker) Forget(key string) {
|
||||
ch.init()
|
||||
ch.mu.Lock()
|
||||
delete(ch.known, key)
|
||||
if w := ch.settling[key]; w != nil {
|
||||
w.abort(errUnsettled)
|
||||
delete(ch.settling, key)
|
||||
}
|
||||
ch.endOthers(key, "", "")
|
||||
ch.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -295,59 +171,38 @@ func (ch *Checker) heads() map[string]string {
|
||||
}
|
||||
|
||||
func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) {
|
||||
heads := ch.heads()
|
||||
cs := CaseOf(c, heads)
|
||||
if left := leftOf(c, heads); len(left) > 0 {
|
||||
ch.untrunked(ctx, c, left, cs == nil)
|
||||
}
|
||||
cs := CaseOf(c, ch.heads())
|
||||
if cs == nil {
|
||||
return // nothing to judge: no trunk known, which untrunked said
|
||||
return // nothing to judge: no trunk known
|
||||
}
|
||||
cs.Queued = ch.now()
|
||||
ch.mu.Lock()
|
||||
if ch.settling[key] != nil {
|
||||
ch.mu.Unlock()
|
||||
return // its heads are settling: the window's end considers it (Settle)
|
||||
}
|
||||
if ch.inflight[cs.Key] {
|
||||
ch.again[cs.Key] = true // looked at once more when the check ends, so a change of the trunk is never lost
|
||||
ch.mu.Unlock()
|
||||
return
|
||||
}
|
||||
ch.inflight[cs.Key] = true
|
||||
run, end := context.WithCancelCause(ctx)
|
||||
ch.live[cs.Key] = liveCheck{group: key, end: end}
|
||||
ch.mu.Unlock()
|
||||
ch.wg.Add(1)
|
||||
go func() {
|
||||
defer ch.wg.Done()
|
||||
vkey := cs.Key
|
||||
defer func() {
|
||||
ch.mu.Lock()
|
||||
delete(ch.inflight, vkey)
|
||||
delete(ch.again, vkey)
|
||||
delete(ch.live, vkey)
|
||||
ch.mu.Unlock()
|
||||
end(nil)
|
||||
}()
|
||||
for {
|
||||
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
|
||||
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
|
||||
// ADR 0536 §4). The checker holds no place and no capacity of its own.
|
||||
ch.prepare(ctx, cs)
|
||||
ch.try(ctx, run, key, cs)
|
||||
cs.Kind = ch.kindOf(ctx, cs)
|
||||
ch.try(ctx, key, cs)
|
||||
ch.mu.Lock()
|
||||
more := ch.again[vkey]
|
||||
delete(ch.again, vkey)
|
||||
ch.mu.Unlock()
|
||||
if !more || run.Err() != nil {
|
||||
more := ch.again[cs.Key]
|
||||
delete(ch.again, cs.Key)
|
||||
if !more || ctx.Err() != nil {
|
||||
delete(ch.inflight, cs.Key)
|
||||
ch.mu.Unlock()
|
||||
return
|
||||
}
|
||||
// Looked at again for the same heads only: a candidate that moved to other heads has its own check (issue 1890).
|
||||
ch.mu.Unlock()
|
||||
if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil {
|
||||
if next.Key != vkey {
|
||||
return
|
||||
}
|
||||
cs = next
|
||||
}
|
||||
cs.Queued = ch.now()
|
||||
@@ -355,206 +210,30 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
||||
}()
|
||||
}
|
||||
|
||||
// untrunked says, loudly, that pull requests of a candidate cannot be judged because the gate's source.main names no trunk
|
||||
// head of their repository (issue 1422): each gets the status error with UntrunkedWords, and when none of the candidate is
|
||||
// judged (whole) its verdict key holds a verdict that says so, which checker_verdicts and checker_status show. It is logged
|
||||
// once per head: when a status or the verdict is written, never at a look that finds them already said. A verdict of these
|
||||
// heads that says anything else is left. Once the trunk head is known the verdict is stale (it names no trunk) and the heads
|
||||
// are judged whole (prepare, claim).
|
||||
func (ch *Checker) untrunked(ctx context.Context, c delivery.Candidate, left []Pull, whole bool) {
|
||||
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
repos := make([]string, 0, len(left))
|
||||
for _, p := range left {
|
||||
repos = append(repos, p.Repository)
|
||||
}
|
||||
words := UntrunkedWordsOf(repos)
|
||||
said := false
|
||||
if whole {
|
||||
v := delivery.Verdict{Group: c.Group, Heads: GateHeads(c), Trunk: map[string]string{}, State: delivery.VerdictFail,
|
||||
Failed: delivery.FailedFirst{Machine: "checker", Words: words}}
|
||||
data, err := delivery.EncodeVerdict(v)
|
||||
if err != nil {
|
||||
ch.logf("%s: %s, and the verdict cannot say so: %v", key, words, err)
|
||||
return
|
||||
}
|
||||
wrote, err := putOnce(ctx, ch.Verdicts, key, data, func(cur []byte) bool {
|
||||
held, err := delivery.DecodeVerdict(cur)
|
||||
// The same words stand; a verdict that says anything else (a check that ran, one running) is not this one's to replace.
|
||||
return err != nil || !IsUntrunked(held) || held.Failed.Words == words
|
||||
})
|
||||
if err != nil {
|
||||
ch.logf("%s: %s, and the verdict cannot say so: %v", key, words, err)
|
||||
}
|
||||
said = said || wrote
|
||||
}
|
||||
for _, p := range left {
|
||||
okey, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head)
|
||||
if err != nil {
|
||||
ch.logf("%s: %v", key, err)
|
||||
continue
|
||||
}
|
||||
o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "error",
|
||||
Target: strings.TrimSuffix(ch.Link, "/") + "/" + key, Text: clipLine(words, MaxLine)}
|
||||
data, err := delivery.EncodeCheckerOwed(o)
|
||||
if err != nil {
|
||||
ch.logf("%s: the status of %s cannot be written: %v", key, okey, err)
|
||||
continue
|
||||
}
|
||||
wrote, err := putOnce(ctx, ch.Owed, okey, data, func(cur []byte) bool {
|
||||
held, err := delivery.DecodeCheckerOwed(cur)
|
||||
o.V = delivery.Version
|
||||
return err == nil && delivery.SameOwed(held, o)
|
||||
})
|
||||
if err != nil {
|
||||
ch.logf("%s: the status of %s cannot be written: %v", key, okey, err)
|
||||
continue
|
||||
}
|
||||
if wrote {
|
||||
said = true
|
||||
if ch.Announce != nil {
|
||||
ch.Announce(ctx, EventOwedWritten, okey)
|
||||
}
|
||||
}
|
||||
}
|
||||
if said {
|
||||
names := make([]string, 0, len(left))
|
||||
for _, p := range left {
|
||||
names = append(names, fmt.Sprintf("%s#%d@%s", p.Repository, p.Number, short12(p.Head)))
|
||||
}
|
||||
ch.logf("%s: %s %s; the status says so (error), and they are judged once the trunk head is known", key, strings.Join(names, ", "), words)
|
||||
}
|
||||
}
|
||||
|
||||
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
|
||||
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
|
||||
// §5), on the recheck stream, pulled after.
|
||||
// the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after.
|
||||
const (
|
||||
KindNew = "new"
|
||||
KindNodes = "nodes"
|
||||
KindTrunk = "trunk"
|
||||
)
|
||||
|
||||
// prepare reads the case's verdict key and says what its check is (ADR 0548 §1, §5, §7): none (or one left running) is a
|
||||
// head never judged, a whole check; a failure of the nodes is asked again after the nodes changed; a verdict judged on trunk
|
||||
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's
|
||||
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing
|
||||
// to do.
|
||||
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
|
||||
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
|
||||
// kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
|
||||
// failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
|
||||
// move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
|
||||
func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
|
||||
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
||||
if err != nil || !found {
|
||||
return
|
||||
return KindNew
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(value)
|
||||
switch {
|
||||
case err != nil || v.State == delivery.VerdictRunning:
|
||||
return
|
||||
return KindNew
|
||||
case IsNodesFailure(v):
|
||||
cs.Kind = KindNodes
|
||||
return
|
||||
case IsConflictFailure(v):
|
||||
return // a head git refused: its next check is whole (ADR 0548 §7)
|
||||
case IsEnded(v), IsUntrunked(v):
|
||||
return // ended unjudged when its heads went (issue 1890), or never judged for want of a trunk head (issue 1422): checked whole
|
||||
return KindNodes
|
||||
}
|
||||
cs.Kind = KindTrunk
|
||||
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
|
||||
cs.CarriedFrom = judgedOn(v)
|
||||
if c := v.Composition; c != nil && c.State == delivery.CompositionPass && c.FPSet != "" && c.FPSet == cs.FPSet {
|
||||
kept := *c
|
||||
cs.Composition = &kept
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stale says whether a verdict was judged on commits that are no longer the case's: a trunk it merged moved, or a sibling it
|
||||
// read at its trunk's head (ADR 0548 §6).
|
||||
func (ch *Checker) stale(v delivery.Verdict, cs *Case) bool {
|
||||
return !sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, ch.besideNow())
|
||||
}
|
||||
|
||||
// besideNow are the commits a sibling is read at now, by repository: its trunk head, and for the node-engine the commit the
|
||||
// mesh runs when the indexer names one, as besideFor places them.
|
||||
func (ch *Checker) besideNow() map[string]string {
|
||||
return besideAt(ch.heads(), ch.Runs)
|
||||
}
|
||||
|
||||
// besideAt places the commits a sibling is read at: the trunk heads, with the node-engine at the commit the mesh runs when
|
||||
// runs names one (ADR 0548 §6). The checker's decision and the status read the same commits through it.
|
||||
func besideAt(now map[string]string, runs interface{ CommitOf(module string) string }) map[string]string {
|
||||
if runs != nil {
|
||||
if c := runs.CommitOf(nodeEngineModule); c != "" {
|
||||
out := make(map[string]string, len(now)+1)
|
||||
for r, h := range now {
|
||||
out[r] = h
|
||||
}
|
||||
out[nodeEngineRepo] = c
|
||||
return out
|
||||
}
|
||||
}
|
||||
return now
|
||||
}
|
||||
|
||||
// besideMoved says whether a sibling read at these commits moved: the commit it is read at now is known and another.
|
||||
func besideMoved(beside, now map[string]string) bool {
|
||||
for r, c := range beside {
|
||||
if h := now[r]; h != "" && h != c {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// judgedOn are the trunk and sibling commits a verdict was judged on, by repository.
|
||||
func judgedOn(v delivery.Verdict) map[string]string {
|
||||
out := make(map[string]string, len(v.Trunk)+len(v.Beside))
|
||||
for r, c := range v.Beside {
|
||||
out[r] = c
|
||||
}
|
||||
for r, c := range v.Trunk {
|
||||
out[r] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// carried is what a carry's verdict names as carried from (ADR 0548 §5): the commits it was judged on before, of the
|
||||
// repositories the new verdict still judges or reads; nil when none of them moved.
|
||||
func carried(from map[string]string, v delivery.Verdict) map[string]string {
|
||||
out, moved := map[string]string{}, false
|
||||
for r, c := range from {
|
||||
now, ok := v.Trunk[r]
|
||||
if !ok {
|
||||
now, ok = v.Beside[r]
|
||||
}
|
||||
if ok && now != "" && c != "" {
|
||||
out[r] = c
|
||||
moved = moved || now != c
|
||||
}
|
||||
}
|
||||
if !moved {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moveWords say a carry's moves, for the words of a failure: "carried from <old12> to <new12> of <repository>", each moved
|
||||
// repository, sorted.
|
||||
func moveWords(from map[string]string, v delivery.Verdict) string {
|
||||
var moves []string
|
||||
for r, c := range from {
|
||||
now, ok := v.Trunk[r]
|
||||
if !ok {
|
||||
now = v.Beside[r]
|
||||
}
|
||||
if now != "" && now != c {
|
||||
moves = append(moves, fmt.Sprintf("%s %s to %s", r, short12(c), short12(now)))
|
||||
}
|
||||
}
|
||||
sort.Strings(moves)
|
||||
return strings.Join(moves, ", ")
|
||||
return KindTrunk
|
||||
}
|
||||
|
||||
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
|
||||
@@ -586,9 +265,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return nil, fmt.Errorf("%s: %w", cs.Key, err)
|
||||
}
|
||||
if !found {
|
||||
if ch.heldByTurn(cs) {
|
||||
return nil, nil
|
||||
}
|
||||
rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return nil, nil // another checker made it first
|
||||
@@ -609,15 +285,15 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return &claimed{rev}, nil
|
||||
}
|
||||
return nil, nil
|
||||
case IsEnded(cur), ch.stale(cur, cs):
|
||||
retake = true // heads held again after their check was ended unjudged (issue 1890), or a trunk that moved
|
||||
case !sameTrunk(cur.Trunk, cs.Trunk):
|
||||
retake = true
|
||||
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
|
||||
ch.mu.Lock()
|
||||
at, ok := ch.nodesFail[cs.Key]
|
||||
ch.mu.Unlock()
|
||||
retake = !ok || epoch > at
|
||||
}
|
||||
if !retake || ch.heldByTurn(cs) {
|
||||
if !retake {
|
||||
return nil, nil
|
||||
}
|
||||
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
|
||||
@@ -630,59 +306,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return &claimed{rev}, nil
|
||||
}
|
||||
|
||||
// heldByTurn says whether a merge turn holds the case's check off (ADR 0548 §3): an approved merge waits on one of the
|
||||
// repositories it merges heads into, and it is not the merge of the case's own heads. The check is claimed when the turn's
|
||||
// key goes (Reconsider on the turns watch). Said once per turn in the log. A check already claimed is never held: one
|
||||
// claimed in the moment before the gate wrote its turn is judged on the commit it read and carried once after.
|
||||
func (ch *Checker) heldByTurn(cs *Case) bool {
|
||||
if ch.Turns == nil {
|
||||
return false
|
||||
}
|
||||
repos := make([]string, 0, len(cs.Trunk))
|
||||
for r := range cs.Trunk {
|
||||
repos = append(repos, r)
|
||||
}
|
||||
repo, w, held := ch.Turns.Holding(repos, cs.Heads)
|
||||
why := ""
|
||||
if held {
|
||||
why = repo + " " + w.Group + " " + w.Heads
|
||||
}
|
||||
ch.mu.Lock()
|
||||
said := ch.held[cs.Key]
|
||||
if why == "" {
|
||||
delete(ch.held, cs.Key)
|
||||
} else {
|
||||
ch.held[cs.Key] = why
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
if held && said != why {
|
||||
ch.logf("%s: not claimed: the approved merge of %s (heads %s) holds a turn on %s, so its check waits for that merge (ADR 0548 §3)",
|
||||
cs.Key, w.Group, w.Heads, repo)
|
||||
}
|
||||
return held
|
||||
}
|
||||
|
||||
// Held is every check a merge turn holds off now, by verdict key, with the turn's repository, group and heads.
|
||||
func (ch *Checker) Held() map[string]string {
|
||||
ch.init()
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
out := make(map[string]string, len(ch.held))
|
||||
for k, v := range ch.held {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ConflictWords are what the words of a head that git would not merge onto its trunk say before the files in conflict.
|
||||
const ConflictWords = "does not merge into the trunk: files in conflict: "
|
||||
|
||||
// IsConflictFailure says whether a verdict failed because its head did not merge onto its trunk (ADR 0548 §7): it has no
|
||||
// pass to carry, so its next check, at a push or at a later trunk move, is a whole check of its head.
|
||||
func IsConflictFailure(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, ConflictWords)
|
||||
}
|
||||
|
||||
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
|
||||
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
|
||||
// seats' streams (ADR 0525 §6, issue 1912).
|
||||
@@ -690,13 +313,9 @@ func IsNodesFailure(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, UnheardWords)
|
||||
}
|
||||
|
||||
// try claims the check, judges it within its bound under run and writes the verdict and the owed statuses. A check that ends
|
||||
// because ctx did is left running: the checker takes it again when it starts. A check whose run was ended because no
|
||||
// candidate holds its heads any more (issue 1890) is ended: its verdict says so, and no status is written.
|
||||
func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
if run.Err() != nil {
|
||||
return // ended before it was claimed: nothing to write
|
||||
}
|
||||
// try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends
|
||||
// because ctx did is left running: the checker takes it again when it starts.
|
||||
func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
||||
ch.mu.Lock()
|
||||
first := !ch.tried[cs.Key]
|
||||
ch.tried[cs.Key] = true
|
||||
@@ -712,47 +331,12 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
}
|
||||
began := ch.now()
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
out := ch.judge(run, cs)
|
||||
// An ask the build seat gave up on judged nothing (issue 734): the check is asked again under the same claim, its status
|
||||
// pending, and after LostAskTries it fails as not heard, judged again when the nodes change.
|
||||
for try := 1; IsLostAsk(out.Verdict) && run.Err() == nil; try++ {
|
||||
if try >= LostAskTries {
|
||||
out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) +
|
||||
" times and given up each time: " + UnheardWords)
|
||||
break
|
||||
}
|
||||
ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries)
|
||||
out = ch.judge(run, cs)
|
||||
}
|
||||
out := ch.judge(ctx, cs)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
var why superseded
|
||||
if errors.As(context.Cause(run), &why) {
|
||||
ch.end(ctx, cs, cl, why)
|
||||
return
|
||||
}
|
||||
v := out.Verdict
|
||||
v.Group, v.Heads, v.Trunk, v.Beside = cs.Group, cs.Heads, cs.Trunk, nil
|
||||
for r, c := range out.Beside {
|
||||
if _, merged := cs.Trunk[r]; !merged && c != "" {
|
||||
if v.Beside == nil {
|
||||
v.Beside = map[string]string{}
|
||||
}
|
||||
v.Beside[r] = c
|
||||
}
|
||||
}
|
||||
if cs.CarriedFrom != nil {
|
||||
// A carry (ADR 0548 §5): a pass names the commits it was carried from; a failure names the move it failed on.
|
||||
switch v.State {
|
||||
case delivery.VerdictPass:
|
||||
v.CarriedFrom = carried(cs.CarriedFrom, v)
|
||||
case delivery.VerdictFail:
|
||||
if moves := moveWords(cs.CarriedFrom, v); moves != "" {
|
||||
v.Failed.Words = strings.TrimSpace(v.Failed.Words + "\nfailed on the carry over the move of " + moves + ": the move changed what these tests read")
|
||||
}
|
||||
}
|
||||
}
|
||||
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
|
||||
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
|
||||
v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
|
||||
}
|
||||
@@ -774,37 +358,17 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
ch.nodesFail[cs.Key] = epoch
|
||||
ch.mu.Unlock()
|
||||
}
|
||||
if ch.movedSince(cs) || besideMoved(v.Beside, ch.besideNow()) {
|
||||
if ch.movedSince(cs) {
|
||||
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
|
||||
// check of the new trunk, which Reconsider starts, ends.
|
||||
ch.owePending(ctx, cs, v)
|
||||
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending until it is carried", cs.Key, v.State)
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
|
||||
return
|
||||
}
|
||||
ch.writeOwed(ctx, cs, v, out)
|
||||
ch.logf("%s: %s", cs.Key, v.State)
|
||||
}
|
||||
|
||||
// end writes the verdict of a check whose heads no candidate holds any more (issue 1890), by compare-and-set on its claim,
|
||||
// and says it once in the log with the asks it had put: they stay in the build seats' streams, since an ask once put cannot
|
||||
// be taken back (buildrun.Asker has no withdraw), and are no longer waited for. No status is written: the heads that
|
||||
// replaced these have their own check, whose pending status stands.
|
||||
func (ch *Checker) end(ctx context.Context, cs *Case, cl *claimed, why superseded) {
|
||||
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail,
|
||||
Failed: delivery.FailedFirst{Machine: "checker", Words: why.Error()}}
|
||||
data, err := delivery.EncodeVerdict(v)
|
||||
if err == nil {
|
||||
_, err = ch.Verdicts.Update(ctx, cs.Key, data, cl.rev)
|
||||
}
|
||||
if err != nil {
|
||||
ch.logf("%s: %s, but the verdict could not say so: %v", cs.Key, why.Error(), err)
|
||||
return
|
||||
}
|
||||
stage, _ := cs.Progress.Now()
|
||||
asks := cs.Asks()
|
||||
ch.logf("%s: %s; it was at %q with %d asks put %v, which are no longer waited for", cs.Key, why.Error(), stage, len(asks), asks)
|
||||
}
|
||||
|
||||
// judge runs the case within the bound. A judge that fails to judge, and a check that ran past its bound, are verdicts of
|
||||
// failure naming the machine and the step.
|
||||
func (ch *Checker) judge(ctx context.Context, cs *Case) Outcome {
|
||||
@@ -903,7 +467,7 @@ func (ch *Checker) Sweep(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
cur.State = delivery.VerdictFail
|
||||
cur.Failed = delivery.FailedFirst{Machine: "checker", Words: EndedWords + "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"}
|
||||
cur.Failed = delivery.FailedFirst{Machine: "checker", Words: "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"}
|
||||
data, err := delivery.EncodeVerdict(cur)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -938,37 +502,14 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
|
||||
if err != nil || !found {
|
||||
continue // never judged: no status to put out of date
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(value)
|
||||
if err == nil && !ch.stale(v, cs) {
|
||||
if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
|
||||
continue
|
||||
}
|
||||
if err == nil && v.State == delivery.VerdictPass {
|
||||
cs.CarriedFrom = judgedOn(v) // the status says it is being carried to the new commit (ADR 0548 §6)
|
||||
}
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
}
|
||||
ch.Reconsider(ctx)
|
||||
}
|
||||
|
||||
// owePending owes mesh/check pending on the heads of a verdict judged on commits that moved while it ran: a pass says it is
|
||||
// being carried to the trunk's head now (ADR 0548 §6), anything else that it is checked again.
|
||||
func (ch *Checker) owePending(ctx context.Context, judged *Case, v delivery.Verdict) {
|
||||
next := judged
|
||||
ckey, err := delivery.CandidatesKey(v.Group)
|
||||
ch.mu.Lock()
|
||||
c, ok := ch.known[ckey]
|
||||
ch.mu.Unlock()
|
||||
if err == nil && ok {
|
||||
if now := CaseOf(c, ch.heads()); now != nil && now.Key == judged.Key {
|
||||
next = now
|
||||
}
|
||||
}
|
||||
if v.State == delivery.VerdictPass {
|
||||
next.CarriedFrom = judgedOn(v)
|
||||
}
|
||||
ch.writeOwed(ctx, next, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
}
|
||||
|
||||
// movedSince says whether a trunk the case was judged on is no longer the trunk's head.
|
||||
func (ch *Checker) movedSince(cs *Case) bool {
|
||||
now := ch.heads()
|
||||
|
||||
@@ -3,7 +3,6 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -239,80 +238,14 @@ func TestAnUnheardForgeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *t
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1422: a pull request of a repository the gate's source.main names no trunk head of (one never merged through the
|
||||
// gate) is never dropped silently. It is not judged, its verdict says it cannot be judged and why, its status is error with
|
||||
// those words, checker_status names it, and the log says it once per head. Once its trunk head is known it is judged whole;
|
||||
// in a group, the members whose trunk is known are judged and the others' status says why they are not.
|
||||
func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testing.T) { // N6/AC6, ADR 0484 clause 11, 1422
|
||||
func TestARepositoryWithNoTrunkGetsNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clause 11
|
||||
ctx, j := context.Background(), passJudge()
|
||||
r := newRig(t, j)
|
||||
var logged []string
|
||||
r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
const unknown = "novox/unknown"
|
||||
cand := candidateOf(t, "feat/y", pullOf(unknown, 4, head1))
|
||||
key := keyOf(t, "feat/y", cand)
|
||||
r.ch.Candidate(ctx, "feat/y", cand)
|
||||
r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf("novox/unknown", 4, head1)))
|
||||
r.ch.Wait()
|
||||
r.ch.Reconsider(ctx) // looked at again: nothing is written or said twice
|
||||
r.ch.NodesChanged(ctx)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, key)
|
||||
if j.count() != 0 || !IsUntrunked(v) || !strings.Contains(v.Failed.Words, "the trunk head of "+unknown+" is not known") {
|
||||
t.Fatalf("judged %d times, verdict %+v, want none judged and a verdict that says it cannot be judged", j.count(), v)
|
||||
if j.count() != 0 || len(r.verdicts.vals) != 0 || len(r.owed.vals) != 0 {
|
||||
t.Errorf("judged %d times, %d verdicts, %d statuses, want none", j.count(), len(r.verdicts.vals), len(r.owed.vals))
|
||||
}
|
||||
if s := owedState(t, r, unknown, 4, head1); s != "error" {
|
||||
t.Errorf("%s#4 owes %q, want error", unknown, s)
|
||||
}
|
||||
if n := countWith(logged, "cannot be judged"); n != 1 {
|
||||
t.Errorf("said %d times in the log, want once for the head: %q", n, logged)
|
||||
}
|
||||
reader := func() Reader {
|
||||
ckey, _ := delivery.CandidatesKey("feat/y")
|
||||
return Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
|
||||
Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}
|
||||
}
|
||||
st, err := reader().Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := st.CannotBeJudged["feat/y"]; !strings.Contains(why, unknown) || len(st.Waiting[KindNew]) != 0 || len(st.Unjudged) != 0 {
|
||||
t.Errorf("cannot_be_judged %v, waiting %v, unjudged %v: want feat/y named as cannot be judged, not as new", st.CannotBeJudged, st.Waiting, st.Unjudged)
|
||||
}
|
||||
if rows, _, err := reader().Verdicts("feat/y"); err != nil || len(rows) != 1 || !strings.Contains(rows[0].Reports[unknown+"#4"], "cannot be judged") {
|
||||
t.Errorf("checker_verdicts %+v (%v), want the report of %s#4 to say it cannot be judged", rows, err, unknown)
|
||||
}
|
||||
|
||||
// Its trunk head becomes known: the heads are judged whole, the same key rewritten, the status success.
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, unknown: trunkA2})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, key); j.count() != 1 || v.State != delivery.VerdictPass || v.Trunk[unknown] != trunkA2 {
|
||||
t.Errorf("judged %d times, verdict %+v, want one pass on %s", j.count(), v, trunkA2)
|
||||
}
|
||||
if s := owedState(t, r, unknown, 4, head1); s != "success" {
|
||||
t.Errorf("%s#4 owes %q once its trunk is known, want success", unknown, s)
|
||||
}
|
||||
|
||||
// A group with one member of a repository whose trunk is not known: the others are judged, its status says why it is not.
|
||||
group := candidateOf(t, "feat/z", pullOf(repoA, 9, head2), pullOf("novox/other", 2, head2))
|
||||
r.ch.Candidate(ctx, "feat/z", group)
|
||||
r.ch.Wait()
|
||||
if s := owedState(t, r, repoA, 9, head2); s != "success" {
|
||||
t.Errorf("%s#9 owes %q, want success", repoA, s)
|
||||
}
|
||||
if s := owedState(t, r, "novox/other", 2, head2); s != "error" {
|
||||
t.Errorf("novox/other#2 owes %q, want error", s)
|
||||
}
|
||||
}
|
||||
|
||||
func countWith(lines []string, words string) int {
|
||||
n := 0
|
||||
for _, l := range lines {
|
||||
if strings.Contains(l, words) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// 1887/A10, ADR 0527 §2: hq is judged as any repository, its checks Go tests, and its pull request gets a verdict and a
|
||||
@@ -400,10 +333,7 @@ func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
// No check of heads no candidate holds is left running (issue 1890): at start the sweep ends a key left running; while the
|
||||
// checker runs, a check whose group moved to other heads is ended at once, unjudged, and writes no status; heads held again
|
||||
// are checked whole; and a group's check starts only once its heads settled, so a partial set of members is never checked.
|
||||
func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) {
|
||||
func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T) {
|
||||
ctx, j := context.Background(), passJudge()
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
@@ -421,91 +351,12 @@ func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) {
|
||||
if err := r.ch.Sweep(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := r.verdict(t, dead); !IsEnded(v) || !strings.Contains(v.Failed.Words, "closed at start") {
|
||||
if v := r.verdict(t, dead); v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "closed at start") {
|
||||
t.Errorf("the dead key = %+v", v)
|
||||
}
|
||||
if v := r.verdict(t, live); v.State != delivery.VerdictRunning {
|
||||
t.Errorf("the live key = %+v, want it left for the check to take over", v)
|
||||
}
|
||||
|
||||
// A member joins while the check of the lone member's heads runs: that check is ended, its judge's run cancelled.
|
||||
started, ended := make(chan struct{}), make(chan error, 1)
|
||||
j = &judgeFunc{f: func(run context.Context, _ *Case, call int) (Outcome, error) {
|
||||
if call == 1 {
|
||||
close(started)
|
||||
<-run.Done()
|
||||
ended <- context.Cause(run)
|
||||
return Outcome{}, run.Err()
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
var logged []string
|
||||
var logMu sync.Mutex
|
||||
r.ch.Logf = func(f string, a ...any) { logMu.Lock(); logged = append(logged, fmt.Sprintf(f, a...)); logMu.Unlock() }
|
||||
alone, both := cand, candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 3, head2))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", alone)
|
||||
<-started
|
||||
r.ch.Candidate(ctx, "feat%2Fx", both)
|
||||
var bothC delivery.Candidate
|
||||
_ = json.Unmarshal(both, &bothC)
|
||||
select {
|
||||
case err := <-ended:
|
||||
if err == nil || !strings.Contains(err.Error(), "superseded by "+GateHeads(bothC)) {
|
||||
t.Errorf("the old check's run ended with %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the check of the heads the candidate left still runs")
|
||||
}
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, live); !IsEnded(v) || v.Failed.Words != EndedWords+"superseded by "+GateHeads(bothC) {
|
||||
t.Errorf("the superseded verdict = %+v", v.Failed)
|
||||
}
|
||||
if v := r.verdict(t, keyOf(t, "feat/x", both)); v.State != delivery.VerdictPass {
|
||||
t.Errorf("the group's verdict = %s", v.State)
|
||||
}
|
||||
okey, _ := delivery.CheckerOwedKey(repoA, 7, head1)
|
||||
if raw, _, _, _ := r.owed.Get(ctx, okey); !strings.Contains(string(raw), `"success"`) {
|
||||
t.Errorf("the head's status = %s, want the group's success, never the ended check's", raw)
|
||||
}
|
||||
logMu.Lock()
|
||||
said := 0
|
||||
for _, l := range logged {
|
||||
if strings.Contains(l, "ended: superseded") {
|
||||
said++
|
||||
}
|
||||
}
|
||||
logMu.Unlock()
|
||||
if said != 1 {
|
||||
t.Errorf("the end was said %d times: %v", said, logged)
|
||||
}
|
||||
// The member leaves again: the ended heads are held again and checked whole.
|
||||
r.ch.Candidate(ctx, "feat%2Fx", alone)
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, live); v.State != delivery.VerdictPass || j.count() != 3 {
|
||||
t.Errorf("the heads held again = %s after %d judgings, want a pass of a third", v.State, j.count())
|
||||
}
|
||||
|
||||
// Members announced one after another: only the settled, whole set is checked.
|
||||
j = passJudge()
|
||||
r = newRig(t, j)
|
||||
r.ch.Settle = time.Hour
|
||||
sdk := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2))
|
||||
whole := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2), pullOf(repoA, 8, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fy", sdk)
|
||||
r.ch.Candidate(ctx, "feat%2Fy", whole)
|
||||
r.ch.Reconsider(ctx) // a trunk move or a turn while the heads settle starts nothing
|
||||
if j.count() != 0 {
|
||||
t.Fatalf("judged %d times before the heads settled", j.count())
|
||||
}
|
||||
r.ch.settleNow()
|
||||
r.ch.Wait()
|
||||
if _, _, found, _ := r.verdicts.Get(ctx, keyOf(t, "feat/y", sdk)); found || j.count() != 1 {
|
||||
t.Errorf("the partial set has a verdict (%v) or the judge ran %d times, want the whole set alone", found, j.count())
|
||||
}
|
||||
if v := r.verdict(t, keyOf(t, "feat/y", whole)); v.State != delivery.VerdictPass {
|
||||
t.Errorf("the whole set = %s", v.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683 (the checker's half of AC5)
|
||||
|
||||
@@ -28,7 +28,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
||||
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
|
||||
b.ex.Compositions = f
|
||||
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||
cs.FPSet = "fps-of-the-candidate"
|
||||
v := b.judge(t, cs).Verdict
|
||||
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
|
||||
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
|
||||
@@ -39,15 +38,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
||||
if !compositionMentions(v, "passed") {
|
||||
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
|
||||
}
|
||||
if v.Composition.FPSet == "" || v.Composition.FPSet != cs.FPSet {
|
||||
t.Errorf("the composition names the fingerprint set %q, want the candidate's %q (ADR 0548 §5)", v.Composition.FPSet, cs.FPSet)
|
||||
}
|
||||
// ADR 0548 §5: a carry whose fingerprints did not change keeps the composition it passed and asks the assigner nothing.
|
||||
carry := caseOf(t, pullOf(repoA, 7, head1))
|
||||
carry.CarriedFrom, carry.Composition = map[string]string{repoA: trunkA2}, v.Composition
|
||||
if kept := b.judge(t, carry).Verdict; kept.Composition == nil || kept.Composition.SDK != "v0.1.14-x" || len(f.asked) != 1 {
|
||||
t.Errorf("the carry's composition is %+v and the assigner was asked %d times; want the kept one and no second ask", kept.Composition, len(f.asked))
|
||||
}
|
||||
}
|
||||
|
||||
func compositionMentions(v delivery.Verdict, word string) bool {
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -117,9 +116,6 @@ type bench struct {
|
||||
fetch Fetched
|
||||
digest map[string]Digested // by level
|
||||
test func(c stepCall) (any, error)
|
||||
// declines is how many asks naming needs a node lacking them declines before shanks, which has every need, takes
|
||||
// one (issue 890); an ask naming none runs on shanks, which then has none.
|
||||
declines int
|
||||
}
|
||||
|
||||
// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
|
||||
@@ -155,17 +151,8 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step {
|
||||
if module == buildrun.GoBuildModule {
|
||||
steps.Digest = tool("digest")
|
||||
}
|
||||
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||
if len(a.Needs) > 0 {
|
||||
if b.declines > 0 {
|
||||
b.declines--
|
||||
result, _ := json.Marshal(delivery.StepDeclined{Declined: true, Needs: a.Needs, Lacks: a.Needs})
|
||||
return delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: "ace", State: delivery.StepRefused, Started: began, Ended: began,
|
||||
Reason: "declined", Result: result}
|
||||
}
|
||||
steps.Has = func(string) bool { return true }
|
||||
}
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
|
||||
if err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
@@ -273,36 +260,8 @@ func TestACleanMergeIsFetchedAtItsTrunkAndBothLevelsPass(t *testing.T) { // 1679
|
||||
t.Errorf("%s was not kept", k)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0567 §12: every check ask carries every node's name for the names check; a check whose names cannot be read, or
|
||||
// that reads none, is not asked and fails saying so.
|
||||
for _, c := range []struct {
|
||||
names []string
|
||||
err error
|
||||
fails string
|
||||
}{{[]string{"anchor", "laptop"}, nil, ""}, {nil, nil, "no node record"}, {nil, errors.New("no bucket"), "no bucket"}} {
|
||||
b := newBench(t)
|
||||
b.ex.Nodes = namesOf{c.names, c.err}
|
||||
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
asks := b.asks.Asks()
|
||||
if c.fails != "" {
|
||||
if out.Verdict.State != delivery.VerdictFail || len(asks) != 0 || !strings.Contains(out.Verdict.Failed.Words, c.fails) {
|
||||
t.Errorf("names %v, %v: verdict %+v, %d asks", c.names, c.err, out.Verdict, len(asks))
|
||||
}
|
||||
} else if len(asks) == 0 || strings.Join(asks[0].Check.NodeNames, ",") != "anchor,laptop" {
|
||||
t.Errorf("the check ask carries the names %v", asks)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// namesOf is the nodes' names as a test gives them.
|
||||
type namesOf struct {
|
||||
names []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (n namesOf) Names(context.Context) ([]string, error) { return n.names, n.err }
|
||||
|
||||
func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
||||
b := newBench(t)
|
||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/slow": digA, "fixture/other": digC}}
|
||||
@@ -317,49 +276,6 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
||||
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
|
||||
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
|
||||
}
|
||||
|
||||
// Issue 890/AC5: a package needing node-build-node is left unmet where the node lacks it; the check keeps the passes
|
||||
// and asks the integration level again for a node with it, puts it again when a node declines it, and that node tests
|
||||
// the package alone, its kept passes not counted twice.
|
||||
// fixture/outside has no kept verdict after its pass (it reads outside itself: its digest moves every call), so a
|
||||
// placed ask that ran more than its unmet packages would run and count it again.
|
||||
b = newBench(t)
|
||||
b.declines = 1
|
||||
outside := 0
|
||||
b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
|
||||
d := echoBeside(b, b.digest[c.args["level"].(string)])
|
||||
if d.Level == "integration" {
|
||||
outside++
|
||||
d.Packages = map[string]string{"fixture/slow": digB, "fixture/node": digC, "fixture/outside": fmt.Sprintf("%064x", outside)}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
b.digest[delivery.LevelIntegration] = Digested{Level: "integration",
|
||||
Needs: []delivery.PackageNeeds{{Package: "fixture/node", Needs: []string{delivery.NeedNodeBuildNode}}}}
|
||||
out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
asks := b.asks.Asks()
|
||||
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
|
||||
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
|
||||
}
|
||||
for _, a := range asks[1:] {
|
||||
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
|
||||
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
|
||||
}
|
||||
}
|
||||
tests = b.steps.tool("test")
|
||||
if last := tests[len(tests)-1]; len(tests) != 3 || strings.Join(packagesOf(last), ",") != "fixture/node" || last.args["level"] != "integration" {
|
||||
t.Errorf("test calls = %+v, want the unit level, the integration level of fixture/slow and fixture/outside, then fixture/node alone", tests)
|
||||
}
|
||||
slow := captured[Tested](t, "go-test-slow")
|
||||
if i := out.Verdict.Levels[delivery.LevelIntegration]; i.Run != 2*slow.Tests || i.Kept != 0 {
|
||||
t.Errorf("integration = %+v, want %d run and nothing counted kept", i, 2*slow.Tests)
|
||||
}
|
||||
// And where no node has the need, the check fails once the declines outlast the bound, naming the need.
|
||||
b.kept, b.declines = newMemKV(), 1000
|
||||
b.ex.Kept, b.ex.DeclineBound = Kept{Store: b.kept}, 200*time.Millisecond
|
||||
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict; v.State != delivery.VerdictFail || !strings.Contains(fmt.Sprint(v), "needs a node with node-build-node") {
|
||||
t.Errorf("with no node having the need: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
|
||||
@@ -704,8 +620,9 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversT
|
||||
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
|
||||
func echoBeside(b *bench, d Digested) Digested {
|
||||
var lines []string
|
||||
if fetches := b.steps.tool("fetch"); len(fetches) > 0 { // the workspace digested is the last fetch's (issue 890: a placed ask fetches again)
|
||||
for _, r := range refsOf(fetches[len(fetches)-1].args["beside"]) {
|
||||
for _, f := range b.steps.tool("fetch") {
|
||||
refs, _ := refsOf(f.args["beside"]), true
|
||||
for _, r := range refs {
|
||||
lines = append(lines, r.Repository+" "+r.Commit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func TestTheStatusLineNamesTheFailureNotTheFirstLoggedLine(t *testing.T) { // 19
|
||||
f := fatalAfterLog()
|
||||
f.Cause = ""
|
||||
v = levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||
if !strings.HasPrefix(afterHid(v.Failed.Words), "agreement_catalogue_test.go:130") {
|
||||
if !strings.HasPrefix(v.Failed.Words, "agreement_catalogue_test.go:130") {
|
||||
t.Errorf("words = %q", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
@@ -56,45 +56,18 @@ func TestWordsLongerThanTheBoundKeepTheirLastLines(t *testing.T) { // 1901/AC1
|
||||
_ = delivery.VerdictFail
|
||||
}
|
||||
|
||||
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before
|
||||
// (1901/AC3). A failure that hides its cause (none, or a passing line) is said loudly first in the words and in the status
|
||||
// line, naming the build module and the version it says it is, and the level still fails; a failure that keeps the
|
||||
// contract is worded as before (1901/AC8).
|
||||
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3, AC8
|
||||
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before.
|
||||
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3
|
||||
old := `{"package":"p","test":"TestX","output":"x_test.go:1: logged\nx_test.go:2: the old kept lines"}`
|
||||
var f Failure
|
||||
if err := json.Unmarshal([]byte(old), &f); err != nil || f.Cause != "" {
|
||||
t.Fatalf("1901/AC3: err = %v, cause = %q", err, f.Cause)
|
||||
}
|
||||
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||
if afterHid(v.Failed.Words) != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
|
||||
if v.Failed.Words != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
|
||||
t.Errorf("1901/AC3: words = %q", v.Failed.Words)
|
||||
}
|
||||
if got := firstFailed(*v); !strings.Contains(got, "the build module hid the failure") || strings.Contains(got, "\n") {
|
||||
t.Errorf("1901/AC3, AC8: status = %q", got)
|
||||
}
|
||||
|
||||
by := "mesh-build-node with mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78"
|
||||
for _, c := range []struct {
|
||||
name, module, by, cause, output, loud string
|
||||
}{
|
||||
{"no cause", "mesh-build-node", by, "", "test/b.test.ts: something", "the build module hid the failure (mesh-build-node at mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78): it sent no cause"},
|
||||
{"a passing line as its cause", "mesh-build-node", by, "\u2714 the same merge has the same id (8.006913ms)", "", "the build module hid the failure (mesh-build-node at mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78): its cause is a passing line"},
|
||||
{"from before the contract", "mesh-build-node", "", "", "\u2714 the same merge has the same id (8.006913ms)", "the build module hid the failure (mesh-build-node at a version before the failure contract): it sent no cause"},
|
||||
{"a real cause", "mesh-build-node", by, "one plus one is not three", "not ok 3 - fails", ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
v := levelFailure(&target{module: c.module}, "unit", Tested{By: c.by, Failed: []Failure{{Package: "modules/gitea", Test: "fails", Cause: c.cause, Output: c.output}}})
|
||||
if v.State != delivery.VerdictFail {
|
||||
t.Errorf("1901/AC8: the verdict is %q, want it failed", v.State)
|
||||
}
|
||||
line := firstFailed(*v)
|
||||
switch {
|
||||
case c.loud == "" && strings.Contains(v.Failed.Words, "hid the failure"):
|
||||
t.Errorf("1901/AC8: a failure that keeps the contract is said hidden: %q", v.Failed.Words)
|
||||
case c.loud != "" && (!strings.HasPrefix(v.Failed.Words, c.loud) || !strings.Contains(line, c.loud) || !strings.Contains(v.Failures[0].Words, "hid the failure")):
|
||||
t.Errorf("1901/AC8: words %q, status %q, want both to start with %q", v.Failed.Words, line, c.loud)
|
||||
}
|
||||
})
|
||||
if got := firstFailed(*v); !strings.Contains(got, "logged") || strings.Contains(got, "\n") {
|
||||
t.Errorf("1901/AC3: status = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -87,3 +88,29 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
|
||||
t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
|
||||
ctx := context.Background()
|
||||
var r *rig
|
||||
release, judged := make(chan struct{}), make(chan struct{})
|
||||
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||
if call == 1 {
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
||||
defer close(judged)
|
||||
} else {
|
||||
<-release // the check asked again for the new trunk waits in the stream meanwhile
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
|
||||
<-judged
|
||||
for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
|
||||
}
|
||||
close(release)
|
||||
r.ch.Wait()
|
||||
}
|
||||
|
||||
@@ -80,9 +80,6 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
|
||||
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
|
||||
type Examiner struct {
|
||||
Asks Asks
|
||||
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
|
||||
// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
|
||||
DeclineBound time.Duration
|
||||
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
|
||||
Toolchains ToolchainEntries
|
||||
Kept Kept
|
||||
@@ -110,14 +107,8 @@ type Examiner struct {
|
||||
// waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil
|
||||
// leaves the composition not run.
|
||||
Compositions CompositionReader
|
||||
// Nodes names every node with a record, read from each `<node>.node` of the node-engine's have (novox/hq ADR 0567
|
||||
// §12): every check ask carries them (CheckJob.NodeNames) for the names check, so no repository lists them. A check
|
||||
// whose names cannot be read, or that reads none, is not asked and fails saying so; nil hands no names.
|
||||
Nodes interface {
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
}
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote
|
||||
@@ -212,15 +203,12 @@ type state struct {
|
||||
nodes map[string]bool
|
||||
clone time.Duration
|
||||
slow []string
|
||||
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
|
||||
beside map[string]string
|
||||
}
|
||||
|
||||
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
||||
// be judged at all.
|
||||
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
||||
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
|
||||
beside: map[string]string{}}
|
||||
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
||||
if v := st.judgeAll(ctx); v != nil {
|
||||
return st.finish(ctx, *v), nil
|
||||
}
|
||||
@@ -242,13 +230,6 @@ func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
|
||||
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
|
||||
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
|
||||
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
if st.c.Composition != nil && st.c.Try == nil {
|
||||
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
|
||||
kept := *st.c.Composition
|
||||
v.Composition = &kept
|
||||
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
|
||||
return
|
||||
}
|
||||
if st.j.Compositions == nil || st.c.Try != nil {
|
||||
return
|
||||
}
|
||||
@@ -262,9 +243,6 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
return
|
||||
}
|
||||
withComposition(v, comp)
|
||||
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
||||
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
|
||||
}
|
||||
}
|
||||
|
||||
// outcome fills the verdict's levels, times, machines and composition from what ran.
|
||||
@@ -304,9 +282,6 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
|
||||
}
|
||||
}
|
||||
out := Outcome{Verdict: v, Slowest: slow}
|
||||
if len(st.beside) > 0 {
|
||||
out.Beside = st.beside
|
||||
}
|
||||
for _, t := range st.targets {
|
||||
if t.judged != "" {
|
||||
if out.Judged == nil {
|
||||
@@ -368,7 +343,6 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
if len(beside) > 0 {
|
||||
args["beside"] = beside
|
||||
t.beside = besideLines(beside)
|
||||
besideAtTrunk(st.c, beside, st.beside)
|
||||
}
|
||||
if p.Repository == catalogueRepository {
|
||||
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
||||
@@ -390,35 +364,11 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
|
||||
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
|
||||
const DefaultDeclineBound = 30 * time.Minute
|
||||
|
||||
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
|
||||
// pins of its go.mod files, then each level's digest, kept verdicts and tests. When the answer leaves packages unmet,
|
||||
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
|
||||
// asked again of a node that has it (placed).
|
||||
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
|
||||
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
||||
return st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
|
||||
}
|
||||
|
||||
// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
|
||||
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
|
||||
// is run or counted again (ADR 0552 §4).
|
||||
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
|
||||
p := t.pull
|
||||
placed := len(needs) > 0
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
|
||||
if st.j.Nodes != nil {
|
||||
names, err := st.j.Nodes.Names(ctx)
|
||||
if err == nil && len(names) == 0 {
|
||||
err = fmt.Errorf("no node record is on the bus")
|
||||
}
|
||||
if err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d is not asked: the nodes' names the names check needs cannot be read: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
job.NodeNames = names
|
||||
}
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
|
||||
if t.runner != "" {
|
||||
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
||||
}
|
||||
@@ -430,53 +380,17 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
||||
}
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
||||
At: st.j.now().UTC(), Check: job, Needs: needs}
|
||||
var step delivery.Step
|
||||
var firstDecline time.Time
|
||||
for {
|
||||
what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number)
|
||||
if placed {
|
||||
what += " for a node with " + strings.Join(needs, ", ")
|
||||
At: st.j.now().UTC(), Check: job}
|
||||
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
step, err := st.j.Asks.Wait(ctx, a)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fail(t.module, ctx.Err().Error())
|
||||
}
|
||||
st.c.Progress.Set(what, "")
|
||||
st.c.Asked(a.ID)
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
waiting := ctx
|
||||
if watch := st.c.Watch; watch != nil {
|
||||
id := a.ID
|
||||
watch(id, nil) // put: a try names it at once, not only when it ends (issue 1890)
|
||||
waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) })
|
||||
}
|
||||
var err error
|
||||
step, err = st.j.Asks.Wait(waiting, a)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fail(t.module, ctx.Err().Error())
|
||||
}
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
d, declined := delivery.DeclinedOf(step)
|
||||
if !declined {
|
||||
break
|
||||
}
|
||||
// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
|
||||
// node that declined looks at the stream again only after its next pull's wait.
|
||||
now := st.j.now()
|
||||
if firstDecline.IsZero() {
|
||||
firstDecline = now
|
||||
}
|
||||
bound := st.j.DeclineBound
|
||||
if bound <= 0 {
|
||||
bound = DefaultDeclineBound
|
||||
}
|
||||
if now.Sub(firstDecline) > bound {
|
||||
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
|
||||
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
|
||||
}
|
||||
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
|
||||
a.ID, a.At = buildstep.NewID(), now.UTC()
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
|
||||
took := time.Duration(0)
|
||||
@@ -500,31 +414,18 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
if st.c.Try == nil {
|
||||
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
||||
}
|
||||
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
|
||||
return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
|
||||
}
|
||||
t.changed = t.fetched.Changed
|
||||
if !changesBuildGo(p.Repository, t.changed) {
|
||||
t.runner = "" // the build module used no runner: the merge does not change it
|
||||
}
|
||||
if !placed { // the placed ask fetched the same merge: its preview and pins were read from the first
|
||||
*merged = append(*merged, t.fetched)
|
||||
if v := st.checkPinsOf(ctx, t); v != nil {
|
||||
return v
|
||||
}
|
||||
if v := st.checkBinariesOf(t); v != nil {
|
||||
return v
|
||||
}
|
||||
*merged = append(*merged, t.fetched)
|
||||
if v := st.checkPinsOf(ctx, t); v != nil {
|
||||
return v
|
||||
}
|
||||
ran := 0.0
|
||||
for _, la := range answer.Levels {
|
||||
if placed {
|
||||
if len(la.Unmet) > 0 {
|
||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||
return fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
|
||||
la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
|
||||
}
|
||||
la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
|
||||
}
|
||||
v, elapsed := st.level(ctx, t, la)
|
||||
ran += elapsed
|
||||
if v != nil {
|
||||
@@ -533,48 +434,9 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
}
|
||||
}
|
||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
|
||||
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
|
||||
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
|
||||
var again, packages []string
|
||||
seen := map[string]bool{}
|
||||
for _, la := range answer.Levels {
|
||||
if len(again) > 0 || len(la.Unmet) > 0 {
|
||||
again = append(again, la.Level)
|
||||
}
|
||||
for _, u := range la.Unmet {
|
||||
if !seen[u.Package] {
|
||||
seen[u.Package] = true
|
||||
packages = append(packages, u.Package)
|
||||
}
|
||||
}
|
||||
}
|
||||
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
|
||||
return st.askNeeding(ctx, t, merged, again, unmet, packages)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unmetNames names the unmet packages and their needs.
|
||||
func unmetNames(unmet []delivery.PackageNeeds) string {
|
||||
var out []string
|
||||
for _, u := range unmet {
|
||||
out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
// unmetOf names the unmet packages of every level.
|
||||
func unmetOf(levels []delivery.LevelAnswer) string {
|
||||
var out []string
|
||||
for _, la := range levels {
|
||||
if len(la.Unmet) > 0 {
|
||||
out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
|
||||
}
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
||||
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
|
||||
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
|
||||
@@ -674,129 +536,23 @@ const MaxWords = 4096
|
||||
|
||||
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
|
||||
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
|
||||
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
|
||||
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
|
||||
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
|
||||
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
|
||||
// status line, which is cut short, names it.
|
||||
func failureWords(f Failure) string {
|
||||
return failureWordsAfter(f, "")
|
||||
}
|
||||
|
||||
// failureWordsAfter is failureWords with the line loud put first, within the same MaxWords: what the checker itself says of
|
||||
// the failure (that the build module hid it), which the status line then names.
|
||||
func failureWordsAfter(f Failure, loud string) string {
|
||||
out, passed := dropSummary(strings.TrimSpace(f.Output))
|
||||
cause := strings.TrimSpace(f.Cause)
|
||||
if cause == "" {
|
||||
cause = causeIn(out)
|
||||
}
|
||||
switch {
|
||||
case cause == "" && out == "" && passed:
|
||||
cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
|
||||
case cause == "" && out == "":
|
||||
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
|
||||
cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
|
||||
}
|
||||
if len(cause) > MaxCause {
|
||||
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
|
||||
}
|
||||
// The words keep MaxWords exactly: the loud line, the cause and their newlines, the marker and the kept lines.
|
||||
budget := MaxWords
|
||||
if cause != "" {
|
||||
budget -= len(cause) + 1
|
||||
}
|
||||
if loud != "" {
|
||||
budget -= len(loud) + 1
|
||||
}
|
||||
if len(out) > budget {
|
||||
const marker = "... "
|
||||
tail := out[len(out)-(budget-len(marker)):]
|
||||
out := strings.TrimSpace(f.Output)
|
||||
if len(out) > MaxWords {
|
||||
out = out[len(out)-MaxWords:]
|
||||
// Start on a whole line, and a whole character.
|
||||
if i := strings.IndexByte(tail, '\n'); i >= 0 {
|
||||
tail = strings.TrimLeft(tail[i:], "\n")
|
||||
if i := strings.IndexByte(out, '\n'); i >= 0 {
|
||||
out = "... " + strings.TrimLeft(out[i:], "\n")
|
||||
}
|
||||
out = marker + strings.ToValidUTF8(tail, "")
|
||||
out = strings.ToValidUTF8(out, "")
|
||||
}
|
||||
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
|
||||
out = strings.TrimRight(cause+"\n"+out, "\n")
|
||||
}
|
||||
if loud != "" {
|
||||
out = strings.TrimRight(loud+"\n"+out, "\n")
|
||||
if c := strings.TrimSpace(f.Cause); c != "" && !strings.HasPrefix(firstLine(out), c) {
|
||||
out = c + "\n" + out
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hidWords is the checker's loud line for a failure the build module sent hiding its cause (issue 1901, its thorough
|
||||
// fix): with no cause, or a passing line as its cause, under the failure contract (buildstep.Hidden). It names the build
|
||||
// module and the version it says it is (Tested.By, "<module> with mesh-sdk <version>"); one from before the contract
|
||||
// says nothing of itself. "" when the failure keeps the contract. The check fails either way: only its words change.
|
||||
func hidWords(module, by string, f Failure) string {
|
||||
why := buildstep.Hidden(f)
|
||||
if why == "" {
|
||||
return ""
|
||||
}
|
||||
version := "a version before the failure contract"
|
||||
if module == "" {
|
||||
module = "an unnamed build module"
|
||||
}
|
||||
if m, v, ok := strings.Cut(by, " with "); ok && m != "" {
|
||||
module, version = m, v
|
||||
}
|
||||
return fmt.Sprintf("the build module hid the failure (%s at %s): %s", module, version, why)
|
||||
}
|
||||
|
||||
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
|
||||
// push the words past MaxWords).
|
||||
const MaxCause = 1024
|
||||
|
||||
// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
|
||||
// whether a "PASS" was among them: the package's tests passed before it failed.
|
||||
func dropSummary(out string) (string, bool) {
|
||||
passed := false
|
||||
lines := strings.Split(out, "\n")
|
||||
kept := lines[:0]
|
||||
for _, l := range lines {
|
||||
// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
|
||||
switch t := strings.TrimRight(l, "\r"); {
|
||||
case t == "PASS":
|
||||
passed = true
|
||||
case t == "FAIL", strings.HasPrefix(t, "ok \t"), strings.HasPrefix(t, "FAIL\t"):
|
||||
default:
|
||||
kept = append(kept, l)
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(kept, "\n")), passed
|
||||
}
|
||||
|
||||
// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
|
||||
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
|
||||
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
|
||||
// line holding the same words is not the cause (review round 1).
|
||||
func causeIn(out string) string {
|
||||
lines := strings.Split(out, "\n")
|
||||
for i, l := range lines {
|
||||
if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
|
||||
continue
|
||||
}
|
||||
words := "WARNING: DATA RACE"
|
||||
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
|
||||
words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
|
||||
if i+2 < len(lines) && strings.HasPrefix(lines[i+2], " ") {
|
||||
words += " in " + strings.TrimSpace(lines[i+2])
|
||||
}
|
||||
}
|
||||
return words
|
||||
}
|
||||
for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
|
||||
for _, l := range lines {
|
||||
if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
|
||||
return l
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// firstLine is the first non-empty line of a failure's words.
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
@@ -831,7 +587,7 @@ func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict
|
||||
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
||||
name = res.Module
|
||||
}
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWordsAfter(f, hidWords(t.module, res.By, f))})
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
|
||||
}
|
||||
}
|
||||
if len(all) == 0 { // no result: a level said failed with nothing to name
|
||||
|
||||
@@ -56,22 +56,12 @@ func (k Kept) Keep(ctx context.Context, level, digest string, v delivery.Kept) (
|
||||
|
||||
// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module
|
||||
// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause
|
||||
// 1): it runs every check. A package with skipped tests is not kept either: a pass that skipped (a test needing a
|
||||
// toolchain the node lacks) would be reused for its digest and never run anywhere (issue 890). A build module that
|
||||
// counts skips only per result, not per package, keeps nothing of a result in which a test skipped. It answers the
|
||||
// packages it kept now.
|
||||
// 1): it runs every check. It answers the packages it kept now.
|
||||
func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) {
|
||||
var kept []string
|
||||
perPackage := false
|
||||
for _, p := range res.Packages {
|
||||
perPackage = perPackage || p.Skipped > 0
|
||||
}
|
||||
if res.Skipped > 0 && !perPackage {
|
||||
return kept, nil // an older build module: the skip cannot be placed in a package
|
||||
}
|
||||
for _, p := range res.Packages {
|
||||
digest, ok := digests[p.Package]
|
||||
if !ok || !p.Passed || p.NoTests || p.Tests == 0 || p.Skipped > 0 {
|
||||
if !ok || !p.Passed || p.NoTests || p.Tests == 0 {
|
||||
continue
|
||||
}
|
||||
if failedIn(res, p.Package) {
|
||||
|
||||
@@ -74,32 +74,6 @@ func TestOnlyPassingPackagesWithADigestAreKept(t *testing.T) { // 1287/AC3, AC4
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResultWithSkippedTestsKeepsNothing(t *testing.T) { // issue 890: a pass that skipped proved nothing
|
||||
ctx, st := context.Background(), newMemKV()
|
||||
k := Kept{Store: st}
|
||||
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{{Package: "m/a", Passed: true, Tests: 4}}}
|
||||
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA}, at0)
|
||||
if err != nil || len(kept) != 0 {
|
||||
t.Fatalf("kept %v, %v, want nothing", kept, err)
|
||||
}
|
||||
if keys, _ := st.Keys(ctx); len(keys) != 0 {
|
||||
t.Errorf("the bucket holds %v, want none", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheSkippingPackageOfAResultIsWithheld(t *testing.T) { // issue 890
|
||||
ctx, st := context.Background(), newMemKV()
|
||||
k := Kept{Store: st}
|
||||
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{
|
||||
{Package: "m/a", Passed: true, Tests: 4, Skipped: 2},
|
||||
{Package: "m/b", Passed: true, Tests: 3},
|
||||
}}
|
||||
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA, "m/b": digB}, at0)
|
||||
if err != nil || strings.Join(kept, ",") != "m/b" {
|
||||
t.Fatalf("kept %v, %v, want only m/b", kept, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7
|
||||
ctx, st := context.Background(), newMemKV()
|
||||
k := Kept{Store: st}
|
||||
|
||||
@@ -85,22 +85,18 @@ type pieces struct {
|
||||
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
|
||||
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
|
||||
judged string
|
||||
// carried says, of a carried verdict, the commits it was carried from (ADR 0548 §8); never dropped.
|
||||
carried string
|
||||
// warning says that a part of the check did not run, for instance that an older build module could not refuse a compiled program; never dropped.
|
||||
warning string
|
||||
}
|
||||
|
||||
// pullKey names a pull request of a case: `<owner/repo>#<number>`.
|
||||
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
|
||||
|
||||
func (p pieces) join(drop int) string {
|
||||
parts := []string{p.head, p.commits, p.carried, p.warning, p.counts, p.judged}
|
||||
parts := []string{p.head, p.commits, p.counts, p.judged}
|
||||
switch {
|
||||
case drop < 3 && p.times != "":
|
||||
parts = append(parts, p.times)
|
||||
case drop == 3 && p.shortTimes != "":
|
||||
parts = []string{p.head, p.commits, p.carried, p.warning, p.shortCounts, p.judged, p.shortTimes}
|
||||
parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
|
||||
}
|
||||
if p.over != "" {
|
||||
parts = append(parts, p.over)
|
||||
@@ -175,62 +171,16 @@ func buildWords(builds []BuildNote) string {
|
||||
return "builds: " + strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// carriedWords say the commits a verdict was carried from, for the line of pull request p (ADR 0548 §8): its own trunk's
|
||||
// commit as twelve characters, a sibling's with its name before it; only those that moved, sorted. Empty when none did.
|
||||
func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) string {
|
||||
var own string
|
||||
var others []string
|
||||
for r, old := range from {
|
||||
now, ok := v.Trunk[r]
|
||||
if cs != nil && !ok {
|
||||
now, ok = cs.Trunk[r]
|
||||
}
|
||||
if !ok {
|
||||
now = v.Beside[r]
|
||||
}
|
||||
if now == "" || now == old {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(r, p.Repository) {
|
||||
own = short12(old)
|
||||
continue
|
||||
}
|
||||
name := r
|
||||
if i := strings.LastIndex(r, "/"); i >= 0 {
|
||||
name = r[i+1:]
|
||||
}
|
||||
others = append(others, name+" "+short12(old))
|
||||
}
|
||||
slices.Sort(others)
|
||||
if own != "" {
|
||||
others = append([]string{own}, others...)
|
||||
}
|
||||
return strings.Join(others, ", ")
|
||||
}
|
||||
|
||||
// parts makes the pieces of the line of a pull request of a case.
|
||||
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
|
||||
var pc pieces
|
||||
pc.judged = judged[pullKey(p)]
|
||||
if cs != nil {
|
||||
pc.warning = cs.Warning(pullKey(p))
|
||||
}
|
||||
commits := short12(p.Head) + " on " + short12(p.Trunk)
|
||||
switch v.State {
|
||||
case delivery.VerdictRunning:
|
||||
pc.head = "checking " + commits
|
||||
if cs != nil && cs.CarriedFrom != nil {
|
||||
// Out of date and being carried to the new commit (ADR 0548 §6): pending, and saying so.
|
||||
pc.head = "being carried to " + commits
|
||||
if w := carriedWords(cs.CarriedFrom, v, cs, p); w != "" {
|
||||
pc.carried = "from " + w
|
||||
}
|
||||
}
|
||||
case delivery.VerdictPass:
|
||||
pc.head = "pass " + commits
|
||||
if w := carriedWords(v.CarriedFrom, v, cs, p); w != "" {
|
||||
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
|
||||
}
|
||||
default:
|
||||
pc.head = "failed: " + firstFailed(v)
|
||||
if n := failedCount(v); n > 1 {
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// **An ask the build seat gave up on ends its step** (novox/hq issue 734): mesh-sdk buildrun.LostAskWaiter hands the
|
||||
// waiting asker a failed step on the node buildrun.LostNode when DEAD_LETTERS keeps the ask's copy (buildrun.HearDeadLetters,
|
||||
// granted by buscompose DeadLetterReads), and, under buildrun.NotTakenBound, when no build node took the ask within the
|
||||
// bound. What follows is the checker's: a try ends as error (Tryer.run), with the not-taken bound at the step bound
|
||||
// (Tryer.NotTaken); a check is asked again under its claim, at most LostAskTries times, then fails as not heard and is
|
||||
// judged again when the nodes change (Checker.try). A check's ask has no not-taken bound: it waits in the stream as long as
|
||||
// the backlog is (ADR 0525 §9), and a check ended for a backlog would owe its pull request a failure.
|
||||
|
||||
// LostAskTries is how often a check asks again whose ask was given up, before it fails as not heard (UnheardWords) and is
|
||||
// judged again when the nodes change.
|
||||
const LostAskTries = 3
|
||||
|
||||
// IsLostAsk says whether a verdict failed because the build seat gave its ask up (buildrun.LostWords).
|
||||
func IsLostAsk(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.LostWords)
|
||||
}
|
||||
|
||||
// IsNotTaken says whether a verdict failed because no build node took its ask within the bound (buildrun.NotTakenWords).
|
||||
func IsNotTaken(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.NotTakenWords)
|
||||
}
|
||||
|
||||
// HearingBucket is the checker's state hearing: whether it hears DEAD_LETTERS (buildrun.Hearing under
|
||||
// buildrun.HearingKey), which checker_status says (dead_letters).
|
||||
const HearingBucket = delivery.CheckerModule + "_" + buildrun.HearingState
|
||||
@@ -1,170 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on
|
||||
// 2026-10-26 during a bus reload: try-1793017340003993788) and answers the rest; with never set it neither answers nor
|
||||
// gives up (an ask no node takes).
|
||||
type losingAsks struct {
|
||||
*buildrun.FakeAsks
|
||||
answer func(delivery.StepAsk) delivery.Step
|
||||
dead *buildrun.DeadLetters
|
||||
never bool
|
||||
|
||||
mu sync.Mutex
|
||||
lose int
|
||||
}
|
||||
|
||||
func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
||||
if err := l.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
|
||||
return err
|
||||
}
|
||||
l.mu.Lock()
|
||||
lost := l.lose > 0
|
||||
if lost {
|
||||
l.lose--
|
||||
}
|
||||
l.mu.Unlock()
|
||||
switch {
|
||||
case l.never:
|
||||
case lost:
|
||||
h := map[string][]string{buildrun.DeadHeaderMsgID: {id}, buildrun.DeadHeaderDeliveries: {"5"}, buildrun.DeadHeaderStream: {"SEAT_NODE_BUILD_GO"},
|
||||
buildrun.DeadHeaderGaveUp: {"2026-10-26T12:26:40Z"}}
|
||||
g, ok := buildrun.GivenUpOf(h)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
go l.dead.Heard(g)
|
||||
default:
|
||||
a, err := delivery.DecodeStepAsk(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l.Write(l.answer(a))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func losing(b *bench, lose int) *losingAsks {
|
||||
l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose}
|
||||
b.ex.Asks = buildrun.Asker{Publisher: l, Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
|
||||
return l
|
||||
}
|
||||
|
||||
// endable is the tries state as the end tool sees it.
|
||||
type endable struct{ kv *memKV }
|
||||
|
||||
func (e endable) Entry(key string) (json.RawMessage, uint64, error) {
|
||||
v, rev, _, err := e.kv.Get(context.Background(), key)
|
||||
return v, rev, err
|
||||
}
|
||||
func (e endable) Update(key string, value []byte, revision uint64) error {
|
||||
_, err := e.kv.Update(context.Background(), key, value, revision)
|
||||
return err
|
||||
}
|
||||
|
||||
// Issue 734: a try whose ask the build seat gave up on ends as error saying so, and the next ask of its head is a new try;
|
||||
// one no node takes ends at the step bound; one ended by hand frees its place and stops being waited for.
|
||||
func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
lose int
|
||||
never bool
|
||||
want string
|
||||
}{
|
||||
{"given up by the build seat", 1, false, buildrun.LostWords + " after 5 deliveries at 2026-10-26T12:26:40Z"},
|
||||
{"taken by no build node within the bound", 0, true, buildrun.NotTakenWords + " within 200ms"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
at := time.Date(2026, 10, 26, 12, 22, 20, 0, time.UTC)
|
||||
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
l := losing(b, row.lose)
|
||||
l.never = row.never
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex, NotTaken: 200 * time.Millisecond}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
got := readTry(t, kv, id)
|
||||
if got.State != TryError || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], row.want) || !strings.Contains(got.Said, row.want) {
|
||||
t.Fatalf("try = %+v, want error saying %q", got, row.want)
|
||||
}
|
||||
again, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at.Add(time.Minute))
|
||||
if err != nil || again == id {
|
||||
t.Errorf("the head asked again answered %q, %v, want a new try", again, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("ended by hand", func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
b := newBench(t)
|
||||
losing(b, 0).never = true
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex} // no bound: only the hand ends it
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for readTry(t, kv, id).State != TryRunning && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if _, err := EndTry(endable{kv}, id, "", time.Now()); err == nil {
|
||||
t.Error("an end without a why was taken")
|
||||
}
|
||||
ended, err := EndTry(endable{kv}, id, "its ask was lost in the bus reload", time.Now())
|
||||
if err != nil || ended.State != TryError || !strings.HasPrefix(ended.Said, "ended by hand ("+TryEndTool+"): its ask was lost") {
|
||||
t.Fatalf("ended = %+v, %v", ended, err)
|
||||
}
|
||||
raw, _, _, _ = kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw) // the watch of the tries hands the end over
|
||||
tr.Wait() // returns: the try's judging stopped
|
||||
if got := readTry(t, kv, id); got.State != TryError || got.Said != ended.Said {
|
||||
t.Errorf("try = %+v, want the hand's end kept", got)
|
||||
}
|
||||
if _, err := EndTry(endable{kv}, id, "again", time.Now()); err == nil || !strings.Contains(err.Error(), "ended already") {
|
||||
t.Errorf("a second end answered %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Issue 734: a pull request's check whose ask the build seat gave up on is asked again under its claim, its status
|
||||
// pending, and judged; one given up every time fails as not heard and is judged again when the nodes change.
|
||||
func TestACheckWhoseAskWasGivenUpIsAskedAgain(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, row := range []struct {
|
||||
lose int
|
||||
state string
|
||||
asks int
|
||||
}{
|
||||
{1, delivery.VerdictPass, 2},
|
||||
{LostAskTries, delivery.VerdictFail, LostAskTries},
|
||||
} {
|
||||
b := newBench(t)
|
||||
l := losing(b, row.lose)
|
||||
r := newRig(t, b.ex)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if v.State != row.state || len(l.Asks()) != row.asks {
|
||||
t.Errorf("lose %d: verdict %s (%s) after %d asks, want %s after %d", row.lose, v.State, v.Failed.Words, len(l.Asks()), row.state, row.asks)
|
||||
}
|
||||
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
|
||||
t.Errorf("a check given up every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// What the build module before this fix sent for two packages that failed after their tests passed (issue 1901, its third
|
||||
// meeting: a try of mesh-controller kept "PASS" as its failure's words). Each is that build module's Output for the real
|
||||
// `go test -json` of a throwaway package (mesh-catalog modules/mesh-build-go testdata/captured/go-test-json-pkg-*.jsonl):
|
||||
// TestMain panicking in a goroutine after m.Run, TestMain exiting 2 after m.Run, and TestMain racing after m.Run.
|
||||
const (
|
||||
oldPassThenPanic = "PASS\npanic: the bus closed after the tests: nats: connection closed\ngoroutine 20 [running]:\n" +
|
||||
"example.com/passthenfail.TestMain.func1()\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
|
||||
"created by example.com/passthenfail.TestMain in goroutine 1\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
|
||||
"FAIL\texample.com/passthenfail\t0.005s"
|
||||
oldPassThenExit = "PASS\nFAIL\texample.com/mainexit\t0.004s"
|
||||
// TestMain racing after m.Run passed, under go test -race (go-test-json-pkg-racemain.jsonl).
|
||||
oldPassThenRace = "PASS\n==================\nWARNING: DATA RACE\nWrite at 0x000000836428 by goroutine 9:\n example.com/racemain.TestMain.func1()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x24\nPrevious write at 0x000000836428 by main goroutine:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:16 +0x44\n main.main()\n _testmain.go:48 +0x171\nGoroutine 9 (running) created at:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x37\n main.main()\n _testmain.go:48 +0x171\n==================\nFound 1 data race(s)\nFAIL\texample.com/racemain\t1.031s"
|
||||
)
|
||||
|
||||
// A failure with no cause told apart never has "PASS" or "ok" as its words: the checker takes the panic line from the
|
||||
// output, or says that the tests passed and the program failed after them (1901/AC5).
|
||||
func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
|
||||
cases := []struct {
|
||||
name, output, want string
|
||||
}{
|
||||
{"panic after the tests", oldPassThenPanic, "panic: the bus closed after the tests: nats: connection closed"},
|
||||
{"exit after the tests", oldPassThenExit, "the tests passed, then the test program exited with a failure"},
|
||||
{"race after the tests", oldPassThenRace, "WARNING: DATA RACE: Write at 0x000000836428 by goroutine 9 in example.com/racemain.TestMain.func1()"},
|
||||
{"a panic deep in a long output", "PASS\npanic: the seat row vanished\n" + strings.Repeat("goroutine 7 [chan receive]:\n\tsome/frame.go:1 +0x1\n", 300) + "FAIL\tp\t2.0s", "panic: the seat row vanished"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "github.com/novox/mesh-controller/cmd/mesh-controller", Output: c.output}}})
|
||||
for _, l := range strings.Split(v.Failed.Words, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "PASS" || strings.HasPrefix(l, "ok ") || strings.HasPrefix(l, "ok\t") {
|
||||
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(afterHid(v.Failed.Words), c.want) {
|
||||
t.Errorf("1901/AC5: words after the loud line start %q, want %q", firstLine(afterHid(v.Failed.Words)), c.want)
|
||||
}
|
||||
if got := firstFailed(*v); !strings.Contains(got, "the build module hid the failure") || strings.HasSuffix(got, ": PASS") {
|
||||
t.Errorf("1901/AC5, AC8: status = %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A cause the build module told apart is cut to a line's bound, so the words keep their bound (the 1901 review's note).
|
||||
func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
|
||||
f := Failure{Package: "p", Test: "TestX", Cause: strings.Repeat("c", 5000), Output: "x_test.go:1: something"}
|
||||
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||
if len(v.Failed.Words) > MaxWords+200 {
|
||||
t.Errorf("1901/AC5: words are %d bytes, bound %d", len(v.Failed.Words), MaxWords)
|
||||
}
|
||||
}
|
||||
|
||||
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as only go test's last line, with no
|
||||
// cause: the words then say so, never nothing (review round 1). Each output is what that build module sends for the
|
||||
// capture of the same name (go-test-json-pkg-timeout, -exiting, -killed): the package's own output, its "FAIL" line.
|
||||
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
|
||||
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
|
||||
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
|
||||
if !strings.HasPrefix(afterHid(v.Failed.Words), "the package failed with no test to name and no panic, signal or build error in its output") {
|
||||
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A line of the test's own that starts as go's summary does, or that holds a runtime's words indented, is not taken for
|
||||
// either (review round 1): go prints its summary in exact forms and the runtime's lines at column 0.
|
||||
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
|
||||
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
|
||||
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
|
||||
if !strings.HasPrefix(afterHid(v.Failed.Words), "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
|
||||
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
|
||||
// The words keep MaxWords exactly, counting the cause, the marker and the newlines (review round 1).
|
||||
func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
|
||||
for _, cause := range []string{"", "a cause", strings.Repeat("c", 3000)} {
|
||||
for _, n := range []int{100, 170, 171, 172, 500} {
|
||||
f := Failure{Package: "p", Test: "TestX", Cause: cause, Output: strings.Repeat("a log line of twenty\n", n) + "the end"}
|
||||
if w := failureWords(f); len(w) > MaxWords {
|
||||
t.Errorf("1901/AC5: cause %d bytes, %d lines: words are %d bytes, bound %d", len(cause), n, len(w), MaxWords)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// afterHid is a verdict's words after the checker's loud line, which a failure with no cause from an older build module
|
||||
// now starts with (1901/AC8).
|
||||
func afterHid(w string) string {
|
||||
if strings.HasPrefix(w, "the build module hid the failure") {
|
||||
_, rest, _ := strings.Cut(w, "\n")
|
||||
return rest
|
||||
}
|
||||
return w
|
||||
}
|
||||
@@ -240,12 +240,7 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
||||
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
|
||||
b.ex.Forge = forge
|
||||
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
|
||||
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
||||
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
|
||||
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
|
||||
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" || out.Beside[SDKRepository] != "9999999999999999999999999999999999999999" {
|
||||
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
|
||||
}
|
||||
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
||||
got := besideOf(t, b, controller)
|
||||
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
|
||||
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
|
||||
@@ -329,11 +324,7 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
|
||||
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
|
||||
b := newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
|
||||
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
||||
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
|
||||
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
|
||||
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
|
||||
}
|
||||
b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
||||
got := besideOf(t, b, catalogueRepository)
|
||||
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
|
||||
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
|
||||
|
||||
@@ -75,40 +75,3 @@ func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4
|
||||
t.Errorf("waiting = %v, want %v", st.Waiting, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 568 note 21: a verdict that read the node-engine at its running commit is not waiting for a trunk move while that commit
|
||||
// is still the one the mesh runs, though the node-engine's trunk has moved on (ADR 0548 §6).
|
||||
func TestCheckerStatusReadsTheNodeEngineAtItsRunningCommit(t *testing.T) { // 568/21
|
||||
cand := candidateOf(t, group(1), pullOf(repoA, 1, head1))
|
||||
ckey, _ := delivery.CandidatesKey(group(1))
|
||||
vkey := keyOf(t, group(1), cand)
|
||||
dc, _ := delivery.DecodeCandidate(cand)
|
||||
judged, err := delivery.EncodeVerdict(delivery.Verdict{Group: group(1), Heads: dc.HeadsHash, Trunk: map[string]string{repoA: trunkA}, Beside: map[string]string{nodeEngineRepo: "run111"}, State: delivery.VerdictPass, Levels: passing().Levels})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heads := func() (map[string]string, error) {
|
||||
return map[string]string{repoA: trunkA, nodeEngineRepo: "trunk222"}, nil
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
runs runsAt
|
||||
want int
|
||||
}{
|
||||
{"the running commit the verdict read", runsAt{nodeEngineModule: "run111"}, 0},
|
||||
{"the running commit moved", runsAt{nodeEngineModule: "run999"}, 1},
|
||||
{"no running commit known: the trunk's", nil, 1},
|
||||
} {
|
||||
rd := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{vkey: judged}.read(), Trunks: heads}
|
||||
if row.runs != nil {
|
||||
rd.Runs = row.runs
|
||||
}
|
||||
st, err := rd.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(st.Waiting[KindTrunk]); got != row.want {
|
||||
t.Errorf("%s: waiting for a trunk move = %v, want %d", row.name, st.Waiting[KindTrunk], row.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
||||
@@ -40,14 +39,7 @@ type Reader struct {
|
||||
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
||||
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
|
||||
Trunks func() (map[string]string, error)
|
||||
// TurnState is the gate's merge turns (ADR 0548 §3); without it no check is named as held by a turn.
|
||||
TurnState StateRead
|
||||
// Runs says which commit the mesh runs of a module (the indexer's modules), as the checker's decision reads it: the
|
||||
// node-engine a verdict names is compared at its running commit, not at the trunk's (ADR 0548 §6); nil knows none.
|
||||
Runs interface{ CommitOf(module string) string }
|
||||
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
|
||||
HearingState StateRead
|
||||
Now func() time.Time
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
||||
@@ -86,11 +78,6 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
||||
cs := CaseOf(c, v.Trunk)
|
||||
out.Reports = map[string]string{}
|
||||
if cs == nil {
|
||||
if IsUntrunked(v) {
|
||||
for _, p := range leftOf(c, v.Trunk) {
|
||||
out.Reports[pullKey(p)] = v.Failed.Words
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
for _, p := range cs.Pulls {
|
||||
@@ -102,46 +89,37 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Verdicts are the verdicts the checker holds, by key; with a group only that group's. The keys are narrowed by the group
|
||||
// before any value is read, at most MaxReads are read (sixteen at a time), and cut says when it stopped there.
|
||||
func (r Reader) Verdicts(group string) (out []VerdictRead, cut string, err error) {
|
||||
// Verdicts are the verdicts the checker holds, by key; with a group only that group's.
|
||||
func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
|
||||
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
|
||||
return nil, "", fmt.Errorf("the verdicts state cannot be read here")
|
||||
return nil, fmt.Errorf("the verdicts state cannot be read here")
|
||||
}
|
||||
keys, err := r.VerdictState.Keys()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var matching []string
|
||||
var out []VerdictRead
|
||||
for _, k := range keys {
|
||||
if group != "" {
|
||||
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
|
||||
continue
|
||||
}
|
||||
}
|
||||
matching = append(matching, k)
|
||||
}
|
||||
matching, cut = capKeys(matching, "give a group to narrow the answer")
|
||||
rows := make([]VerdictRead, len(matching))
|
||||
found := make([]bool, len(matching))
|
||||
if err := parallel(len(matching), func(i int) error {
|
||||
raw, err := r.VerdictState.Get(matching[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return err
|
||||
raw, err := r.VerdictState.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows[i], err = r.read(matching[i], raw)
|
||||
found[i] = err == nil
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
for i, row := range rows {
|
||||
if found[i] {
|
||||
out = append(out, row)
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
vr, err := r.read(k, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, vr)
|
||||
}
|
||||
return out, cut, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
|
||||
@@ -155,95 +133,74 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
|
||||
for from := 0; from < len(keys); from += ReadWorkers * 4 {
|
||||
batch := keys[from:min(from+ReadWorkers*4, len(keys))]
|
||||
cands := make([]delivery.Candidate, len(batch))
|
||||
ok := make([]bool, len(batch))
|
||||
_ = parallel(len(batch), func(i int) error {
|
||||
raw, err := r.CandidateState.Get(batch[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
cands[i], ok[i] = c, err == nil
|
||||
return nil
|
||||
})
|
||||
for i, c := range cands {
|
||||
if !ok[i] {
|
||||
for _, k := range keys {
|
||||
raw, err := r.CandidateState.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository != repository || p.Number != number {
|
||||
continue
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository != repository || p.Number != number {
|
||||
continue
|
||||
}
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
vraw, err := r.VerdictState.Get(vkey)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
if len(vraw) == 0 {
|
||||
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
||||
}
|
||||
return r.read(vkey, vraw)
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
vraw, err := r.VerdictState.Get(vkey)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
if len(vraw) == 0 {
|
||||
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
||||
}
|
||||
return r.read(vkey, vraw)
|
||||
}
|
||||
}
|
||||
if cut != "" {
|
||||
return VerdictRead{}, fmt.Errorf("no candidate among the first %d holds %s#%d", MaxReads, repository, number)
|
||||
}
|
||||
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
|
||||
}
|
||||
|
||||
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
|
||||
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, string, error) {
|
||||
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
|
||||
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
|
||||
return nil, "", fmt.Errorf("the kept state cannot be read here")
|
||||
return nil, fmt.Errorf("the kept state cannot be read here")
|
||||
}
|
||||
if digest != "" {
|
||||
key, err := delivery.KeptKey(level, digest)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
raw, err := r.KeptState.Get(key)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, "", fmt.Errorf("no kept verdict %s", key)
|
||||
return nil, fmt.Errorf("no kept verdict %s", key)
|
||||
}
|
||||
return map[string]json.RawMessage{key: raw}, "", nil
|
||||
return map[string]json.RawMessage{key: raw}, nil
|
||||
}
|
||||
keys, err := r.KeptState.Keys()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var matching []string
|
||||
for _, k := range keys {
|
||||
if l, _, err := delivery.ParseKeptKey(k); err == nil && (level == "" || l == level) {
|
||||
matching = append(matching, k)
|
||||
}
|
||||
}
|
||||
matching, cut := capKeys(matching, "give a level and a digest to read one")
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
if err := parallel(len(matching), func(i int) (err error) {
|
||||
values[i], err = r.KeptState.Get(matching[i])
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
out := map[string]json.RawMessage{}
|
||||
for i, k := range matching {
|
||||
if len(values[i]) > 0 {
|
||||
out[k] = values[i]
|
||||
for _, k := range keys {
|
||||
if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
|
||||
continue
|
||||
}
|
||||
raw, err := r.KeptState.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out[k] = raw
|
||||
}
|
||||
}
|
||||
return out, cut, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PreviewRead is one preview as stored, with the parts of its key.
|
||||
@@ -257,18 +214,17 @@ type PreviewRead struct {
|
||||
|
||||
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
||||
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
||||
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, string, error) {
|
||||
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
|
||||
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
||||
return nil, nil, "", fmt.Errorf("the previews state cannot be read here")
|
||||
return nil, nil, fmt.Errorf("the previews state cannot be read here")
|
||||
}
|
||||
keys, err := r.PreviewState.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, "", err
|
||||
return nil, nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
type part struct{ repo, head, trunk string }
|
||||
parts := map[string]part{}
|
||||
var matching, bad []string
|
||||
out := []PreviewRead{}
|
||||
var bad []string
|
||||
for _, k := range keys {
|
||||
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
||||
if err != nil {
|
||||
@@ -278,25 +234,15 @@ func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, stri
|
||||
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
||||
continue
|
||||
}
|
||||
parts[k] = part{repo, h, trunk}
|
||||
matching = append(matching, k)
|
||||
}
|
||||
matching, cut := capKeys(matching, "give a repository and a head to narrow the answer")
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
if err := parallel(len(matching), func(i int) (err error) {
|
||||
values[i], err = r.PreviewState.Get(matching[i])
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, nil, "", err
|
||||
}
|
||||
out := []PreviewRead{}
|
||||
for i, k := range matching {
|
||||
if len(values[i]) > 0 {
|
||||
p := parts[k]
|
||||
out = append(out, PreviewRead{Key: k, Repository: p.repo, Head: p.head, Trunk: p.trunk, Preview: values[i]})
|
||||
raw, err := r.PreviewState.Get(k)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
|
||||
}
|
||||
}
|
||||
return out, bad, cut, nil
|
||||
return out, bad, nil
|
||||
}
|
||||
|
||||
// RunningStep is a step in flight on a build module.
|
||||
@@ -325,71 +271,7 @@ type Status struct {
|
||||
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
|
||||
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
|
||||
Waiting map[string][]string `json:"waiting_by_kind"`
|
||||
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and
|
||||
// the group of the merge that waits. Their check is claimed when that turn goes.
|
||||
HeldByTurn map[string]string `json:"held_by_turn"`
|
||||
// CannotBeJudged are the candidates no pull request of which can be judged, by group, with why: the gate's source.main
|
||||
// names no trunk head of their repositories (issue 1422). They are judged once it does; until then their status is error.
|
||||
CannotBeJudged map[string]string `json:"cannot_be_judged"`
|
||||
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
|
||||
// whose ask is given up waits until its bound.
|
||||
DeadLetters DeadLettersHeard `json:"dead_letters"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// DeadLettersHeard is the checker's hearing of DEAD_LETTERS as the status says it: healthy or not, and why.
|
||||
type DeadLettersHeard struct {
|
||||
Heard bool `json:"heard"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// deadLetters reads the checker's kept hearing (buildrun.HearingHealth).
|
||||
func (r Reader) deadLetters() DeadLettersHeard {
|
||||
if r.HearingState.Get == nil {
|
||||
return DeadLettersHeard{Why: "the checker's state hearing is not readable here"}
|
||||
}
|
||||
raw, err := r.HearingState.Get(buildrun.HearingKey)
|
||||
if err != nil {
|
||||
return DeadLettersHeard{Why: "the checker's state hearing cannot be read: " + err.Error()}
|
||||
}
|
||||
a := buildrun.HearingHealth(raw)
|
||||
return DeadLettersHeard{Heard: a.Healthy, Why: a.Why}
|
||||
}
|
||||
|
||||
// turns reads the gate's merge turns, by repository; nil when they cannot be read, said in a note.
|
||||
func (r Reader) turns(st *Status) *Turns {
|
||||
if r.TurnState.Keys == nil || r.TurnState.Get == nil {
|
||||
st.Notes = append(st.Notes, "the gate's merge turns are not readable here, so no check is named as held by one")
|
||||
return nil
|
||||
}
|
||||
keys, err := r.TurnState.Keys()
|
||||
if err != nil {
|
||||
st.Notes = append(st.Notes, "the gate's merge turns cannot be listed, so no check is named as held by one: "+err.Error())
|
||||
return nil
|
||||
}
|
||||
t := &Turns{}
|
||||
for _, k := range keys {
|
||||
if raw, err := r.TurnState.Get(k); err == nil && len(raw) > 0 {
|
||||
t.Put(k, raw, false)
|
||||
}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// heldBy says the turn that holds a candidate's check off, or "".
|
||||
func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
|
||||
cs := CaseOf(c, heads)
|
||||
if t == nil || cs == nil {
|
||||
return ""
|
||||
}
|
||||
repos := make([]string, 0, len(cs.Trunk))
|
||||
for r := range cs.Trunk {
|
||||
repos = append(repos, r)
|
||||
}
|
||||
if repo, w, ok := t.Holding(repos, cs.Heads); ok {
|
||||
return repo + ": the approved merge of " + w.Group + " waits on it"
|
||||
}
|
||||
return ""
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// Status reads the build modules' steps and the candidates against their verdicts.
|
||||
@@ -418,28 +300,16 @@ func (r Reader) Status() (Status, error) {
|
||||
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
|
||||
newestFirst(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
if cut != "" {
|
||||
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
|
||||
}
|
||||
steps := make([]delivery.Step, len(keys))
|
||||
isRunning := make([]bool, len(keys))
|
||||
_ = parallel(len(keys), func(i int) error {
|
||||
raw, err := sr.Get(keys[i])
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
raw, err := sr.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
|
||||
return nil
|
||||
})
|
||||
for i := range keys {
|
||||
if !isRunning[i] {
|
||||
continue
|
||||
}
|
||||
step := steps[i]
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
if err != nil || step.State != delivery.StepRunning {
|
||||
continue
|
||||
}
|
||||
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
||||
if !step.Started.IsZero() {
|
||||
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
||||
@@ -460,8 +330,6 @@ func (r Reader) Status() (Status, error) {
|
||||
}
|
||||
st.Unjudged = []string{}
|
||||
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
|
||||
st.HeldByTurn, st.CannotBeJudged = map[string]string{}, map[string]string{}
|
||||
turns := r.turns(&st)
|
||||
var heads map[string]string
|
||||
if r.Trunks == nil {
|
||||
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
|
||||
@@ -476,56 +344,26 @@ func (r Reader) Status() (Status, error) {
|
||||
return st, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
total := len(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
if cut != "" {
|
||||
st.Notes = append(st.Notes, fmt.Sprintf("read the first %d candidates of %d", MaxReads, total))
|
||||
}
|
||||
type judged struct {
|
||||
c delivery.Candidate
|
||||
raw json.RawMessage
|
||||
ok bool
|
||||
}
|
||||
rows := make([]judged, len(keys))
|
||||
_ = parallel(len(keys), func(i int) error {
|
||||
raw, err := r.CandidateState.Get(keys[i])
|
||||
for _, k := range keys {
|
||||
raw, err := r.CandidateState.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
rows[i].c = c
|
||||
if vraw, err := r.VerdictState.Get(vkey); err == nil {
|
||||
rows[i].raw, rows[i].ok = vraw, true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for _, row := range rows {
|
||||
if !row.ok {
|
||||
continue
|
||||
}
|
||||
c, raw := row.c, row.raw
|
||||
if heads != nil && CaseOf(c, heads) == nil {
|
||||
// Nothing of it can be judged now, whatever its verdict says (issue 1422): never counted as waiting.
|
||||
var repos []string
|
||||
for _, p := range leftOf(c, heads) {
|
||||
repos = append(repos, p.Repository)
|
||||
}
|
||||
st.CannotBeJudged[c.Group] = UntrunkedWordsOf(repos)
|
||||
raw, err = r.VerdictState.Get(vkey)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
st.Unjudged = append(st.Unjudged, c.Group)
|
||||
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||
if why := heldBy(turns, c, heads); why != "" {
|
||||
st.HeldByTurn[c.Group] = why
|
||||
}
|
||||
continue
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(raw)
|
||||
@@ -533,20 +371,13 @@ func (r Reader) Status() (Status, error) {
|
||||
case err != nil || v.State == delivery.VerdictRunning:
|
||||
case IsNodesFailure(v):
|
||||
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
||||
case IsUntrunked(v):
|
||||
// Its trunk head is known now: it waits for its whole check.
|
||||
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||
case heads != nil:
|
||||
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) {
|
||||
if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
|
||||
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
|
||||
if why := heldBy(turns, c, heads); why != "" {
|
||||
st.HeldByTurn[c.Group] = why
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
st.DeadLetters = r.deadLetters()
|
||||
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
|
||||
return st, nil
|
||||
}
|
||||
|
||||
@@ -3,9 +3,7 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -53,7 +51,7 @@ func readerOf(t *testing.T) (Reader, string) {
|
||||
|
||||
func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
|
||||
rd, key := readerOf(t)
|
||||
vs, _, err := rd.Verdicts("")
|
||||
vs, err := rd.Verdicts("")
|
||||
if err != nil || len(vs) != 1 || vs[0].Key != key {
|
||||
t.Fatalf("verdicts = %+v, %v", vs, err)
|
||||
}
|
||||
@@ -63,7 +61,7 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
|
||||
if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
|
||||
t.Errorf("reports = %v", vs[0].Reports)
|
||||
}
|
||||
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
|
||||
if only, _ := rd.Verdicts("feat/other"); len(only) != 0 {
|
||||
t.Errorf("another group's verdicts = %+v", only)
|
||||
}
|
||||
}
|
||||
@@ -87,11 +85,11 @@ func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) {
|
||||
|
||||
func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
|
||||
rd, _ := readerOf(t)
|
||||
all, _, err := rd.Kept("", "")
|
||||
all, err := rd.Kept("", "")
|
||||
if err != nil || len(all) != 1 {
|
||||
t.Fatalf("kept = %v, %v", all, err)
|
||||
}
|
||||
one, _, err := rd.Kept("unit", digA)
|
||||
one, err := rd.Kept("unit", digA)
|
||||
if err != nil || len(one) != 1 {
|
||||
t.Fatalf("one kept = %v, %v", one, err)
|
||||
}
|
||||
@@ -100,10 +98,10 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
|
||||
t.Errorf("kept value = %+v, %v", k, err)
|
||||
}
|
||||
}
|
||||
if none, _, _ := rd.Kept("integration", ""); len(none) != 0 {
|
||||
if none, _ := rd.Kept("integration", ""); len(none) != 0 {
|
||||
t.Errorf("integration = %v", none)
|
||||
}
|
||||
if _, _, err := rd.Kept("unit", digB); err == nil {
|
||||
if _, err := rd.Kept("unit", digB); err == nil {
|
||||
t.Error("a digest nothing kept was answered")
|
||||
}
|
||||
}
|
||||
@@ -117,17 +115,17 @@ func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
|
||||
}
|
||||
key, _ := delivery.PreviewKey(repoA, head1, head2)
|
||||
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
|
||||
got, bad, _, err := rd.Previews(repoA, head1[:8])
|
||||
got, bad, err := rd.Previews(repoA, head1[:8])
|
||||
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
|
||||
t.Fatalf("previews = %+v, %v", got, err)
|
||||
}
|
||||
if len(bad) != 1 || bad[0] != "junk" {
|
||||
t.Errorf("unreadable = %v", bad)
|
||||
}
|
||||
if got, _, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
||||
if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
||||
t.Errorf("another repository's = %+v", got)
|
||||
}
|
||||
if _, _, _, err := (Reader{}).Previews("", ""); err == nil {
|
||||
if _, _, err := (Reader{}).Previews("", ""); err == nil {
|
||||
t.Error("answered with no state to read")
|
||||
}
|
||||
}
|
||||
@@ -144,10 +142,7 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
||||
CandidateState: fakeState{ckey: cand}.read(),
|
||||
VerdictState: fakeState{}.read(),
|
||||
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||
HearingState: fakeState{"dead-letters": []byte(`{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],` +
|
||||
`"why":"DEAD_LETTERS is not heard, so an ask a build seat gives up on is waited for until its bound: SEAT_NODE_BUILD_GO: permissions violation",` +
|
||||
`"at":"2026-10-26T13:00:00Z"}`)}.read(),
|
||||
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
|
||||
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
|
||||
}
|
||||
st, err := rd.Status()
|
||||
if err != nil {
|
||||
@@ -159,13 +154,9 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
||||
t.Errorf("unjudged = %v", st.Unjudged)
|
||||
}
|
||||
// Issue 734, review round 1: a checker that does not hear DEAD_LETTERS says so in its status, with why.
|
||||
if st.DeadLetters.Heard || !strings.Contains(st.DeadLetters.Why, "permissions violation") {
|
||||
t.Errorf("dead letters = %+v, want not heard, saying why", st.DeadLetters)
|
||||
}
|
||||
// What cannot be read is said, not guessed.
|
||||
st, err = Reader{}.Status()
|
||||
if err != nil || len(st.Notes) == 0 || st.DeadLetters.Heard {
|
||||
if err != nil || len(st.Notes) == 0 {
|
||||
t.Errorf("empty status = %+v, %v", st, err)
|
||||
}
|
||||
}
|
||||
@@ -241,134 +232,4 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
|
||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
|
||||
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
|
||||
}
|
||||
// ADR 0548 §3: a candidate whose check an approved merge's turn holds off is named with the turn.
|
||||
held := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
hkey, _ := delivery.CandidatesKey("feat/x")
|
||||
st, err = Reader{CandidateState: fakeState{hkey: held}.read(), VerdictState: fakeState{}.read(),
|
||||
Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA}, nil },
|
||||
TurnState: fakeState{turnKey(t, repoA): turnOf(t, repoA, "fix/other", "0123456789ab")}.read()}.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := st.HeldByTurn["feat/x"]; !strings.Contains(why, repoA) || !strings.Contains(why, "fix/other") {
|
||||
t.Errorf("held_by_turn = %v, want feat/x held by the merge of fix/other on %s", st.HeldByTurn, repoA)
|
||||
}
|
||||
}
|
||||
|
||||
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
|
||||
type slowBucket struct {
|
||||
vals fakeState
|
||||
reads, lists atomic.Int64
|
||||
inFlight, maxFlight atomic.Int64
|
||||
}
|
||||
|
||||
func (b *slowBucket) get(key string) (json.RawMessage, error) {
|
||||
b.reads.Add(1)
|
||||
now := b.inFlight.Add(1)
|
||||
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
b.inFlight.Add(-1)
|
||||
return b.vals.read().Get(key)
|
||||
}
|
||||
|
||||
func (b *slowBucket) read() StateRead {
|
||||
return StateRead{Keys: func() ([]string, error) { b.lists.Add(1); return b.vals.read().Keys() }, Get: b.get}
|
||||
}
|
||||
|
||||
func (b *slowBucket) Keys() ([]string, error) { return b.read().Keys() }
|
||||
func (b *slowBucket) Get(key string) (json.RawMessage, error) { return b.get(key) }
|
||||
func (b *slowBucket) Create(string, []byte) error { return nil }
|
||||
|
||||
// 1827: a read tool lists the keys of a bucket of thousands and reads only the values its answer needs, sixteen at a time, at
|
||||
// most MaxReads, saying when it cut (on the live shape of 2026-10-26: kept 7990 keys, steps 10034, tries 1000s).
|
||||
func TestReadToolsReadOnlyWhatTheyNeedInParallelAndSayWhenCut(t *testing.T) { // 1827
|
||||
kept := fakeState{}
|
||||
for i := 0; i < 1800; i++ {
|
||||
level := map[bool]string{true: "unit", false: "integration"}[i%9 != 0]
|
||||
k, _ := delivery.KeptKey(level, fmt.Sprintf("%064x", i))
|
||||
kept[k] = []byte(`{"v":1}`)
|
||||
}
|
||||
previews := fakeState{}
|
||||
for i := 0; i < 1500; i++ {
|
||||
repo := map[bool]string{true: repoA, false: repoSDK}[i < 300]
|
||||
k, _ := delivery.PreviewKey(repo, fmt.Sprintf("%08x%032x", i, 0), trunkA)
|
||||
previews[k] = []byte(`{"v":1}`)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
call func(*slowBucket) (rows int, cut string, err error)
|
||||
rows int
|
||||
reads, lists int64
|
||||
cut bool
|
||||
}{
|
||||
{"kept of one key", func(b *slowBucket) (int, string, error) {
|
||||
m, c, err := Reader{KeptState: b.read()}.Kept("unit", fmt.Sprintf("%064x", 1))
|
||||
return len(m), c, err
|
||||
}, 1, 1, 0, false},
|
||||
{"kept of a level", func(b *slowBucket) (int, string, error) {
|
||||
m, c, err := Reader{KeptState: b.read()}.Kept("integration", "")
|
||||
return len(m), c, err
|
||||
}, 200, 200, 1, false},
|
||||
{"kept of everything", func(b *slowBucket) (int, string, error) {
|
||||
m, c, err := Reader{KeptState: b.read()}.Kept("", "")
|
||||
return len(m), c, err
|
||||
}, MaxReads, MaxReads, 1, true},
|
||||
{"previews of a repository", func(b *slowBucket) (int, string, error) {
|
||||
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoSDK, "")
|
||||
return len(p), c, err
|
||||
}, 1200, 1000, 1, true},
|
||||
{"previews of a repository and a head", func(b *slowBucket) (int, string, error) {
|
||||
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoA, fmt.Sprintf("%08x", 7))
|
||||
return len(p), c, err
|
||||
}, 1, 1, 1, false},
|
||||
} {
|
||||
b := &slowBucket{}
|
||||
b.vals = kept
|
||||
if strings.HasPrefix(row.name, "previews") {
|
||||
b.vals = previews
|
||||
}
|
||||
start := time.Now()
|
||||
rows, cut, err := row.call(b)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
if rows != min(row.rows, MaxReads) || b.reads.Load() != row.reads || b.lists.Load() != row.lists || (cut != "") != row.cut {
|
||||
t.Errorf("%s: %d rows, %d reads, %d lists, cut %q; want %d, %d, %d, cut %v", row.name, rows, b.reads.Load(), b.lists.Load(), cut, min(row.rows, MaxReads), row.reads, row.lists, row.cut)
|
||||
}
|
||||
if row.reads > 20 && b.maxFlight.Load() < 2 {
|
||||
t.Errorf("%s: values were read one at a time", row.name)
|
||||
}
|
||||
t.Logf("%s: %d values in %v (one after the other: %v)", row.name, b.reads.Load(), time.Since(start), time.Duration(b.reads.Load())*time.Millisecond)
|
||||
}
|
||||
|
||||
// The status reads the newest steps of a bucket of 1500 first, at most MaxReads, and finds the one that runs.
|
||||
now := time.Date(2026, 10, 26, 7, 0, 0, 0, time.UTC)
|
||||
steps := fakeState{}
|
||||
for i := 0; i < 1500; i++ {
|
||||
id := fmt.Sprintf("mvo5%04dabcdef", i)
|
||||
step := delivery.Step{ID: id, Tool: "t", Node: "g14", State: delivery.StepPassed}
|
||||
if i == 1499 {
|
||||
step.State, step.Started = delivery.StepRunning, now.Add(-time.Minute)
|
||||
}
|
||||
raw, _ := delivery.EncodeStep(step)
|
||||
steps["g14."+id] = raw
|
||||
}
|
||||
sb := &slowBucket{vals: steps}
|
||||
st, err := Reader{StepStates: map[string]StateRead{"mesh-build-go": sb.read()}, HaveState: fakeState{"g14.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||
Now: func() time.Time { return now }}.Status()
|
||||
if err != nil || sb.reads.Load() != MaxReads || len(st.Running) != 1 || sb.maxFlight.Load() < 2 {
|
||||
t.Errorf("status: %d reads (want %d), running %+v, in flight %d, %v", sb.reads.Load(), MaxReads, st.Running, sb.maxFlight.Load(), err)
|
||||
}
|
||||
|
||||
// A new try reads the newest TryScan of 1500 ended tries, not all, and still finds the same head asked twice.
|
||||
tries := fakeState{}
|
||||
for i := 0; i < 1500; i++ {
|
||||
raw, _ := encodeTry(Try{ID: fmt.Sprintf("try-%d", 1700000000000000000+i), Repository: repoA, Head: head1, Trunk: trunkA, State: TryPass})
|
||||
tries[fmt.Sprintf("try-%d", 1700000000000000000+i)] = raw
|
||||
}
|
||||
tb := &slowBucket{vals: tries}
|
||||
if _, err := AskTry(tb, tryTrunks, TryAsk{Repository: repoA, Head: head2, Trunk: trunkA}, now); err != nil || tb.reads.Load() != TryScan {
|
||||
t.Errorf("AskTry read %d of 1500 tries (want %d), %v", tb.reads.Load(), TryScan, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,9 +3,6 @@ package check
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go
|
||||
@@ -45,17 +42,6 @@ type Fetched struct {
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
Trees map[string]MergeTree `json:"trees,omitempty"`
|
||||
// Programs are the changed files whose first bytes are an executable binary, read by the build module in its workspace
|
||||
// (issue 1577); nil for a build module that does not report it, a pointer to an empty list for one that found none.
|
||||
Programs *[]ProgramFile `json:"programs,omitempty"`
|
||||
}
|
||||
|
||||
// ProgramFile is a changed file that is a compiled program, and its format (ELF, Mach-O or PE).
|
||||
type ProgramFile struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
// Commit is set for a program a commit of the branch added and a later one removed: it stays in history once merged.
|
||||
Commit string `json:"commit,omitempty"`
|
||||
}
|
||||
|
||||
// GoPackage is one package of a Go test result.
|
||||
@@ -66,13 +52,16 @@ type GoPackage struct {
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
Tests int `json:"tests"`
|
||||
NoTests bool `json:"no_tests,omitempty"`
|
||||
// Skipped is how many of the package's tests skipped; an older build module names none.
|
||||
Skipped int `json:"skipped,omitempty"`
|
||||
}
|
||||
|
||||
// Failure is a test that failed, or a package that failed with no test to name: the build modules' failure contract
|
||||
// (buildstep.Failure, issue 1901), its cause never empty and never a passing line from a build module that keeps it.
|
||||
type Failure = buildstep.Failure
|
||||
// Failure is a test that failed, or a package that failed with no test to name.
|
||||
type Failure struct {
|
||||
Package string `json:"package"`
|
||||
Test string `json:"test,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
// Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901).
|
||||
Cause string `json:"cause,omitempty"`
|
||||
}
|
||||
|
||||
// TestTime is one test and the seconds it took.
|
||||
type TestTime struct {
|
||||
@@ -105,9 +94,6 @@ type Tested struct {
|
||||
// JudgedBy is the program that ran the step (ADR 0535 §5): "installed", or "candidate <merge commit>" under runner
|
||||
// workspace; absent from a build module that does not take the runner.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
// By is the build module that answered and the SDK it was built with (buildstep.By); absent from a build module from
|
||||
// before the failure contract.
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import
|
||||
@@ -121,6 +107,4 @@ type Digested struct {
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
// JudgedBy is the program that ran the digest, as Tested.JudgedBy says.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
// Needs are the packages whose tests need something of the node at this level (issue 890).
|
||||
Needs []delivery.PackageNeeds `json:"needs,omitempty"`
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ TestFetchWithABaseMergesAndAnswersThePreview and TestFetchWithAConflictEndsPasse
|
||||
into a throwaway test that wrote them out, at mesh-catalog feat/r4c5-checker d5804927 (the git commands are the test's fake,
|
||||
so the shas are its constants; the shape and the field names are the module's own marshalling). node-fetch-clean.json is the
|
||||
FetchResult json.Marshal of mesh-build-node's fetch (fetch.go, preview.go) in its test TestFetchWithABaseMergesAndAnswersThePreview
|
||||
(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679. Both clean fetches carry `"programs":[]`, which a merging fetch of the build modules answers since issue 1577 (mesh-sdk buildstep.MergeInto's Programs), added by hand to the captured bytes: a fixture without it is an older build module's, and its line warns that no compiled program is refused.
|
||||
(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679.
|
||||
|
||||
forge-get-file-go-mod.json is the answer of the forge's gitea_get_file (mesh-catalog modules/gitea, tools/index.ts) for modules/mesh-delivery-builder/go.mod of
|
||||
novox/mesh-delivery at main, and forge-compare-trunk.json the answer of gitea_compare for novox/mesh-sdk base main, head cb54756c3865 (a commit on the
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"programs":[],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}}
|
||||
{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"programs":[],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}}
|
||||
{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -14,7 +13,6 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501
|
||||
@@ -26,19 +24,11 @@ import (
|
||||
const (
|
||||
TryTool = "checker_try"
|
||||
TryReadTool = "checker_try_read"
|
||||
TryEndTool = "checker_try_end"
|
||||
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
|
||||
TriesState = "tries"
|
||||
TriesBucket = delivery.CheckerModule + "_" + TriesState
|
||||
// MaxTries and MaxTriesWaiting are checked when a try is asked, mesh-wide: a new try is refused while MaxTries tries run
|
||||
// on a build node or MaxTriesWaiting wait for one (queued, or their asks not yet taken by a node). It is no bound on
|
||||
// what runs later: tries let in while waiting may later all run at once, and each node's build capacity bounds what
|
||||
// runs on it. A try waiting behind the seat's other asks holds no node, so it no longer blocks the next try (issue
|
||||
// 1890: at 19:53 on 2026-10-26 four tries sat queued behind 44 checks for up to 47 minutes and no fifth could be asked).
|
||||
MaxTries = 4
|
||||
MaxTriesWaiting = 8
|
||||
// TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827).
|
||||
TryScan = 64
|
||||
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
|
||||
MaxTries = 4
|
||||
// TryKeep is how long an ended try's key is kept before the checker removes it.
|
||||
TryKeep = 7 * 24 * time.Hour
|
||||
)
|
||||
@@ -83,104 +73,9 @@ type Try struct {
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
|
||||
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
|
||||
Asks []string `json:"asks,omitempty"`
|
||||
// Steps say where each ask stands while the try runs and after: asked until a node takes it, then the node, its
|
||||
// state, the phase its step says and when (issue 1890: a 31-minute try could not be told queued from running).
|
||||
Steps []TryStep `json:"steps,omitempty"`
|
||||
Asked time.Time `json:"asked"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
}
|
||||
|
||||
// TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream.
|
||||
const TryAsked = "asked"
|
||||
|
||||
// TryStep is one ask of a try as its step key last said it.
|
||||
type TryStep struct {
|
||||
Ask string `json:"ask"`
|
||||
Node string `json:"node,omitempty"`
|
||||
State string `json:"state"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
PhaseAt time.Time `json:"phase-at,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
}
|
||||
|
||||
// tryWriter writes a running try's record again as its asks are put and their steps move: one writer per try, holding
|
||||
// the revision it wrote last, so the try's end is written at that revision. A write refused (the try ended by hand)
|
||||
// stops the writes; the end's own write then says it.
|
||||
type tryWriter struct {
|
||||
mu sync.Mutex
|
||||
tr *Tryer
|
||||
ctx context.Context
|
||||
key string
|
||||
t Try
|
||||
rev uint64
|
||||
stopped bool
|
||||
}
|
||||
|
||||
// heard takes an ask put (step nil) or a value of its step key in, and writes the record when it changed.
|
||||
func (w *tryWriter) heard(askID string, step *delivery.Step) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped {
|
||||
return
|
||||
}
|
||||
changed := false
|
||||
if !slices.Contains(w.t.Asks, askID) {
|
||||
w.t.Asks, changed = append(w.t.Asks, askID), true
|
||||
}
|
||||
i := slices.IndexFunc(w.t.Steps, func(s TryStep) bool { return s.Ask == askID })
|
||||
if i < 0 {
|
||||
w.t.Steps, changed = append(w.t.Steps, TryStep{Ask: askID, State: TryAsked}), true
|
||||
i = len(w.t.Steps) - 1
|
||||
}
|
||||
if step != nil {
|
||||
now := TryStep{Ask: askID, Node: step.Node, State: step.State, Phase: step.Phase, Started: step.Started,
|
||||
PhaseAt: step.PhaseAt, Ended: step.Ended}
|
||||
changed = changed || now != w.t.Steps[i]
|
||||
w.t.Steps[i] = now
|
||||
}
|
||||
if changed {
|
||||
w.write()
|
||||
}
|
||||
}
|
||||
|
||||
// write writes the record at the revision written last; a refusal stops the writes.
|
||||
func (w *tryWriter) write() {
|
||||
data, err := encodeTry(w.t)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
rev, err := w.tr.Tries.Update(w.ctx, w.key, data, w.rev)
|
||||
if err != nil {
|
||||
w.stopped = true
|
||||
w.tr.logf("%s: its asks' progress is no longer written: %v", w.key, err)
|
||||
return
|
||||
}
|
||||
w.rev = rev
|
||||
}
|
||||
|
||||
// end stops the writes and answers the record and its revision, for the try's end.
|
||||
func (w *tryWriter) end() (Try, uint64) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.stopped = true
|
||||
return w.t, w.rev
|
||||
}
|
||||
|
||||
// HoldsANode says whether the try holds a build node now: it runs, and a step of its asks runs on a node, or it names
|
||||
// no step (a checker that writes its asks only at the end, or one about to put its first).
|
||||
func (t Try) HoldsANode() bool {
|
||||
if t.State != TryRunning {
|
||||
return false
|
||||
}
|
||||
if len(t.Steps) == 0 {
|
||||
return true
|
||||
}
|
||||
return slices.ContainsFunc(t.Steps, func(s TryStep) bool { return s.State == delivery.StepRunning })
|
||||
Asked time.Time `json:"asked"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
}
|
||||
|
||||
// Over says whether the try came to its end.
|
||||
@@ -222,8 +117,8 @@ func NotChecked(repository string) error {
|
||||
}
|
||||
|
||||
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
|
||||
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the
|
||||
// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
|
||||
// ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a
|
||||
// repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
|
||||
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
|
||||
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
|
||||
head := strings.ToLower(strings.TrimSpace(a.Head))
|
||||
@@ -262,50 +157,24 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the tries cannot be listed: %w", err)
|
||||
}
|
||||
// An ended try's key stays TryKeep, so the bucket holds thousands; one that waits or runs was asked lately. Only the
|
||||
// newest TryScan keys are read, in parallel (a try id is the time in nanoseconds, so the newest sorts last).
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(keys[i]) != len(keys[j]) {
|
||||
return len(keys[i]) > len(keys[j])
|
||||
}
|
||||
return keys[i] > keys[j]
|
||||
})
|
||||
if len(keys) > TryScan {
|
||||
keys = keys[:TryScan]
|
||||
}
|
||||
tries := make([]Try, len(keys))
|
||||
live := make([]bool, len(keys))
|
||||
if err := parallel(len(keys), func(i int) error {
|
||||
raw, err := store.Get(keys[i])
|
||||
sort.Strings(keys)
|
||||
var open []string
|
||||
for _, k := range keys {
|
||||
raw, err := store.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
cur, err := DecodeTry(raw)
|
||||
tries[i], live[i] = cur, err == nil && !cur.Over()
|
||||
return nil
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var holding, waiting []string
|
||||
for i, cur := range tries {
|
||||
if !live[i] {
|
||||
if err != nil || cur.Over() {
|
||||
continue
|
||||
}
|
||||
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
|
||||
return cur.ID, nil
|
||||
}
|
||||
said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)
|
||||
if cur.HoldsANode() {
|
||||
holding = append(holding, said)
|
||||
} else {
|
||||
waiting = append(waiting, said)
|
||||
}
|
||||
open = append(open, fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State))
|
||||
}
|
||||
// Checked at this ask only: a try queued behind the seat's checks holds no node; tries let in while waiting may later
|
||||
// all run at once, and each node's build capacity bounds what runs on it.
|
||||
if len(holding) >= MaxTries || len(waiting) >= MaxTriesWaiting {
|
||||
return "", fmt.Errorf("a new try is refused while %d tries run on a build node or %d wait for one; now %d run and %d wait: running %s; waiting %s; ask again when one ends",
|
||||
MaxTries, MaxTriesWaiting, len(holding), len(waiting), orNone(holding), orNone(waiting))
|
||||
if len(open) >= MaxTries {
|
||||
return "", fmt.Errorf("%d tries wait or run, the most at once: %s; ask again when one ends", len(open), strings.Join(open, ", "))
|
||||
}
|
||||
t.ID = "try-" + strconv.FormatInt(now.UnixNano(), 10)
|
||||
data, err := encodeTry(t)
|
||||
@@ -333,52 +202,6 @@ func ReadTry(store TryStore, id string) (json.RawMessage, error) {
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// TryEnder is the tries state as the end tool sees it: one key with its revision, and a write at that revision.
|
||||
type TryEnder interface {
|
||||
Entry(key string) (json.RawMessage, uint64, error)
|
||||
Update(key string, value []byte, revision uint64) error
|
||||
}
|
||||
|
||||
// EndTry ends a try that waits or runs as error, saying why and that it was ended by hand (issue 734): its place is
|
||||
// free at once, and the next checker_try of its head asks anew. A try that ended already is refused, naming its end.
|
||||
func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
|
||||
if !tryKeyOK.MatchString(id) {
|
||||
return Try{}, fmt.Errorf("%q is not a try's id, try-<digits>", id)
|
||||
}
|
||||
why = strings.TrimSpace(why)
|
||||
if why == "" {
|
||||
return Try{}, errors.New("say why the try is ended: the why is kept in what it says")
|
||||
}
|
||||
raw, rev, err := store.Entry(id)
|
||||
if err != nil {
|
||||
return Try{}, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return Try{}, fmt.Errorf("no try %s: it was never asked, or it ended more than %s ago and was removed", id, TryKeep)
|
||||
}
|
||||
t, err := DecodeTry(raw)
|
||||
if err != nil {
|
||||
return Try{}, err
|
||||
}
|
||||
if t.Over() {
|
||||
return Try{}, fmt.Errorf("%s ended already, %s at %s", id, t.State, t.Ended.Format(time.RFC3339))
|
||||
}
|
||||
t.State, t.Ended = TryError, now.UTC()
|
||||
if !t.Started.IsZero() {
|
||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||
}
|
||||
t.Said = strings.TrimSpace("ended by hand (" + TryEndTool + "): " + why + " | " + t.Said)
|
||||
t.Said = strings.TrimSuffix(t.Said, " |")
|
||||
data, err := encodeTry(t)
|
||||
if err != nil {
|
||||
return Try{}, err
|
||||
}
|
||||
if err := store.Update(id, data, rev); err != nil {
|
||||
return Try{}, fmt.Errorf("%s changed while it was ended, read it and end it again: %w", id, err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// TryOptions narrow what a try runs; a pull request's check has none.
|
||||
type TryOptions struct {
|
||||
Level string
|
||||
@@ -391,14 +214,12 @@ type Tryer struct {
|
||||
Tries KV
|
||||
Judge Judge
|
||||
Bound time.Duration
|
||||
// NotTaken ends a try whose ask no build node took within it as error (issue 734); none when 0.
|
||||
NotTaken time.Duration
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
inflight map[string]context.CancelFunc
|
||||
inflight map[string]bool
|
||||
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
|
||||
}
|
||||
|
||||
@@ -426,31 +247,25 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
|
||||
tr.logf("tries.%s holds what is not a try, left: %v", key, err)
|
||||
return
|
||||
}
|
||||
if t.Over() {
|
||||
return
|
||||
}
|
||||
tr.mu.Lock()
|
||||
if tr.inflight == nil {
|
||||
tr.inflight, tr.seen = map[string]context.CancelFunc{}, map[string]bool{}
|
||||
tr.inflight, tr.seen = map[string]bool{}, map[string]bool{}
|
||||
}
|
||||
if t.Over() {
|
||||
// Ended while it ran (checker_try_end): its judging stops, and its asks are no longer waited for.
|
||||
if stop := tr.inflight[key]; stop != nil {
|
||||
stop()
|
||||
}
|
||||
if tr.inflight[key] || (t.State == TryRunning && tr.seen[key]) {
|
||||
tr.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
|
||||
tr.mu.Unlock()
|
||||
return
|
||||
}
|
||||
run, stop := context.WithCancel(ctx)
|
||||
tr.seen[key] = true
|
||||
tr.inflight[key] = stop
|
||||
tr.inflight[key] = true
|
||||
tr.mu.Unlock()
|
||||
tr.wg.Add(1)
|
||||
go func() {
|
||||
defer tr.wg.Done()
|
||||
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock(); stop() }()
|
||||
tr.run(run, key)
|
||||
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock() }()
|
||||
tr.run(ctx, key)
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -501,37 +316,27 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
}
|
||||
return
|
||||
}
|
||||
progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev}
|
||||
cs := &Case{Key: key, Group: key, Watch: progress.heard,
|
||||
cs := &Case{Key: key, Group: key,
|
||||
Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}},
|
||||
Trunk: map[string]string{t.Repository: t.Trunk},
|
||||
Queued: t.Asked,
|
||||
Progress: &Progress{},
|
||||
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
|
||||
judging := ctx
|
||||
if tr.NotTaken > 0 {
|
||||
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
|
||||
}
|
||||
out, judged := judgeWithin(judging, tr.Judge, cs, tr.Bound)
|
||||
out, judged := judgeWithin(ctx, tr.Judge, cs, tr.Bound)
|
||||
if ctx.Err() != nil {
|
||||
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
|
||||
return // left running: the checker takes it over when it starts
|
||||
}
|
||||
t, rev = progress.end()
|
||||
v := out.Verdict
|
||||
t.Ended = tr.now().UTC()
|
||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
|
||||
t.Levels, t.Machines = v.Levels, v.Machines
|
||||
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
||||
for _, n := range cs.Notes() {
|
||||
t.Said += " | " + n
|
||||
}
|
||||
switch {
|
||||
case !judged || IsNodesFailure(v) || IsLostAsk(v) || IsNotTaken(v):
|
||||
// Not the change's: its ask was given up or never taken (issue 734), or a part of the mesh was not heard.
|
||||
case !judged || IsNodesFailure(v):
|
||||
t.State = TryError
|
||||
if IsLostAsk(v) || IsNotTaken(v) {
|
||||
t.Failed = []string{v.Failed.Words}
|
||||
}
|
||||
case v.State == delivery.VerdictPass:
|
||||
t.State = TryPass
|
||||
default:
|
||||
@@ -578,11 +383,3 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// orNone joins the tries said, or says none.
|
||||
func orNone(said []string) string {
|
||||
if len(said) == 0 {
|
||||
return "none"
|
||||
}
|
||||
return strings.Join(said, ", ")
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user