diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 07368f8a..b87acaac 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -60,6 +60,8 @@ var readBuckets = []string{ delivery.PresenceBucket, delivery.HaveBucket, provisions.ProvisionsBucket, + delivery.SourceBucket, + delivery.MergesBucket, } // readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every @@ -140,6 +142,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err) continue } + if (name == delivery.SourceBucket || name == delivery.MergesBucket) && err != nil { + // The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of + // a record module is asked of a person, as before. + logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err) + continue + } if name == delivery.PartGrantsBucket && err != nil { // The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound. logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err) @@ -158,7 +166,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any func startsWithout(name string) bool { switch name { case delivery.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket, - delivery.AssignerPutsBucket: + delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket: return true } return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket @@ -241,6 +249,8 @@ var watched = []string{ assign.PoliciesBucket, provisions.ProvisionsBucket, delivery.SeatHoldersBucket, + delivery.SourceBucket, + delivery.MergesBucket, } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after @@ -336,8 +346,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started++ var opts []jetstream.WatchOpt - if name == delivery.HaveBucket { - opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back + if name == delivery.HaveBucket || name == delivery.SourceBucket { + // what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1) + opts = append(opts, jetstream.IncludeHistory()) } if err := watch(name, opts...); err != nil { return err diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index e3726890..1992c412 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -174,7 +174,8 @@ func (p *passer) wants(ctx context.Context) { Allowed: func(node, digest string) bool { key, err := delivery.RecordAskKey(node, name, digest) return err == nil && p.answeredWith(key, assign.OptionYes) - }} + }, + Release: func(fp string) (string, bool) { return p.w.release(name, fp) }} if fp != "" { held, _ := delivery.HeldKey(name, fp) grant, _ := delivery.PartGrantKey(name, fp) @@ -819,7 +820,9 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d // yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a // person must allow (ADR 0547 §3): for a have of phase // waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes -// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. +// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. A +// move of the build alone (the have says so) to a build an approved release carries is given its yes by that Approve, the +// want naming the release, with no ask (ADR 0601 §1). func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) { for _, node := range m.Nodes { if m.Self == nil || !m.Self(node) { @@ -836,25 +839,37 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo if w.Removed { continue } - yes := "" + yes, release := "", "" if had && was.FP == w.FP { - yes = was.Yes + yes, release = was.Yes, was.Release } key, err := delivery.HaveKey(node, module) if err != nil { continue } if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes { - if m.Allowed != nil && m.Allowed(node, h.Waits) { - yes = h.Waits - } else { + switch { + case m.Allowed != nil && m.Allowed(node, h.Waits): + yes, release = h.Waits, "" + case h.BuildAlone && h.FP != w.FP && m.Release != nil: + // A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1), + // named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked. + if r, ok := m.Release(w.FP); ok { + yes, release = h.Waits, r + break + } + p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail}) + default: p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail}) } } // A want carried over from before the node composed its own may still name the controller's composition; a placed // node's want names the build alone (ADR 0547 §2). - if yes != w.Yes || w.Composed != "" || !w.Self { - w.Yes, w.Composed, w.Self = yes, "", true + if yes == "" { + release = "" + } + if yes != w.Yes || release != w.Release || w.Composed != "" || !w.Self { + w.Yes, w.Release, w.Composed, w.Self = yes, release, "", true out.Wants[node] = w } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index a527a890..dd4c767f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -788,6 +788,62 @@ func mustKey(key string, err error) string { func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) { // A placed node's yes comes from its have (issue 1126): folded here to hold the count. aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t) + aReleasesApproveAnswersTheMoveOfItsBuildAlone(t) +} + +// ADR 0601 §1: a placed node's have waiting for a yes to a move of the build alone, to a build an approved release carries, is +// given its yes by that Approve, the want naming the release, and no record ask is raised; a move that changes a setting or a +// secret too, one at the build it runs, and one no approved release carries are asked of a person as before. +func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) { + fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64) + waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64) + commit := strings.Repeat("1", 40) + for _, c := range []struct { + name string + alone bool + at string // the fingerprint the have runs + mergedAt string // the commit the release merged web at; "" for no merge of the release + yes, asked bool + }{ + {"the build alone, its release approved", true, fpOld, commit, true, false}, + {"a setting changed too", false, fpOld, commit, false, true}, + {"a setting changed at the build it runs", true, fpTest, commit, false, true}, + {"no approved release carries it", true, fpOld, "", false, true}, + {"the release merged another commit", true, fpOld, strings.Repeat("2", 40), false, true}, + } { + asksKV := &memKV{values: map[string][]byte{}} + p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV}, time.Now()) + p.logf = t.Logf + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, Path: delivery.RootPath}) + p.w.put(delivery.ModulesBucket, "web", mod) + src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/web", Heads: map[string]string{"novox/web": commit}}) + p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, src, 7, time.Now()) + later, _ := delivery.EncodeSource(delivery.Source{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}}) + p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release + if c.mergedAt != "" { + merge, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []delivery.MergePull{ + {Head: delivery.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)}, State: delivery.MergeMerged, Commit: c.mergedAt}}}) + if err != nil { + t.Fatal(err) + } + p.w.put(delivery.MergesBucket, "feat/web", merge) + } + have, _ := delivery.EncodeHave(delivery.Have{FP: c.at, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK, + Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone}) + p.w.put(delivery.HaveBucket, "g14.web", have) + m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true }, + Release: func(fp string) (string, bool) { return p.w.release("web", fp) }} + out := assign.Outcome{Wants: map[string]delivery.Want{"g14": {FP: fpTest, Self: true}}} + p.yeses(context.Background(), "web", m, map[string]delivery.Want{"g14": {FP: fpOld, Self: true}}, &out) + w := out.Wants["g14"] + if got := w.Yes == waiting && w.Release == "feat/web"; got != c.yes || (!c.yes && (w.Yes != "" || w.Release != "")) { + t.Errorf("%s: the want is %+v", c.name, w) + } + akey, _ := delivery.RecordAskKey("g14", "web", waiting) + if _, there := asksKV.values[akey]; there != c.asked { + t.Errorf("%s: a record ask raised %v, want %v", c.name, there, c.asked) + } + } } // 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 2277786d..751b6c2c 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -37,6 +37,9 @@ type world struct { // placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk // delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner. placement *delivery.PlacementKeys + // sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its + // history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1). + sourceGroups map[uint64]string } // entryMeta is a value's revision in its bucket and when it was written. @@ -54,7 +57,8 @@ type parsed struct { func newWorld() *world { return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}, - meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys()} + meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys(), + sourceGroups: map[uint64]string{}} } // putEntry is put, keeping the value's revision and the time it was written. @@ -70,6 +74,43 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t if bucket == delivery.PlacementBucket { _, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision } + if bucket == delivery.SourceBucket && key == delivery.SourceMainKey { + if s, err := delivery.DecodeSource(value); err == nil && s.Group != "" { + w.sourceGroups[revision] = s.Group + } + } +} + +// release is the release whose Approve carries module's build fp (ADR 0601 §1), and whether one does: the trunk's value of +// the module names fp, the source revision it was first merged at names the release that moved it, and the gate's merge of +// that release, which it writes on the operator's Approve, holds the module's own trunk merge among its merged pull +// requests. Any of them unknown: none, and the move is asked of a person. +func (w *world) release(module, fp string) (string, bool) { + t, ok := w.trunk()[module] + if !ok || fp == "" || t.FP2 != fp || t.Src == 0 || t.Commit == "" { + return "", false + } + w.mu.Lock() + group := w.sourceGroups[t.Src] + w.mu.Unlock() + key, err := delivery.MergesKey(group) + if group == "" || err != nil { + return "", false + } + raw := w.value(delivery.MergesBucket, key) + if raw == nil { + return "", false + } + merge, err := delivery.DecodeMerge(raw) + if err != nil { + return "", false + } + for _, pull := range merge.Pulls { + if pull.State == delivery.MergeMerged && pull.Commit == t.Commit && delivery.SameRepository(pull.Repository, t.Repository) { + return group, true + } + } + return "", false } // metaOf is the revision and time of the value a bucket holds at key, as the watch said them. diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 629014e9..ab020fca 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -45,6 +45,10 @@ type Module struct { // Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition // with this digest (ADR 0539 §14). Nil: none did. Allowed func(node, digest string) bool + // Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads, + // read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no + // approved release is known to carry it. Nil: none is read. + Release func(fp string) (string, bool) // Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed. Released bool // Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index d5e14e06..c594947e 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -34,8 +34,8 @@ func TestManifest(t *testing.T) { reads := slices.Clone(m.Reads) slices.Sort(reads) // ADR 0593 §8: it composes nothing, so it reads no candidate, held or resolved value, compose step or build log; it reads - // every node's published compositions to compare them. - want := []string{"mesh-delivery-builder.builds", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + // every node's published compositions to compare them, and the gate's source and merges, for the release that carries a build (ADR 0601 §1). + want := []string{"mesh-delivery-builder.builds", "mesh-delivery-gate.merges", "mesh-delivery-gate.source", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed", "mesh-provisioner.provisions", "node-engine.have"} if !reflect.DeepEqual(reads, want) { diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 3c7deb4b..86ee65ec 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -25,6 +25,8 @@ ], "reads": [ "mesh-delivery-builder.builds", + "mesh-delivery-gate.merges", + "mesh-delivery-gate.source", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers",