diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go index 98c0465d..8d7cbb3b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go @@ -1,26 +1,25 @@ package main import ( + "context" "encoding/json" "fmt" "sync" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "mesh-delivery-assigner/internal/assign" + "mesh-delivery-assigner/internal/assignment" ) // What a want names of a build is heard, never read from the builder's state (design 47 G §379, §526, A-94, A-96; ADR 0596 §2): the // builder's event `build.ended` (a done build's parts by digest, its keeping node and its branch) and the catalog's // `version-released` (a released version and each part's digest, with the source revision it was built from), each kept as heard. -// buildEndedSubjects is the builder's event build.ended on its own stream, as the manifest's consumes names it -// (`mesh-delivery-builder.build.ended`), each module and fingerprint after it. -var buildEndedSubjects = process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>") - // builtEnds are the done builds' ends heard, by module and fingerprint (hearBuildEnded). type builtEnds struct { mu sync.Mutex @@ -38,6 +37,9 @@ func (b *builtEnds) put(data []byte) error { if e.Module == "" || e.Fingerprint == "" { return fmt.Errorf("a build's end naming no module or fingerprint: %s", data) } + if e.Outcome == "" { // an end said before A-162 is a done build's + e.Outcome = builds.BuildDone + } b.mu.Lock() defer b.mu.Unlock() if b.ends == nil { @@ -47,6 +49,14 @@ func (b *builtEnds) put(data []byte) error { return nil } +// endOf is the end heard of module's build at fp, done or failed (A-162): none while the build has not ended. +func (b *builtEnds) endOf(module, fp string) (builds.BuildEnded, bool) { + b.mu.Lock() + defer b.mu.Unlock() + e, ok := b.ends[b.key(module, fp)] + return e, ok +} + // partsOf is the parts of module's build at fp as its end names them (each artifact's digest by `:`), with the // node that keeps it, for a kept build or a build of another watched branch; false for a build of the trunk not kept, whose // parts the version's release says (releasedVersions). @@ -68,6 +78,8 @@ func (b *builtEnds) partsOf(module, fp string) (assign.KeptBuild, bool) { type releasedVersions struct { mu sync.Mutex seen map[string]assign.KeptBuild + // newest is, by module, the last version-released heard of it (delivered.go). + newest map[string]releaseHeard } func (r *releasedVersions) key(module string, source int64) string { @@ -93,6 +105,14 @@ func (r *releasedVersions) put(data []byte) error { r.seen = map[string]assign.KeptBuild{} } r.seen[r.key(e.Module, e.Source)] = assign.KeptBuild{Version: e.Version, Parts: parts} + if r.newest == nil { + r.newest = map[string]releaseHeard{} + } + h := releaseHeard{Version: e.Version, Parts: parts, Source: e.Source} + if e.At != nil { + h.At = *e.At + } + r.newest[e.Module] = h // the stream's order is the order of release return nil } @@ -104,21 +124,33 @@ func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bo return k, ok } -// hearBuildsAndReleases registers the hub's takers of build.ended and version-released, each waking the pass; before the hub starts. -func hearBuildsAndReleases(hub *process.EventHub, ends *builtEnds, released *releasedVersions, wake func(), logf func(string, ...any)) error { - taker := func(put func([]byte) error) func(jetstream.Msg) { - return func(m jetstream.Msg) { - if err := put(m.Data()); err != nil { - logf("%s is not kept: %v", m.Subject(), err) - } - if err := m.Ack(); err != nil { - logf("acknowledging %s: %v", m.Subject(), err) - } - wake() +// buildsStreamName is the builder's stream of build ends as the bus names it: kept last per subject (one end per module and +// fingerprint, its manifest's `last-per-subject`, design 47 G A-181, §703). +var buildsStreamName = catalogue.StreamBusName(builds.BuildsStream) + +// hearBuilds reads the builder's stream of build ends whole at the start and then follows it (design 47 G A-181, §556): what was +// built, and a failed build's error, survive the assigner's restart, and it reads none of the builder's buckets. The read ends before +// it returns, so the first pass knows every build; a stream it cannot hear is said, and no want names a build until the assigner +// starts again. +func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wake func(), findings assign.Findings, logf func(string, ...any)) { + acct := process.Account{Node: thisNode, Module: assignment.Module} + nh := notHeardWriter(findings, logf) + cc, err := acct.HearStream(ctx, js, buildsStreamName, func(m jetstream.Msg) { + if err := ends.put(m.Data()); err != nil { + logf("%s is not kept: %v", m.Subject(), err) } + if err := m.Ack(); err != nil { + logf("acknowledging %s: %v", m.Subject(), err) + } + wake() + }, logf) + if err != nil { + nh.NotHeard(buildsStreamName, "what the builder built is not heard until the assigner starts again, so no want names a build's parts or its keeping node and no put goes on", err) + return } - if err := hub.Handle(buildEndedSubjects, taker(ends.put)); err != nil { - return err - } - return hub.Handle(meshcatalog.SubjectVersionReleased, taker(released.put)) + nh.Heard(buildsStreamName) + go func() { + <-ctx.Done() + cc.Stop() + }() } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go index d73743e2..7a86f9b3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go @@ -86,3 +86,25 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *tes t.Error("no revision kept names a release") } } + +// A-162: the assigner knows a failed build from its `build.ended`: the error whole; a done end or none heard is not failed. +func TestTheStandingOfABuildIsHeardFromItsEnd(t *testing.T) { + w := newWorld() + say := func(e builds.BuildEnded) { + raw, _ := json.Marshal(e) + if err := w.ends.put(raw); err != nil { + t.Fatal(err) + } + } + if failed, _ := w.buildStanding("web", "fp1"); failed { + t.Error("a build nothing was heard of failed") + } + say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildFailed, Error: "transport fault: connection"}) + if failed, why := w.buildStanding("web", "fp1"); !failed || why != "transport fault: connection" { + t.Errorf("failed end: failed %v why %q", failed, why) + } + say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildDone}) + if failed, _ := w.buildStanding("web", "fp1"); failed { + t.Error("a later done end leaves the build failed") + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildends_helper_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildends_helper_test.go new file mode 100644 index 00000000..ce5afcda --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildends_helper_test.go @@ -0,0 +1,24 @@ +package main + +import ( + "encoding/json" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// endDone is the builder's `build.ended` of a done build of module at fp heard by w, naming the digest of its manifest (design 47 G +// A-181: what was built is heard, never read from the builder's bucket). +func endDone(t testing.TB, w *world, module, fp, manifest string) { + t.Helper() + data, err := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: builds.BuildDone, Manifest: manifest}) + if err != nil { + t.Fatal(err) + } + if err := w.ends.put(data); err != nil { + t.Fatal(err) + } +} + +// trunkBranch is the first watched branch the tests give the world: the code names none (design 47 G §507). +const trunkBranch = "main" diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/candidates.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/candidates.go new file mode 100644 index 00000000..c19cfb45 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/candidates.go @@ -0,0 +1,113 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "strings" + + mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + + "mesh-delivery-assigner/internal/assigngraph" +) + +// A run's candidate assignment graph (design 47 G §189, §190, A-201, A-235): the candidate `run-`'s catalog graph revision, +// handed by meshcatalogclient's candidates hook, is read whole through the catalog seat's verb revision; the assignment graph is +// computed from it in place of its branch's half, beside every other followed branch's, and kept under candidate..; +// the runner's group-ended of the run drops it. It writes no want and says no event; the gate's heads-replaced drops nothing. + +// runGroupPrefix is a run on request's candidate group before its id, `run-` (A-160). +const runGroupPrefix = "run-" + +// readCandidateHalf reads a candidate's catalog graph revision whole through the seat's verb revision, page by page, with the +// candidate's group and heads and the revision its event names. +func readCandidateHalf(call mcc.Call, ev meshcatalog.CatalogRevisionEvent) (mcc.Half, error) { + half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}} + after := "" + for { + a, err := askCandidatePage(call, meshcatalog.RevisionInput{Branch: ev.Branch, Group: ev.Group, Heads: ev.Heads, Revision: ev.Revision, After: after}) + if err != nil { + return half, err + } + if a.Revision == nil || a.Revision.Revision != ev.Revision || a.Revision.Group != ev.Group { + return half, fmt.Errorf("%s answered %+v for the candidate %s's revision %d", mcc.RevisionVerb, a.Revision, ev.Group, ev.Revision) + } + half.Revision = *a.Revision + for _, e := range a.Entries { + half.Entries[e.Module] = e + } + if a.Next == "" { + return half, nil + } + after = a.Next + } +} + +// keepCandidate computes and keeps the candidate's assignment graph of a run's catalog-revision; another candidate's (a try's, a +// group's) is passed over. +func (p *passer) keepCandidate(ctx context.Context, call mcc.Call, ev meshcatalog.CatalogRevisionEvent) error { + if !strings.HasPrefix(ev.Group, runGroupPrefix) { + return nil + } + g := p.graph + if g == nil || g.objects == nil || g.halves == nil { + return errors.New("the assigner keeps no assignment graph, so no candidate's is kept") + } + half, err := readCandidateHalf(call, ev) + if err != nil { + return fmt.Errorf("the candidate %s's revision %d is not read, so its assignment graph is not kept: %w", ev.Group, ev.Revision, err) + } + in := p.graphInputs() + in.Halves = maps.Clone(in.Halves) + in.Halves[half.Revision.Branch] = half + c := assigngraph.Compute(in) + for _, u := range c.Uncomputed { + p.logf("the candidate %s's assignment graph leaves out %s", ev.Group, u) + } + rev, fresh, err := assigngraph.KeepCandidate(ctx, g.objects, c, ev.Group, ev.Heads, meshcatalog.EventCatalogRevision, p.clock()) + if err != nil { + return fmt.Errorf("the candidate %s's assignment graph is not kept: %w", ev.Group, err) + } + if fresh { + p.logf("the candidate %s's assignment graph revision %d is kept, from its catalog graph revision %d", ev.Group, rev.Revision, ev.Revision) + } + return nil +} + +// dropCandidate removes a run's candidate assignment graph on the runner's group-ended of that run (A-235). +func (p *passer) dropCandidate(ctx context.Context, group string) error { + g := p.graph + if g == nil || g.objects == nil { + return nil + } + o, ok := g.objects.(assigngraph.Remover) + if !ok { + return fmt.Errorf("the objects %s remove nothing, so the candidate %s is kept", assigngraph.RevisionsObjects, group) + } + n, err := assigngraph.DropCandidate(ctx, o, group) + if err == nil && n > 0 { + p.logf("the candidate %s's assignment graph is dropped: its run ended", group) + } + return err +} + +// askCandidatePage is one page of a candidate's revision as the seat's verb revision answers it. +func askCandidatePage(call mcc.Call, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) { + raw, err := call(mcc.RevisionVerb, in) + if err != nil { + return meshcatalog.RevisionAnswer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return meshcatalog.RevisionAnswer{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said) + } + var a meshcatalog.RevisionAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return a, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err) + } + return a, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/delivered.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/delivered.go new file mode 100644 index 00000000..8e861022 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/delivered.go @@ -0,0 +1,451 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-assigner/internal/assign" + "mesh-delivery-assigner/internal/assignment" + "mesh-delivery-assigner/internal/buckets" +) + +// A delivery is waited for mesh-wide (design 47 G §376, §389, A-189, A-246). At its start the assigner reads whole, from their first +// message, the streams of the catalog seat's version-released and of its own version-delivered (mesh-sdk process.FromFirst, A-233), +// and the watchman's machine-away and machine-back, never its state; then it hears them. A released version of a module is said +// version-delivered on the seat mesh-wants once, when every present machine placing it on the branch it is released from runs, healthy, +// the want the assigner wrote naming that version; a version no present machine places is not delivered. A dependent whose want +// changes fingerprint waits, by an edge of kind 2, 4 or 6, for its dependency's newest released version not yet said delivered, on +// the branch it is released from; on another branch no version is released and nothing waits. The wait is said wait-held, step +// delivery, bound 60 minutes from the version's release, kept in the state delivery-waits under the dependent's name, and said +// wait-ended when the version is delivered (ran) or released by hand (assigner_release, released). + +// deliveryWaitBound is a dependent's delivery wait's bound, from its dependency's version-released. +const deliveryWaitBound = 60 * time.Minute + +// Wait-ended's how of a delivery wait. +const ( + deliveryWaitRan = "ran" + deliveryWaitReleased = "released" +) + +// releaseHeard is one version-released heard: the version, each part's digest, the source revision and when it was said. +type releaseHeard struct { + Version string + Parts map[string]string + Source int64 + At time.Time +} + +// newestOf is the newest version of module released, the last heard of it in the stream's order. +func (r *releasedVersions) newestOf(module string) (releaseHeard, bool) { + r.mu.Lock() + defer r.mu.Unlock() + h, ok := r.newest[module] + return h, ok +} + +// deliveredVersions are the version-delivered said, the assigner's own, by module and version: each is said once. +type deliveredVersions struct { + mu sync.Mutex + said map[string]bool +} + +func (d *deliveredVersions) key(module, version string) string { return module + "@" + version } + +// put takes one version-delivered in. +func (d *deliveredVersions) put(data []byte) error { + var e meshwants.VersionDeliveredEvent + if err := json.Unmarshal(data, &e); err != nil { + return fmt.Errorf("a version-delivered that is none: %w", err) + } + if e.Module == "" || e.Version == "" { + return fmt.Errorf("a version-delivered naming no module or version: %s", data) + } + d.mark(e.Module, e.Version) + return nil +} + +func (d *deliveredVersions) mark(module, version string) { + d.mu.Lock() + defer d.mu.Unlock() + if d.said == nil { + d.said = map[string]bool{} + } + d.said[d.key(module, version)] = true +} + +// has says whether version-delivered of module's version was said. +func (d *deliveredVersions) has(module, version string) bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.said[d.key(module, version)] +} + +// presenceOf reads one machine-away or machine-back: the machine and when it was said. +func presenceOf(data []byte) (string, time.Time, error) { + var e wd.MachineAwayEvent // machine-back has the same fields + if err := json.Unmarshal(data, &e); err != nil { + return "", time.Time{}, err + } + if e.Machine == "" { + return "", time.Time{}, fmt.Errorf("it names no machine: %s", data) + } + return e.Machine, e.At, nil +} + +// hearDelivery reads whole, then follows, the four streams a delivery is judged from: version-released, version-delivered, +// machine-away and machine-back. A stream not heard is said as the assigner's finding, and what it carries is not known until +// the assigner starts again. +func hearDelivery(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) { + acct := process.Account{Node: thisNode, Module: assignment.Module} + nh := notHeardWriter(findings, logf) + for _, s := range []struct { + stream, words string + take func([]byte) error + }{ + {meshcatalog.StreamVersionReleased, "no want names a released version and no delivery is judged", w.released.put}, + {meshwants.StreamVersionDelivered, "a version said delivered before is said again once it is judged", w.delivered.put}, + {wd.StreamMachineAway, "no machine is known away, so a delivery waits for a machine that is away", func(data []byte) error { + m, at, err := presenceOf(data) + if err == nil { + w.heardPresence(m, true, at) + } + return err + }}, + {wd.StreamMachineBack, "a machine once away is not known back", func(data []byte) error { + m, at, err := presenceOf(data) + if err == nil { + w.heardPresence(m, false, at) + } + return err + }}, + } { + take := s.take + cc, err := acct.HearSeatEvents(ctx, js, s.stream, func(m jetstream.Msg) { + if err := take(m.Data()); err != nil { + logf("%s is not kept: %v", m.Subject(), err) + } + if err := m.Ack(); err != nil { + logf("acknowledging %s: %v", m.Subject(), err) + } + wake() + }, logf, process.FromFirst()) + if err != nil { + nh.NotHeard(s.stream, s.words+" until the assigner starts again", err) + continue + } + nh.Heard(s.stream) + go func() { + <-ctx.Done() + cc.Stop() + }() + } +} + +// releaseBranch is the branch module's versions are released from: the first watched branch of its registration (§371). +func (p *passer) releaseBranch(module string) string { + if p.ends.following() { + return p.ends.branchOf(module, "") + } + return p.w.trunkName() +} + +// placing is every machine placing module on the branch its versions are released from, sorted. +func (p *passer) placing(module string) []string { + branch := p.releaseBranch(module) + var out []string + for node, placed := range p.w.placedBranches()[module] { + if p.ends.following() { + placed = p.ends.branchOf(module, placed) + } else { + placed = p.w.trunkName() // the trunk-only form: every placement runs the trunk's + } + if placed == branch { + out = append(out, node) + } + } + sort.Strings(out) + return out +} + +// deliveryStanding is where module's version stands: the present placing machines not yet running it healthy as its want names +// it, and the placing machines away. +func (p *passer) deliveryStanding(module, version string) (notRunning, away []string, placing int) { + gone := p.w.awayNow() + n := assign.Nodes{Running: p.w.runningNow()} + wanted := p.w.wantsOf()[module] + nodes := p.placing(module) + for _, node := range nodes { + if gone[node] { + away = append(away, node) + continue + } + want, ok := wanted[node] + if !ok || want.Removed || want.Version != version || !n.RunsHealthy(node, module, want.Fp) { + notRunning = append(notRunning, node) + } + } + return notRunning, away, len(nodes) +} + +// judgeDelivered says version-delivered of each module's newest released version not said yet, once every present machine placing +// it runs it healthy; judged again at each pass, which each running, machine-away, machine-back and placement change wakes. +func (p *passer) judgeDelivered() { + for _, module := range p.releasedModules() { + rel, _ := p.w.released.newestOf(module) + if p.w.delivered.has(module, rel.Version) { + continue + } + notRunning, away, placing := p.deliveryStanding(module, rel.Version) + if placing == 0 || len(away) == placing || len(notRunning) > 0 { + continue + } + if err := p.sayDelivered(module, rel, away); err != nil { + p.logf("version-delivered of %s %s is not said, and is judged again at the next pass: %v", module, rel.Version, err) + } + } +} + +// sayDelivered says version-delivered of module's release rel, with the machines away, and keeps it said. +func (p *passer) sayDelivered(module string, rel releaseHeard, away []string) error { + at := p.clock().UTC() + body := meshwants.VersionDeliveredEvent{V: keys.Version, Module: module, Version: rel.Version, Parts: rel.Parts, Away: away, At: &at} + if err := p.wantsSay(meshwants.EventVersionDelivered, body, fmt.Sprintf("%s.%s.%s.%s", meshwants.EventVersionDelivered, module, rel.Version, at.Format(time.RFC3339Nano))); err != nil { + return err + } + p.w.delivered.mark(module, rel.Version) + return nil +} + +// releasedModules is every module a version-released was heard of, sorted. +func (p *passer) releasedModules() []string { + r := &p.w.released + r.mu.Lock() + defer r.mu.Unlock() + out := make([]string, 0, len(r.newest)) + for m := range r.newest { + out = append(out, m) + } + sort.Strings(out) + return out +} + +// deliveryWait is what a dependent waits for: its dependency, the version and the machines not yet running it. +type deliveryWait struct { + dependency string + rel releaseHeard + notRunning []string + away []string +} + +// waitsForDelivery says whether module, whose want on its nodes changes to fp on branch, waits for a dependency's delivery: by +// each edge of kind 2, 4 or 6 into it in branch's catalog half, the dependency's newest released version on branch not yet said +// delivered. A want that changes no fingerprint waits for nothing; on a branch no version is released from, nothing waits. +func (p *passer) waitsForDelivery(module, fp, branch string, nodes []string) (deliveryWait, bool) { + wanted := p.w.wantsOf()[module] + changes := false + for _, node := range nodes { + if w, ok := wanted[node]; !ok || w.Removed || w.Fp != fp { + changes = true + } + } + if !changes || fp == "" { + return deliveryWait{}, false + } + var deps []string + for _, e := range p.w.edgesOf(branch, module) { + if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 { + continue + } + dep, _, _ := strings.Cut(e.From, ":") + if dep != "" && dep != module && !slices.Contains(deps, dep) { + deps = append(deps, dep) + } + } + sort.Strings(deps) + for _, dep := range deps { + if p.releaseBranch(dep) != branch { + continue + } + rel, ok := p.w.released.newestOf(dep) + if !ok || p.w.delivered.has(dep, rel.Version) { + continue + } + notRunning, away, _ := p.deliveryStanding(dep, rel.Version) + return deliveryWait{dependency: dep, rel: rel, notRunning: notRunning, away: away}, true + } + return deliveryWait{}, false +} + +// words is a delivery wait in plain words, as assigner_why_not and the status view say it. +func (d deliveryWait) words() string { + if len(d.notRunning) == 0 && len(d.away) > 0 { + return fmt.Sprintf("waiting for %s to be back", strings.Join(d.away, ", ")) + } + return fmt.Sprintf("waiting for delivery of %s %s on %s", d.dependency, d.rel.Version, strings.Join(d.notRunning, ", ")) +} + +// deliveryWaitBody is the wait-held of module's delivery wait d. +func deliveryWaitBody(module, fp string, d deliveryWait, since time.Time) meshwants.Wait { + from := d.rel.At.UTC() + if from.IsZero() { + from = since.UTC() + } + return meshwants.Wait{Step: meshwants.WaitStepDelivery, Owner: meshwants.Seat, Module: module, Version: fp, Part: d.dependency, + On: d.dependency + "@" + d.rel.Version, Machines: d.notRunning, Since: since.UTC(), ClockFrom: &from, Bound: deliveryWaitBound.String(), + Words: d.words()} +} + +// deliveryWaiting is one delivery wait the state delivery-waits keeps, by its dependent's name: the wait-held said, and a release +// by hand written by assigner_release. +type deliveryWaiting struct { + meshwants.Wait + // Released is set by assigner_release: the wait ends released and the dependent's want is written. + Released bool `json:"released,omitempty"` +} + +// saidDeliveryWaits is every delivery wait the state delivery-waits holds, by module; a value that does not decode is said and left. +func (p *passer) saidDeliveryWaits() map[string]deliveryWaiting { + out := map[string]deliveryWaiting{} + for key, value := range p.w.read(buckets.DeliveryWaits) { + var w deliveryWaiting + if err := json.Unmarshal(value, &w); err != nil || w.Module == "" { + p.logf("%s %s holds no delivery wait, left: %v", buckets.DeliveryWaits, key, err) + continue + } + out[w.Module] = w + } + return out +} + +// holdForDelivery says whether module's want, changing to fp on branch, is held for a dependency's delivery this pass: it says +// wait-held once and keeps it; a wait released by hand holds nothing; a wait that no longer holds is ended ran. +func (p *passer) holdForDelivery(ctx context.Context, module, fp, branch string, nodes []string) bool { + if p.kvs[buckets.DeliveryWaits] == nil { + return false + } + key := keys.EscapeName(module) + said, was := p.saidDeliveryWaits()[module] + d, waits := p.waitsForDelivery(module, fp, branch, nodes) + switch { + case was && said.Released: + p.endDeliveryWait(ctx, key, said, deliveryWaitReleased) // the want is written now, so it changes nothing at the next pass + return false + case !waits: + if was { + p.endDeliveryWait(ctx, key, said, deliveryWaitRan) + } + return false + } + body := deliveryWaitBody(module, fp, d, p.clock()) + if was && said.On == body.On && said.Version == fp && slices.Equal(said.Machines, body.Machines) { + return true + } + if was { + body.Since = said.Since + } + id := fmt.Sprintf("%s.delivery.%s.%s.%s", key, fp, body.On, strings.Join(body.Machines, ",")) + if err := p.wantsSay(meshwants.EventWaitHeld, body, id); err != nil { + p.logf("the delivery wait of %s is not said, and is said at the next pass: %v", module, err) + return true + } + kept := deliveryWaiting{Wait: body} + p.put(ctx, buckets.DeliveryWaits, key, func() ([]byte, error) { return json.Marshal(kept) }) + return true +} + +// endDeliveryWait says wait-ended of a delivery wait with how and removes it from the state. +func (p *passer) endDeliveryWait(ctx context.Context, key string, w deliveryWaiting, how string) { + body := w.Wait + body.How = how + if err := p.wantsSay(meshwants.EventWaitEnded, body, key+".delivery."+body.Version+"."+how); err != nil { + p.logf("the end of the delivery wait of %s is not said, and is said at the next pass: %v", w.Module, err) + return + } + p.remove(ctx, buckets.DeliveryWaits, key) +} + +// againAsk is one ask of assigner_delivered_again, kept in the state delivered-again under its module (A-250): the version to say +// delivered again, who asked and why, and, while it does not hold, why not. +type againAsk struct { + Module string `json:"module"` + Version string `json:"version"` + By string `json:"by"` + Why string `json:"why"` + Waits string `json:"waits,omitempty"` +} + +// releaseOf is the release heard of module's version. +func (r *releasedVersions) releaseOf(module, version string) (releaseHeard, bool) { + r.mu.Lock() + defer r.mu.Unlock() + if h, ok := r.newest[module]; ok && h.Version == version { + return h, true + } + for key, k := range r.seen { + if strings.HasPrefix(key, module+"@") && k.Version == version { + return releaseHeard{Version: version, Parts: k.Parts}, true + } + } + return releaseHeard{}, false +} + +// deliveredAgain takes each ask of assigner_delivered_again, heard at once through the watch of the state delivered-again: it says +// version-delivered again when every present machine placing the module runs that version healthy and removes the ask, else keeps +// it with why it does not hold, which assigner_waits reads. +func (p *passer) deliveredAgain(ctx context.Context) { + if p.kvs[buckets.DeliveredAgain] == nil { + return + } + for key, value := range p.w.read(buckets.DeliveredAgain) { + var a againAsk + if err := json.Unmarshal(value, &a); err != nil || a.Version == "" { + p.logf("%s %s holds no ask, left: %v", buckets.DeliveredAgain, key, err) + continue + } + module := a.Module + if module == "" { + module = key + } + waits := "" + rel, ok := p.w.released.releaseOf(module, a.Version) + notRunning, away, placing := p.deliveryStanding(module, a.Version) + switch { + case !ok: + waits = fmt.Sprintf("no version-released of %s %s is heard", module, a.Version) + case placing == 0: + waits = fmt.Sprintf("no machine places %s", module) + case len(away) == placing: + waits = fmt.Sprintf("every machine placing %s is away: %s", module, strings.Join(away, ", ")) + case len(notRunning) > 0: + waits = fmt.Sprintf("%s %s does not run healthy on %s", module, a.Version, strings.Join(notRunning, ", ")) + } + if waits == "" { + if err := p.sayDelivered(module, rel, away); err != nil { + p.logf("version-delivered of %s %s asked again is not said, and is asked at the next pass: %v", module, a.Version, err) + continue + } + p.logf("version-delivered of %s %s said again, asked by %s: %s", module, a.Version, a.By, a.Why) + p.remove(ctx, buckets.DeliveredAgain, key) + continue + } + if a.Waits != waits { + a.Waits = waits + p.put(ctx, buckets.DeliveredAgain, key, func() ([]byte, error) { return json.Marshal(a) }) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/delivered_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/delivered_test.go new file mode 100644 index 00000000..75364e8f --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/delivered_test.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + + "mesh-delivery-assigner/internal/buckets" +) + +// Design 47 G §376, §389, §498, A-189, A-246: a released version is said version-delivered once, when every present machine placing +// it runs healthy the want naming it, a machine away named and not waited for, none when every placing machine is away; a dependent +// whose want changes fingerprint waits by an edge of kind 2, 4 or 6 for its dependency's newest released version not yet said +// delivered, said wait-held (step delivery, 60 minutes from the release) once and kept in delivery-waits, and wait-ended ran once +// the version is delivered; a kind-1 edge, or a dependency with no version released, waits for nothing; one released by hand ends +// released and its want is written. +func TestADependentWaitsForItsDependencysDelivery(t *testing.T) { + ctx := context.Background() + at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) + waits := &memKV{values: map[string][]byte{}} + again := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Wants: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaits: waits, buckets.DeliveredAgain: again}, at) + p.logf = t.Logf + var events []said + p.publishWants = func(event string, body any, id string) error { + raw, _ := json.Marshal(body) + events = append(events, said{event, id, raw}) + return nil + } + p.w.runningHeard = true + placeOn(t, p.w, "ace", "db", "app") + placeOn(t, p.w, "g14", "db") + fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } + want := func(node, module, fp, version string) { + key, _ := mwc.WantKey(node, module) + raw, err := mwc.EncodeWant(mwc.Want{Fp: fp, Version: version, Self: true}) + if err != nil { + t.Fatal(err) + } + p.w.put(buckets.Wants, key, raw) + } + runs := func(node, module, fp string) { + raw, _ := json.Marshal(map[string]any{"v": 1, "node": node, "module": module, "fingerprint": fp, "at": at}) + if err := p.w.hearRunning("running", raw); err != nil { + t.Fatal(err) + } + } + source := int64(3) + release := func(module, version string) { + source++ + raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{V: 1, Module: module, Version: version, Source: source, At: &at, + Artifacts: []meshcatalog.VersionArtifact{{Artifact: "program", Digest: "sha256:d"}}}) + if err := p.w.released.put(raw); err != nil { + t.Fatal(err) + } + } + saidOf := func(event string) []said { + var out []said + for _, e := range events { + if e.event == event { + out = append(out, e) + } + } + return out + } + want("ace", "app", fp("1"), "") + + // No version of db is released: app's new fingerprint waits for nothing. + p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}}) + if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) { + t.Fatal("app waits with no version of db released") + } + release("db", "1.1.0") + want("ace", "db", fp("d"), "1.1.0") + want("g14", "db", fp("d"), "1.1.0") + runs("ace", "db", fp("d")) + // A kind-1 edge orders nothing. + p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 1, From: "db:program", To: "app:program"}}) + if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) { + t.Fatal("a kind-1 edge holds app") + } + p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}}) + // app's want unchanged waits for nothing. + if p.holdForDelivery(ctx, "app", fp("1"), trunkBranch, []string{"ace"}) { + t.Fatal("app's unchanged want waits") + } + // g14 places db and does not run 1.1.0 yet: not delivered, and app's new version waits, said once. + p.judgeDelivered() + if !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) || !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) { + t.Fatal("app does not wait for db 1.1.0") + } + if len(saidOf(meshwants.EventVersionDelivered)) != 0 { + t.Fatalf("db is said delivered while g14 does not run it: %+v", events) + } + held := saidOf(meshwants.EventWaitHeld) + if len(held) != 1 { + t.Fatalf("wait-held said %d times, want once", len(held)) + } + var w meshwants.Wait + _ = json.Unmarshal(held[0].body, &w) + if w.Step != meshwants.WaitStepDelivery || w.Part != "db" || w.Module != "app" || w.Version != fp("2") || w.Bound != "1h0m0s" || + w.ClockFrom == nil || !w.ClockFrom.Equal(at) || len(w.Machines) != 1 || w.Machines[0] != "g14" || w.Words != "waiting for delivery of db 1.1.0 on g14" { + t.Errorf("the wait said is %+v", w) + } + if _, ok := waits.values["app"]; !ok { + t.Errorf("the wait is not kept in delivery-waits: %v", waits.values) + } + // g14 away: every present machine runs it, so it is delivered, g14 named away; once only. + p.w.heardPresence("g14", true, at) + p.judgeDelivered() + p.judgeDelivered() + delivered := saidOf(meshwants.EventVersionDelivered) + var d meshwants.VersionDeliveredEvent + if len(delivered) != 1 || json.Unmarshal(delivered[0].body, &d) != nil || d.Module != "db" || d.Version != "1.1.0" || + len(d.Away) != 1 || d.Away[0] != "g14" || d.Parts["db:program"] != "sha256:d" { + t.Fatalf("version-delivered said %+v", delivered) + } + // The wait ends ran, and app's want goes on. + if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) { + t.Fatal("app still waits once db 1.1.0 is delivered") + } + ended := saidOf(meshwants.EventWaitEnded) + if len(ended) != 1 || json.Unmarshal(ended[0].body, &w) != nil || w.How != deliveryWaitRan { + t.Errorf("wait-ended said %+v", ended) + } + if _, ok := waits.values["app"]; ok { + t.Error("the ended wait is kept") + } + // A newer release with every placing machine away is not delivered, and its wait says the machines it waits to be back. + release("db", "1.2.0") + p.w.heardPresence("ace", true, at) + p.judgeDelivered() + if len(saidOf(meshwants.EventVersionDelivered)) != 1 { + t.Error("a version is said delivered while every placing machine is away") + } + if !p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) { + t.Fatal("app does not wait for db 1.2.0") + } + held = saidOf(meshwants.EventWaitHeld) + if json.Unmarshal(held[len(held)-1].body, &w) != nil || w.Words != "waiting for ace, g14 to be back" { + t.Errorf("the wait while every machine is away says %q", w.Words) + } + // Released by hand (assigner_release writes it): the wait ends released and app's want is written. + var kept deliveryWaiting + _ = json.Unmarshal(waits.values["app"], &kept) + kept.Released = true + raw, _ := json.Marshal(kept) + p.w.put(buckets.DeliveryWaits, "app", raw) + if p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) { + t.Fatal("a released wait holds app") + } + ended = saidOf(meshwants.EventWaitEnded) + if json.Unmarshal(ended[len(ended)-1].body, &w) != nil || w.How != deliveryWaitReleased { + t.Errorf("the released wait ended %+v", w) + } + // Presence: the later of machine-away and machine-back holds, whichever stream is read first. + p.w.heardPresence("ace", false, at.Add(time.Minute)) + p.w.heardPresence("ace", true, at) + if p.w.awayNow()["ace"] || !p.w.online()["ace"] { + t.Error("an older machine-away read after a newer machine-back leaves ace away") + } + // A-250: an ask of assigner_delivered_again of 1.1.0, which ace (present) runs and g14 (away) does not, is said again and taken + // away; one of 1.2.0, which no machine runs, is kept with why. + ask := func(version string) { + raw, _ := json.Marshal(againAsk{Module: "db", Version: version, By: "op", Why: "the catalog missed it"}) + p.w.put(buckets.DeliveredAgain, "db", raw) + } + before := len(saidOf(meshwants.EventVersionDelivered)) + ask("1.1.0") + p.deliveredAgain(ctx) + if len(saidOf(meshwants.EventVersionDelivered)) != before+1 || p.w.has(buckets.DeliveredAgain, "db") { + t.Errorf("the ask of 1.1.0 said %d version-delivered and is kept %v", len(saidOf(meshwants.EventVersionDelivered))-before, p.w.has(buckets.DeliveredAgain, "db")) + } + ask("1.2.0") + p.deliveredAgain(ctx) + var a againAsk + if json.Unmarshal(p.w.value(buckets.DeliveredAgain, "db"), &a) != nil || a.Waits != "db 1.2.0 does not run healthy on ace" || + len(saidOf(meshwants.EventVersionDelivered)) != before+1 { + t.Errorf("the ask of 1.2.0 = %+v", a) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go index c013f018..7a6c49c7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go @@ -34,6 +34,8 @@ type graphKeeper struct { source int64 ok bool said string + // pins removes a revision that ceased to be current once nothing pins it (pins.go); nil keeps every revision. + pins *pinKeeper } // keepGraph computes the revision of this pass and keeps it when the pass's inputs differ from the last computed. @@ -42,13 +44,7 @@ func (p *passer) keepGraph(ctx context.Context) { if g == nil || g.objects == nil || g.halves == nil { return } - in := assigngraph.Inputs{Halves: g.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()} - // With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's. - in.FirstBranch = func(string) string { return trunkBranch } - if p.ends != nil { - in.FirstBranch = func(module string) string { return p.ends.branchOf(module, "") } - } - c := assigngraph.Compute(in) + c := assigngraph.Compute(p.graphInputs()) for _, u := range c.Uncomputed { if g.said != u { g.said = u @@ -66,19 +62,35 @@ func (p *passer) keepGraph(ctx context.Context) { g.last, g.has, g.kept, g.ok = c.Revision, true, rev, true g.source = 0 for _, cat := range rev.Catalogs { - if cat.Branch == trunkBranch { + if cat.Branch == p.w.trunkName() { g.source = cat.Source } } if !fresh { return } + if g.pins != nil { + g.pins.judge() // the revision before ceased to be current + } id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision) if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil { p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err) } } +// graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each +// node's capability reading and the first branch of each registration. +func (p *passer) graphInputs() assigngraph.Inputs { + in := assigngraph.Inputs{Halves: p.graph.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()} + // With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's. + first := p.w.trunkName() + in.FirstBranch = func(string) string { return first } + if p.ends != nil { + in.FirstBranch = func(module string) string { return p.ends.branchOf(module, "") } + } + return in +} + // capabilityDigests is the digest of each node's capability reading, the statement its engine made, by node. func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesItem { if p.reads == nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go index ad7637be..973a66ec 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "mesh-delivery-assigner/internal/assigngraph" "mesh-delivery-assigner/internal/buckets" ) @@ -77,4 +78,47 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) { if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest { t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events)) } + + // A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment + // graph under candidate.. in place of its branch's half; it says nothing and writes no want, a try's candidate + // keeps none, and the run's group-ended drops it. + before, puts := len(events), kv.puts + ev := meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "run-7", Heads: "h1", Revision: 5} + var asked []meshcatalog.RevisionInput + call := func(key string, body any) (json.RawMessage, error) { + in := body.(meshcatalog.RevisionInput) + asked = append(asked, in) + rev := meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "candidate", Group: "run-7", Heads: "h1", Revision: 5, Source: 9, + Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("b", 64)}}} + a := meshcatalog.RevisionAnswer{Revision: &rev, Entries: []meshcatalog.CatalogEntry{{Module: "web", Fp: "src2:w2"}}} + return json.Marshal(a) + } + if err := p.keepCandidate(ctx, call, meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "try-3", Heads: "h", Revision: 2}); err != nil || len(asked) != 0 { + t.Errorf("a try's candidate asked %v: %v", asked, err) + } + if err := p.keepCandidate(ctx, call, ev); err != nil { + t.Fatal(err) + } + if len(asked) != 1 || asked[0].Group != "run-7" || asked[0].Heads != "h1" || asked[0].Revision != 5 { + t.Errorf("the candidate's revision was asked as %+v", asked) + } + a, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{Group: "run-7"}) + if err != nil || a.Revision.Kind != "candidate" || len(a.Revision.Catalogs) != 1 || a.Revision.Catalogs[0].Revision != 5 { + t.Errorf("the run's assignment graph is %+v: %v", a.Revision, err) + } + if e, err := assigngraph.Entry(ctx, objects, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"}); err != nil || e.Entry.Fp != "src2:w2" { + t.Errorf("the run's entry of web is %+v: %v", e.Entry, err) + } + if len(events) != before || kv.puts != puts { + t.Errorf("a candidate said %d events and wrote %d wants", len(events)-before, kv.puts-puts) + } + if err := p.dropCandidate(ctx, "run-7"); err != nil { + t.Fatal(err) + } + if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; ok { + t.Error("the run's group-ended left its candidate kept") + } + if _, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{}); err != nil { + t.Errorf("dropping the run's candidate dropped trunk: %v", err) + } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go index 8d617071..7d07b518 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go @@ -16,11 +16,12 @@ import ( "git.novox.be/novox/mesh-sdk/go/views" ) -// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195): one -// meshcatalogclient.Catalog per branch, adopted through the module's views reader (A-26), which asks the first view of the verb -// `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings; each entry is taken -// into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog half, never its -// own output back (§195). +// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195, §507, §532): the +// branch follower of meshcatalogclient, adopted on the module's views reader (A-26), follows the branches the registrations name +// and, before the first registrations, the branches the gate's verb `source` answers and its event `source-moved` brings; each +// branch's view asks the verb `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings; +// each entry is taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog +// half, never its own output back (§195). type halves struct { w *world call mcc.Call @@ -29,42 +30,70 @@ type halves struct { logf func(string, ...any) ctx context.Context - mu sync.Mutex - cats map[string]*mcc.Catalog - // settled is told once when a branch's first view is answered, for the first pass to wait on (the trunk's alone). + branches *mcc.Branches + mu sync.Mutex + // settled is told once when the first watched branch's first view is answered, for the first pass to wait on. settled chan<- struct{} + told bool } -// follow adopts the catalog of branch through the reader, once; true when it was made now. -func (h *halves) follow(branch string) bool { - h.mu.Lock() - if h.cats[branch] != nil { - h.mu.Unlock() - return false - } - c, _, err := mcc.WireCatalogProcess(h.reader, h.call, branch, func(module string, entry json.RawMessage, deleted bool, _ time.Time) { - h.changed(branch, module, entry, deleted) - }) +// follow makes the branch follower on the reader; the reader's start asks the first views. +func (h *halves) follow() error { + b, err := mcc.FollowBranchesProcess(h.reader, h.call) if err != nil { - h.mu.Unlock() - h.logf("the catalog half of branch %q is not followed: %v", branch, err) - return false + return err } - if h.cats == nil { - h.cats = map[string]*mcc.Catalog{} + b.Logf = h.logf + b.Changed = func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) { + h.changed(branch, module, entry, deleted) } - h.cats[branch] = c + b.Adopted = h.adopted + h.mu.Lock() + h.branches = b h.mu.Unlock() - if branch == trunkBranch && h.settled != nil { - go func() { - select { - case <-c.Ready(): - h.settled <- struct{}{} - case <-h.ctx.Done(): - } - }() + return nil +} + +// first is the first watched branch followed, the trunk's: "" while none is. +func (h *halves) first() string { + h.mu.Lock() + b := h.branches + h.mu.Unlock() + if b == nil { + return "" } - return true + if f := b.Followed(); len(f) > 0 { + return f[0] + } + return "" +} + +// adopted is told of each branch whose view the follower adopted: the first branch is the trunk's, and the first pass waits for its +// first view. +func (h *halves) adopted(branch string, c *mcc.Catalog) { + first := h.first() + if first == "" { + return + } + h.w.setTrunk(first) + h.wake() + if branch != first || h.settled == nil { + return + } + h.mu.Lock() + told := h.told + h.told = true + h.mu.Unlock() + if told { + return + } + go func() { + select { + case <-c.Ready(): + h.settled <- struct{}{} + case <-h.ctx.Done(): + } + }() } // changed takes one entry of branch's half into the world, or a module taken out of it. @@ -81,6 +110,7 @@ func (h *halves) changed(branch, module string, raw json.RawMessage, deleted boo } v, n, m := fromEntry(e) h.w.putModule(branch, module, v, n, m) + h.w.putEdges(branch, module, e.Edges) } // fromEntry is a catalog entry as the value and the graph node the rules read, and its manifest parsed once under the digest of @@ -114,11 +144,17 @@ func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, gr // views is the catalog graph revision held now of each followed branch whose first view is answered. func (h *halves) views() map[string]mcc.Half { h.mu.Lock() - defer h.mu.Unlock() - out := make(map[string]mcc.Half, len(h.cats)) - for branch, c := range h.cats { - if half, ok := c.View(); ok { - out[branch] = half + b := h.branches + h.mu.Unlock() + out := map[string]mcc.Half{} + if b == nil { + return out + } + for _, branch := range b.Followed() { + if c, ok := b.Catalog(branch); ok { + if half, ok := c.View(); ok { + out[branch] = half + } } } return out diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 80f04ef5..6cd46b78 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -8,6 +8,8 @@ import ( "errors" "fmt" mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/keys" "io" "maps" @@ -44,6 +46,7 @@ import ( occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" + mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/buscompose" @@ -176,7 +179,34 @@ func startBed(t *testing.T, leaveOut string) *bed { Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.mesh-catalog.event.>", "mesh.seat.mesh-conditions.event.>", "mesh.seat.mesh-module-assigner.event.>", enginereads.SubjectPrefix + "*"}}); err != nil { + t.Fatal(err) + } + // The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520). + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: nodeengine.StreamRunning, Subjects: []string{nodeengine.SubjectRunningAll}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, nodeengine.StreamRunning, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + // The watchman's machine-away and machine-back and the assigner's own version-delivered, kept by age, which it reads whole from their + // first message at its start and then follows through its durable (design 47 G §376, §389, A-189, A-233). + for _, k := range []struct{ stream, subject string }{{wd.StreamMachineAway, wd.SubjectMachineAwayAll}, {wd.StreamMachineBack, wd.SubjectMachineBackAll}, + {meshwants.StreamVersionDelivered, meshwants.SubjectVersionDelivered}} { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: k.stream, Subjects: []string{k.subject}, MaxAge: 90 * 24 * time.Hour}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, k.stream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + } + // The builder's stream of build ends, kept last per subject (its manifest's `last-per-subject`, design 47 G A-181), which the assigner + // reads whole at its start and follows through its durable, as the grants part makes it (from the last of each subject). + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildsStreamName, Subjects: []string{process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, buildsStreamName, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy, + FilterSubjects: []string{process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}); err != nil { t.Fatal(err) } // The assigner's own stream of its findings' events (design 47 G §383), which its finding writer says into. @@ -187,7 +217,8 @@ func startBed(t *testing.T, leaveOut string) *bed { // event names the engine's machine last, so the filter is the event with every machine (mesh-grants buscompose // machineEvent; the grants part's own test holds the derivation). if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", - FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", + "mesh.mod." + buckets.TellerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } // The operator-channel seat's stream of conditions, which the assigner says its conditions on. @@ -196,6 +227,8 @@ func startBed(t *testing.T, leaveOut string) *bed { } // The indexer as the owner of its manifests and modules (DLV-F2r), with the real owner's side. serveIndexer(t, b, nc) + // The teller as the owner of its answers (A-175). + serveTeller(t, b, nc) // The seat mesh-catalog's catalog half (design 47 G §194), which the assigner follows per branch. serveCatalog(t, b, nc) // The assigner's views family is its ordinary events on EVENTS (ADR 0635 §5); its marks are its state. @@ -356,7 +389,7 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) { } for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) { // A change of a bucket it watches starts a pass, which opens the puts' bucket. - if _, err := b.buckets[buckets.Presence].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil { + if _, err := b.buckets[buckets.Have].Put(b.ctx, "n1.pulse", []byte(time.Now().String())); err != nil { t.Fatal(err) } if e, err := putsKV.Get(b.ctx, pkey); err == nil { @@ -407,11 +440,38 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) { } // putJSON writes a value into one of the bed's buckets. +// putBuild says the builder's `build.ended` of the build a builder's key and value (the old form of a done build, as the tests write it) name, +// on the builder's stream (design 47 G A-181): the assigner hears what was built, it reads no bucket of the builder. +func (b *bed) putBuild(t *testing.T, key string, value []byte) { + t.Helper() + module, fp, err := builds.ParseBuilderBuildKey(key) + built, derr := builds.DecodeBuilderBuild(value) + if err != nil || derr != nil { + t.Fatalf("%s: %v %v", key, err, derr) + } + raw, _ := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: built.State, Error: built.Error, Manifest: built.Manifest, Needs: built.Needs}) + if _, err := b.js.Publish(b.ctx, process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEventName(module, fp)), raw); err != nil { + t.Fatal(err) + } +} + func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) { t.Helper() if _, err := b.buckets[bucket].Put(b.ctx, key, value); err != nil { t.Fatalf("%s %s: %v", bucket, key, err) } + if bucket == buckets.Have { // the node-engine says `running` with each have it writes (§520): the assigner reads health from it alone + node, module, isNode, err := nodeengineclient.ParseHaveKey(key) + h, derr := nodeengineclient.DecodeHave(value) + if err != nil || isNode || derr != nil || h.Phase != nodeengineclient.PhaseRunning { + return + } + ok := h.Health == nodeengineclient.HealthOK + raw, _ := json.Marshal(nodeengine.Running{At: time.Now(), Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok}) + if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, module), raw); err != nil { + t.Fatal(err) + } + } } // place adds module to the seat's assignments of node, written as the seat writes it: one node key and a key per module @@ -510,7 +570,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { t.Fatal(err) } bkey, _ := builds.BuilderBuildKey("web", fNew) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) for _, fp := range []string{fNew, fOld} { gkey, _ := grantsstate.PartGrantKey("web", fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) @@ -519,8 +579,6 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld} for _, node := range []string{"ace", "g14", "shanks"} { b.compose(t, node, 'c') - on, _ := presenceValue(true) - b.putJSON(t, buckets.Presence, node, on) have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: runs[node], Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) b.putJSON(t, buckets.Have, node+".web", have) b.place(t, node, "web") @@ -660,7 +718,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { for _, fp := range fps { built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: row.digest}) bkey, _ := builds.BuilderBuildKey(module, fp) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) gkey, _ := grantsstate.PartGrantKey(module, fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) b.putJSON(t, buckets.Grants, gkey, grant) @@ -736,7 +794,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { t.Fatal(err) } bkey, _ := builds.BuilderBuildKey("forms", fp) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) gkey, _ := grantsstate.PartGrantKey("forms", fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) b.putJSON(t, buckets.Grants, gkey, grant) @@ -847,8 +905,6 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) { digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) b.putModule(t, "web", mod) - on, _ := presenceValue(true) - b.putJSON(t, buckets.Presence, "ace", on) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) b.putJSON(t, buckets.Have, "ace.web", failed) b.place(t, "ace", "web") @@ -905,8 +961,6 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) { digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) b.putModule(t, "mail", mod) - on, _ := presenceValue(true) - b.putJSON(t, buckets.Presence, "ace", on) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) b.putJSON(t, buckets.Have, "ace.mail", failed) wantKey, _ := mwc.WantKey("ace", "mail") @@ -945,8 +999,6 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) { func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) { b := startBed(t, "") const modules = 40 - on, _ := presenceValue(true) - b.putJSON(t, buckets.Presence, "ace", on) names := make([]string, modules) for i := range names { name := fmt.Sprintf("mod%02d", i) @@ -960,7 +1012,7 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) { t.Fatal(err) } bkey, _ := builds.BuilderBuildKey(name, fp) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) gkey, _ := grantsstate.PartGrantKey(name, fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) b.putJSON(t, buckets.Grants, gkey, grant) @@ -1003,12 +1055,10 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) { b.putModule(t, name, mod) built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) bkey, _ := builds.BuilderBuildKey(name, fp) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) gkey, _ := grantsstate.PartGrantKey(name, fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) b.putJSON(t, buckets.Grants, gkey, grant) - on, _ := presenceValue(true) - b.putJSON(t, buckets.Presence, "ace", on) b.place(t, "ace", name) } @@ -1326,6 +1376,13 @@ func (s bedStore) history(string, string) ([]stdio.StateRevision, error) { retur // and declined, writes nothing and is withdrawn. func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) { b := startBed(t, "") + // Both machines are away throughout (§389): a machine the watchman never said away is present. + for _, node := range []string{"anchor", "laptop"} { + raw, _ := json.Marshal(wd.MachineAwayEvent{V: 1, Machine: node, At: time.Now(), LastHeard: time.Now()}) + if _, err := b.js.Publish(b.ctx, wd.SubjectMachineAway(node), raw); err != nil { + t.Fatal(err) + } + } if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "MOD_" + assignment.Module + "_assignments", Subjects: assignmentsSubjects()}); err != nil { // the seat's four events: the views family has a stream of its own t.Fatal(err) @@ -1664,13 +1721,32 @@ var catalogLate chan struct{} // read by the assigner's durable consumer; and its first-view tools of the three answered from the buckets at their marks. func serveIndexer(t *testing.T, b *bed, nc *nats.Conn) { t.Helper() - marks, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: buckets.IndexerModule + "_" + views.MarksState, History: 1}) + tools := map[string][2]string{} + for state, bucket := range indexerFirstViews { + tools[state] = [2]string{state, bucket} + } + serveOwner(t, b, nc, buckets.IndexerModule, tools, catalogLate) +} + +// serveTeller is the teller as the owner of its answers (design 47 G A-175, A-178): its bucket, which a test fills as the +// teller writes it, said by its views family, and its read tool teller_read_answers answering the first view. +func serveTeller(t *testing.T, b *bed, nc *nats.Conn) { + t.Helper() + serveOwner(t, b, nc, buckets.TellerModule, map[string][2]string{buckets.AnswersReadTool: {buckets.AnswersState, buckets.Answers}}, nil) +} + +// serveOwner serves an owner's states as the owner does: each state's bucket, its views family on EVENTS, and the first-view +// tool of each (by tool, its state and bucket) answered from the bucket at its mark. While late is open every ask fails. +func serveOwner(t *testing.T, b *bed, nc *nats.Conn, owner string, tools map[string][2]string, late chan struct{}) { + t.Helper() + marks, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: owner + "_" + views.MarksState, History: 1}) if err != nil { t.Fatal(err) } - em := views.Emitter{Owner: buckets.IndexerModule, Subject: process.Account{Node: "n1", Module: buckets.IndexerModule}.EventSubject, + em := views.Emitter{Owner: owner, Subject: process.Account{Node: "n1", Module: owner}.EventSubject, Publish: views.JetStreamPublish(b.js), Marks: marks} - for state, bucket := range indexerFirstViews { + for tool, sb := range tools { + state, bucket := sb[0], sb[1] kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 5}) if err != nil { t.Fatal(err) @@ -1679,13 +1755,12 @@ func serveIndexer(t *testing.T, b *bed, nc *nats.Conn) { if err := em.Start(b.ctx, state, kv, nil); err != nil { t.Fatal(err) } - late := catalogLate - sub, err := nc.Subscribe("mesh.mod.mesh-delivery-indexer.call."+state+".>", func(m *nats.Msg) { - if late != nil { // its runtime comes up after the assigner started (DLV-F5r): until then every ask fails, as on a down indexer + sub, err := nc.Subscribe("mesh.mod."+owner+".call."+tool+".>", func(m *nats.Msg) { + if late != nil { // its runtime comes up after the assigner started (DLV-F5r): until then every ask fails, as on a down owner select { case <-late: default: - _ = m.Respond([]byte(`{"error":"the indexer's runtime is not up"}`)) + _ = m.Respond([]byte(`{"error":"the owner's runtime is not up"}`)) return } } @@ -1693,7 +1768,7 @@ func serveIndexer(t *testing.T, b *bed, nc *nats.Conn) { After string `json:"after"` } _ = json.Unmarshal(m.Data, &args) - a, err := views.FirstView(buckets.IndexerModule, state, bedSource{b.ctx, kv}, bedSource{b.ctx, marks}, args.After, time.Now()) + a, err := views.FirstView(owner, state, bedSource{b.ctx, kv}, bedSource{b.ctx, marks}, args.After, time.Now()) out := map[string]any{"result": a, "node": "n0"} if err != nil { out = map[string]any{"error": err.Error()} @@ -1911,6 +1986,29 @@ func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) { raw, _ := json.Marshal(map[string]any{"result": meshcatalog.RevisionAnswer{Revision: &rev, Entries: entries}, "node": "n0"}) _ = m.Respond(raw) } + // The gate's verb `source` given no branch (design 47 G §507, §690): the branches it keeps a source revision of, which the assigner + // follows before the first registrations, and the trunk's `main` first. + source := func(m *nats.Msg) { + b.cat.mu.Lock() + defer b.cat.mu.Unlock() + names := map[string]bool{"main": true} + for branch := range b.cat.rev { + names[branch] = true + } + var escaped []string + for _, n := range slices.Sorted(maps.Keys(names)) { + escaped = append(escaped, keys.EscapeName(n)) + } + raw, _ := json.Marshal(map[string]any{"result": mdseat.SourceAnswer{Branches: escaped}, "node": "n0"}) + _ = m.Respond(raw) + } + for _, subject := range []string{"mesh.seat.mesh-delivery.tool.source", "mesh.seat.mesh-delivery.call.source.*", "mesh.seat.mesh-delivery.tool.source.*"} { + sub, err := nc.Subscribe(subject, source) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } for _, subject := range []string{"mesh.seat.mesh-catalog.tool.revision", "mesh.seat.mesh-catalog.call.revision.*", "mesh.seat.mesh-catalog.tool.revision.*"} { sub, err := nc.Subscribe(subject, answer) if err != nil { @@ -1920,3 +2018,52 @@ func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) { } subscribed(t, nc) } + +// A-181: the assigner reads the builder's stream of build ends whole at its start (the last of each fingerprint, a failed one's error +// among them) though its durable is made after them, then follows it: what was built survives its restart, and it reads no builder bucket. +func TestTheAssignerReadsTheBuildersEndsWholeAtItsStart(t *testing.T) { + b := startBed(t, "") + thisNode = "n1" + b.putBuild(t, mustBuildKey(t, "web", "src2:"+strings.Repeat("a", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildFailed, Error: "first try"})) + b.putBuild(t, mustBuildKey(t, "web", "src2:"+strings.Repeat("a", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64)})) + b.putBuild(t, mustBuildKey(t, "api", "src2:"+strings.Repeat("b", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildFailed, Error: "transport fault: x"})) + var ends builtEnds + woken := make(chan struct{}, 8) + hearBuilds(b.ctx, b.js, &ends, func() { woken <- struct{}{} }, assign.Findings{}, t.Logf) + if e, ok := ends.endOf("web", "src2:"+strings.Repeat("a", 64)); !ok || e.Outcome != builds.BuildDone || e.Manifest != strings.Repeat("d", 64) { + t.Fatalf("the read of the stream left web's end as %+v, %v: want the last, done", e, ok) + } + if e, ok := ends.endOf("api", "src2:"+strings.Repeat("b", 64)); !ok || e.Outcome != builds.BuildFailed || e.Error != "transport fault: x" { + t.Fatalf("the read of the stream left api's end as %+v, %v", e, ok) + } + b.putBuild(t, mustBuildKey(t, "api", "src2:"+strings.Repeat("c", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64)})) + deadline := time.After(10 * time.Second) + for { + if _, ok := ends.endOf("api", "src2:"+strings.Repeat("c", 64)); ok { + return + } + select { + case <-woken: + case <-deadline: + t.Fatal("an end said after the read was not followed") + } + } +} + +func mustBuildKey(t *testing.T, module, fp string) string { + t.Helper() + k, err := builds.BuilderBuildKey(module, fp) + if err != nil { + t.Fatal(err) + } + return k +} + +func mustEncodeBuild(t *testing.T, b builds.BuilderBuild) []byte { + t.Helper() + raw, err := builds.EncodeBuilderBuild(b) + if err != nil { + t.Fatal(err) + } + return raw +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index baa43787..6d410c29 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -29,6 +29,7 @@ import ( mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" @@ -62,13 +63,8 @@ func main() { // holds the two lists equal), so the controller's broker grants exactly these. var readBuckets = []string{ buckets.Grants, - buckets.Answers, - buckets.Builds, - buckets.Presence, buckets.Have, buckets.Provisions, - buckets.Source, - buckets.Merges, } // readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every @@ -80,7 +76,7 @@ var readFamilies = []string{buckets.ComposedRead} var ownBuckets = []string{buckets.Asks, buckets.Differences, buckets.Puts, buckets.Wants, plc.AssignmentsBucket, plc.ProposalsBucket, plc.CandidatesBucket, plc.PulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, plc.SeatHoldersBucket, - engineReadsBucket, retiredBucket, copiesBucket, writtenBucket, buckets.Waits, buckets.Inputs} + engineReadsBucket, retiredBucket, copiesBucket, writtenBucket, buckets.Waits, buckets.PutWaits, buckets.DeliveryWaits, buckets.DeliveredAgain, buckets.Inputs, pinsBucket} // engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key ``, as // the verb manifest-reads last answered it or the fallback read it: what assigner_engine_reads lists and asks again. @@ -121,7 +117,11 @@ var keptStates = []announce.State{ {Name: copiesState, Readable: true}, {Name: writtenState, Readable: true}, {Name: buckets.WaitsState, Readable: true}, - {Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354) + {Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211) + {Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246) + {Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250) + {Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354) + {Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579) } // kvSet is every bucket the assigner opened, by name. @@ -168,12 +168,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err) continue } - if (name == buckets.Source || name == buckets.Merges) && err != nil { - // The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of - // a record module is asked of a person, as before. - logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err) - continue - } if name == engineReadsBucket && err != nil { // New with ADR 0618 §3: without it the statements the verb answers are kept in memory only, and // assigner_engine_reads lists nothing. @@ -185,12 +179,32 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is read by the tool process alone", name, err) continue } + if name == pinsBucket && err != nil { + // New with the pins (design 47 G §579): without it no pin is kept and no revision is removed. + logf("the bucket %s cannot be opened (%v): no pin is kept and no revision removed until it starts again", name, err) + continue + } if name == buckets.Waits && err != nil { // New with the wait on an outside cause (design 47 G §706): without it no wait is made, and a failing check is judged // as before. logf("the bucket %s cannot be opened (%v): no outside wait is kept until the assigner starts again", name, err) continue } + if name == buckets.DeliveredAgain && err != nil { + // New with A-250: without it assigner_delivered_again is not taken until the assigner starts again. + logf("the bucket %s cannot be opened (%v): no version is said delivered again until the assigner starts again", name, err) + continue + } + if name == buckets.DeliveryWaits && err != nil { + // New with A-246: without it no dependent waits for its dependency's delivery until the assigner starts again. + logf("the bucket %s cannot be opened (%v): no delivery wait is kept until the assigner starts again", name, err) + continue + } + if name == buckets.PutWaits && err != nil { + // New with A-211: without it no put's wait is said until the assigner starts again. + logf("the bucket %s cannot be opened (%v): no put's wait is said until the assigner starts again", name, err) + continue + } if name == retiredBucket && err != nil { // The retired state is read only for the copy into `assignments` (migrate.go): a mesh made after the switch has // none, and nothing is copied. @@ -215,7 +229,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any func startsWithout(name string) bool { switch name { case plc.SeatHoldersBucket, buckets.Provisions, assign.PoliciesBucket, assign.FindingsBucket, - buckets.Puts, buckets.Source, buckets.Merges, engineReadsBucket, retiredBucket, writtenBucket, buckets.Waits, buckets.Inputs: + buckets.Puts, engineReadsBucket, retiredBucket, writtenBucket, buckets.Waits, buckets.PutWaits, buckets.DeliveryWaits, buckets.DeliveredAgain, buckets.Inputs, pinsBucket: return true } return slices.Contains(seatBuckets, name) || name == buckets.Grants @@ -321,18 +335,18 @@ type emitting struct{ *process.Conn } func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) } +// optionalEnds is what stops when the watch of a bucket that never gates the first pass ends. +var optionalEnds = map[string]string{buckets.Puts: "no put is taken further", buckets.Answers: "no ask is answered"} + // watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and // watched only for keys, so a manifest that arrives wakes a composition that waits for it. var watched = []string{ buckets.Grants, - buckets.Answers, - buckets.Builds, plc.AssignmentsBucket, plc.ProposalsBucket, plc.CandidatesBucket, plc.PulsesBucket, copiesBucket, - buckets.Presence, buckets.Have, buckets.Asks, buckets.Differences, @@ -340,11 +354,12 @@ var watched = []string{ assign.PoliciesBucket, buckets.Provisions, plc.SeatHoldersBucket, - buckets.Source, - buckets.Merges, engineReadsBucket, writtenBucket, buckets.Waits, + buckets.PutWaits, + buckets.DeliveryWaits, + buckets.DeliveredAgain, // its watch wakes a pass at once on an ask of assigner_delivered_again (A-250) } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after @@ -377,7 +392,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, return fmt.Errorf("the bucket %s cannot be watched: %w", name, err) } // The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal. - optional := name == buckets.Puts + optional := name == buckets.Puts || name == buckets.Answers go func() { defer kw.Stop() for { @@ -388,7 +403,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, case e, ok := <-kw.Updates(): if !ok { if optional { - logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name) + logger.Printf("the watch of %s ended: %s until the assigner starts again", name, optionalEnds[name]) return } errs <- fmt.Errorf("the watch of %s ended", name) @@ -455,8 +470,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started++ var opts []jetstream.WatchOpt - if name == buckets.Have || name == buckets.Source { - // what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1) + if name == buckets.Have { + // what each node ran healthy before, for a set-back opts = append(opts, jetstream.IncludeHistory()) } if err := watch(name, opts...); err != nil { @@ -477,6 +492,14 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if iv != nil { iv.hub = hub started += iv.start(ctx, w, wake, settled) + // The teller's answers' picture is watched as a bucket is, but never gates the first pass: a teller that does not + // answer its first view only leaves the asks unanswered, said by the reader's finding (A-175). + if iv.answers != nil { + kvs[buckets.Answers] = iv.answers + if err := watch(buckets.Answers); err != nil { + logger.Printf("%v: no ask is answered until the assigner starts again", err) + } + } } // The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is // bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no @@ -547,7 +570,26 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if js != nil { p.publishWants = wantsPublisher(ctx, js, thisNode) if iv != nil && iv.h != nil { - p.graph = &graphKeeper{objects: newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module}, assigngraph.RevisionsObjects), halves: iv.h.views} + objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module}, assigngraph.RevisionsObjects) + p.graph = &graphKeeper{objects: objects, halves: iv.h.views} + keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error { + return p.keepCandidate(ctx, iv.call, ev) + } + iv.candidates.Store(&keep) + // The pins of the revisions (pins.go, design 47 G §579): the gate's pinned, freed, heads-replaced and pins-stated + // on the hub, registered before it starts. + if kv := kvs[pinsBucket]; kv != nil { + k := &pinKeeper{ctx: ctx, st: kvEnds{ctx, kv}, objects: objects, inUse: inUseOf(w), findings: findings, now: time.Now, logf: logger.Printf} + if err := k.made(); err != nil { + logger.Printf("the state %s is not marked made, so no revision is removed until it is: %v", pinsState, err) + } + if err := hearPins(hub, k); err != nil { + logger.Printf("a pin is not heard, so no revision is removed until the assigner starts again: %v", err) + } else { + p.graph.pins = k + k.watchUnpins(kv) + } + } } } // The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer. @@ -564,13 +606,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow // once the first registrations are written (§95, §353). p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf) - if iv != nil && iv.h != nil { - p.ends.onBranches = func(branches []string) { - for _, b := range branches { - iv.h.follow(b) - } - } - } + p.ends.trunk = w.trunkName + p.ends.runEnded = func(group string) error { return p.dropCandidate(ctx, group) } err := hearEnds(hub, p.ends) if err == nil && bus != nil { err = followRegistrations(ctx, hub, bus.Call, p.ends) @@ -579,9 +616,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err) } } - if err := hearBuildsAndReleases(hub, &w.ends, &w.released, wake, logger.Printf); err != nil { - logger.Printf("a build's end or a version's release is not heard, so no want names a build's parts, its keeping node or its version until the assigner starts again: %v", err) - } + hearDelivery(ctx, js, w, wake, findings, logger.Printf) // version-released, version-delivered, machine-away and machine-back + hearBuilds(ctx, js, &w.ends, wake, findings, logger.Printf) hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf) hearRunning(ctx, js, w, wake, findings, logger.Printf) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/objects.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/objects.go index 9844df61..73540150 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/objects.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/objects.go @@ -87,3 +87,15 @@ func (b *busObjects) Names(ctx context.Context) ([]string, error) { } return names, nil } + +// Delete removes the object of that name; one not held is no error (design 47 G §579: a revision nothing pins is removed). +func (b *busObjects) Delete(ctx context.Context, name string) error { + o, err := b.open(ctx) + if err != nil { + return err + } + if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go index 1af5f53c..6fc18759 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go @@ -269,7 +269,7 @@ func (p *passer) outsideReview(ctx context.Context, module, fp string, nodes []s dirty := o.dirty[node] cleared := prev != nil && prev.Cause != "" && o.cleared[node+"|"+prev.Cause] o.mu.Unlock() - failing, healthy := ok && h.Fp == fp && assign.Failed(h), ok && assign.Healthy(h, fp) + failing, healthy := ok && h.Fp == fp && assign.Failed(h), n.RunsHealthy(node, module, fp) if !failing && !healthy && prev == nil { continue } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go index 4a06236e..8438cdb3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go @@ -66,6 +66,8 @@ func (r *outsideRig) review(keeperSelf bool, moved func(string, string) bool) as r.t.Helper() n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "ace": {"nats": {Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: r.health}}}} + healthy := r.health == nodeengineclient.HealthOK + n.Running = map[string]map[string]nodeengine.Running{"ace": {"nats": {Node: "ace", Module: "nats", Fingerprint: fpTest, Healthy: &healthy}}} if moved == nil { moved = func(string, string) bool { return false } } @@ -257,7 +259,7 @@ func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) { r.cond("machine.hub.unreachable", "ace") r.review(false, nil) var acts []string - s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) { + s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) { acts = append(acts, key) return json.RawMessage(`{"ok":true,"log":"logged"}`), nil }) @@ -294,4 +296,38 @@ func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) { if r.bodies[len(r.bodies)-1].How != "go-on" || n.Haves["ace"]["nats"].Health != nodeengineclient.HealthOK { t.Errorf("last %+v health %s", r.bodies[len(r.bodies)-1], n.Haves["ace"]["nats"].Health) } + // A-246: a dependent's delivery wait is listed beside them, and released by its module and step delivery, its hand act first. + dw, _ := json.Marshal(deliveryWaiting{Wait: meshwants.Wait{Step: meshwants.WaitStepDelivery, Module: "app", Part: "db", On: "db@1.1.0", Machines: []string{"g14"}}}) + s.st.put(buckets.DeliveryWaitsState, "app", dw) + got, err = s.doWaits(map[string]any{"node": "g14"}) + if d := got.(map[string]any)["delivery"].([]deliveryWaiting); err != nil || len(d) != 1 || d[0].Module != "app" { + t.Errorf("the delivery waits listed = %+v %v", got, err) + } + if _, err := s.doRelease(map[string]any{"module": "app", "step": "build", "why": "x"}, "shanks.node-tools"); err == nil { + t.Error("a release of another step is taken") + } + if _, err := s.doRelease(map[string]any{"module": "web", "step": "delivery", "why": "x"}, "shanks.node-tools"); err == nil { + t.Error("a release of a module waiting for no delivery is taken") + } + before := len(acts) + if _, err := s.doRelease(map[string]any{"module": "app", "step": "delivery", "why": "db is fine as it runs"}, "shanks.node-tools"); err != nil { + t.Fatal(err) + } + raw, _, _, _ = s.st.get(buckets.DeliveryWaitsState, "app") + var released deliveryWaiting + if json.Unmarshal(raw, &released) != nil || !released.Released || !strings.Contains(released.By, "db is fine") || len(acts) != before+1 { + t.Errorf("the delivery wait after its release = %+v, hand acts %v", released, acts[before:]) + } + // A-250: assigner_delivered_again records its hand act, writes the ask and answers asked; assigner_waits lists it. + if _, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0"}, "op"); err == nil { + t.Error("an ask with no why is taken") + } + ans, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0", "why": "the catalog missed it"}, "op") + if err != nil || ans.(map[string]any)["asked"] != "db" || len(acts) != before+2 { + t.Fatalf("assigner_delivered_again = %+v %v", ans, err) + } + got, _ = s.doWaits(map[string]any{}) + if a := got.(map[string]any)["delivered-again"].([]againAsk); len(a) != 1 || a[0].Version != "1.1.0" || a[0].By != "op" { + t.Errorf("the asks listed = %+v", a) + } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_tools.go index 9d22b933..e915e82a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_tools.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "errors" "fmt" "slices" @@ -9,7 +10,9 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" "mesh-delivery-assigner/internal/buckets" "mesh-delivery-assigner/internal/outside" @@ -58,7 +61,111 @@ func (s *putTools) doWaits(args map[string]any) (any, error) { } out = append(out, said) } - return map[string]any{"waits": out}, nil + // And each dependent's wait for its dependency's delivery (design 47 G §376, A-246), from the state delivery-waits. + delivery := []deliveryWaiting{} + dkeys, err := s.st.keys(buckets.DeliveryWaitsState) + if err != nil { + return nil, fmt.Errorf("the delivery waits cannot be listed: %w", err) + } + sort.Strings(dkeys) + for _, key := range dkeys { + raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key) + if err != nil || !found { + continue + } + var w deliveryWaiting + if json.Unmarshal(raw, &w) != nil || (arg(args, "module") != "" && w.Module != arg(args, "module")) || + (arg(args, "node") != "" && !slices.Contains(w.Machines, arg(args, "node"))) { + continue + } + delivery = append(delivery, w) + } + // And each ask of assigner_delivered_again not yet holding, with why (A-250). + again := []againAsk{} + akeys, err := s.st.keys(buckets.DeliveredAgainState) + if err != nil { + return nil, fmt.Errorf("the asks to say a version delivered again cannot be listed: %w", err) + } + sort.Strings(akeys) + for _, key := range akeys { + raw, _, found, err := s.st.get(buckets.DeliveredAgainState, key) + var a againAsk + if err != nil || !found || json.Unmarshal(raw, &a) != nil || (arg(args, "module") != "" && a.Module != arg(args, "module")) { + continue + } + again = append(again, a) + } + return map[string]any{"waits": out, "delivery": delivery, "delivered-again": again}, nil +} + +// toolDeliveredAgain is the tool that asks version-delivered said again for one version (design 47 G §376, A-250). +const toolDeliveredAgain = "assigner_delivered_again" + +// doDeliveredAgain is assigner_delivered_again: the hand act is recorded first, then the ask is written into the state +// delivered-again, whose watch the assigner's process hears at once; it answers that it is asked. +func (s *putTools) doDeliveredAgain(args map[string]any, by string) (any, error) { + module, version, why := arg(args, "module"), arg(args, "version"), arg(args, "why") + if module == "" || version == "" || why == "" { + return nil, errors.New("assigner_delivered_again names the module, its version and why: it is kept in the hand-act log") + } + rec := meshhandactsclient.Recorder{Ask: s.ask} + taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_delivered_again %s %s", module, version), Why: why, + Cause: "version-delivered", Verb: "mesh-wants.assigner_delivered_again"}) + if err != nil { + return nil, fmt.Errorf("the hand act could not be recorded, so nothing was asked: %w", err) + } + data, err := json.Marshal(againAsk{Module: module, Version: version, By: by, Why: why}) + if err != nil { + return nil, err + } + if err := s.st.put(buckets.DeliveredAgainState, keys.EscapeName(module), data); err != nil { + return nil, fmt.Errorf("the ask of %s %s is not written: %w", module, version, err) + } + return map[string]any{"asked": module, "version": version, "hand-act": taken.Log, + "follow": "the assigner says version-delivered again once every present machine placing it runs it healthy; assigner_waits says why it does not yet"}, nil +} + +// deliveredAgainTool is assigner_delivered_again as the runtime serves it. +func (s *putTools) deliveredAgainTool() stdio.Tool { + return stdio.Tool{Name: toolDeliveredAgain, Description: "Say version-delivered again for one version of a module, for the operator and the healers " + + "(design 47 G §376): written in the hand-act log first, then asked of the assigner, which says it once every present machine placing " + + "the module runs that version healthy; until then assigner_waits lists the ask with why it does not hold. Answers that it is asked.", + Input: schema(map[string]string{"module": "the module", "version": "its MAJOR.MINOR.PATCH", "why": "why, kept in the hand-act log"}, + []string{"module", "version", "why"}), + Run: func(a map[string]any) (any, error) { return s.doDeliveredAgain(a, stdio.CallerOf(a)) }} +} + +// doReleaseDelivery is assigner_release with step delivery: the hand act is recorded first, then the release is written on the +// module's delivery wait, which the assigner's process ends released at its next pass, writing the module's want. +func (s *putTools) doReleaseDelivery(args map[string]any, by string) (any, error) { + module, why := arg(args, "module"), arg(args, "why") + if why == "" { + return nil, errors.New("release says why: it is kept in the hand-act log") + } + key := keys.EscapeName(module) + raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key) + if err != nil { + return nil, fmt.Errorf("the delivery wait of %s is not read: %w", module, err) + } + var w deliveryWaiting + if !found || json.Unmarshal(raw, &w) != nil || w.Released { + return nil, fmt.Errorf("%s waits for no delivery: assigner_waits lists the delivery waits", module) + } + rec := meshhandactsclient.Recorder{Ask: s.ask} + taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s step delivery (waits on %s)", module, w.On), Why: why, + Cause: "delivery-wait", Verb: "mesh-wants.assigner_release"}) + if err != nil { + return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err) + } + w.Released, w.By = true, by+": "+why + data, err := json.Marshal(w) + if err != nil { + return nil, err + } + if err := s.st.put(buckets.DeliveryWaitsState, key, data); err != nil { + return nil, fmt.Errorf("the release of %s's delivery wait is not written: %w", module, err) + } + return map[string]any{"released": module, "step": meshwants.WaitStepDelivery, "by": by, "hand-act": taken.Log}, nil } // doReleaseWait is assigner_release with how: the hand act is recorded first, then the release is written on the wait of node and diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 71581d0e..99e5985d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -205,6 +205,8 @@ func (p *passer) clock() time.Time { func (p *passer) pass(ctx context.Context) { p.assignmentsSeat(ctx) p.keepGraph(ctx) + p.judgeDelivered() + p.deliveredAgain(ctx) p.wants(ctx) p.puts(ctx) p.nodeWants(ctx) @@ -247,7 +249,7 @@ func (p *passer) wants(ctx context.Context) { views[branch] = v return v } - nodes := viewOf(trunkBranch).nodes + nodes := viewOf(p.w.trunkName()).nodes p.engineReads(nodes) nodesOf := map[string][]string{} for node, modules := range assigned { @@ -303,6 +305,10 @@ func (p *passer) wants(ctx context.Context) { if fp != "" && !p.ends.mayWant(name, fp, branches, wantedOf[name]) { return } + // A dependent's new version waits for its dependency's newest released version to be delivered (§376, A-189). + if p.holdForDelivery(ctx, name, fp, g.branch, g.nodes) { + return + } m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp), Self: func(node string) bool { return placed[node] }, Composed: func(node, fp string) (assign.ComposedDigest, bool) { @@ -319,8 +325,7 @@ func (p *passer) wants(ctx context.Context) { Allowed: func(node, digest string) bool { key, err := asks.RecordAskKey(node, name, digest) return err == nil && p.answeredWith(key, assign.OptionYes) - }, - Release: func(fp string) (string, bool) { return p.w.release(name, fp) }} + }} if fp != "" { held, _ := mwc.HeldKey(name, fp) grant, _ := grantsstate.PartGrantKey(name, fp) @@ -329,6 +334,7 @@ func (p *passer) wants(ctx context.Context) { m.Released = p.answeredWith(hold, "release") m.Kept = p.answeredWith(hold, "keep") m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant) + m.BuildFailed, m.BuildError = p.w.buildStanding(name, fp) if raw := p.w.value(grants, grant); raw != nil { if g, err := grantsstate.DecodeGrant(raw); err == nil { m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } @@ -404,7 +410,7 @@ func (p *passer) wants(ctx context.Context) { func (p *passer) movedTo(wanted map[string]map[string]mwc.Want, n assign.Nodes) func(owner, node string) bool { return func(owner, node string) bool { w, ok := wanted[owner][node] - return ok && !w.Removed && !assign.Healthy(n.Haves[node][owner], w.Fp) + return ok && !w.Removed && !n.RunsHealthy(node, owner, w.Fp) } } @@ -1060,9 +1066,8 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want m // yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a // person must allow (ADR 0547 §3): for a have of phase // waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes -// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. A -// move of the build alone (the have says so) to a build an approved release carries is given its yes by that Approve, the -// want naming the release, with no ask (ADR 0601 §1). +// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. No +// release's Approve is a yes: the Approve per merge goes and the assigner reads no `merges` (design 47 G §431). func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]mwc.Want, out *assign.Outcome) { for _, node := range m.Nodes { if m.Self == nil || !m.Self(node) { @@ -1091,14 +1096,6 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo switch { case m.Allowed != nil && m.Allowed(node, h.Waits): yes, release = h.Waits, "" - case h.BuildAlone && h.Fp != w.Fp && m.Release != nil: - // A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1), - // named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked. - if r, ok := m.Release(w.Fp); ok { - yes, release = h.Waits, r - break - } - p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail}) default: p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail}) } @@ -1408,14 +1405,14 @@ type group struct { func (p *passer) groups(module string, nodes []string, placed map[string]string) []group { by := map[string][]string{} for _, node := range nodes { - b := trunkBranch + b := p.w.trunkName() if p.ends.following() { b = p.ends.branchOf(module, placed[node]) } by[b] = append(by[b], node) } if len(by) == 0 { - b := trunkBranch + b := p.w.trunkName() if p.ends.following() { b = p.ends.branchOf(module, "") } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index e8398ead..59b992ba 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -16,14 +16,11 @@ import ( "time" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" - wdc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" @@ -70,7 +67,9 @@ func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt // newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given. func newPasser(kvs kvSet, at time.Time) *passer { - return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{}, + w := newWorld() + w.setTrunk(trunkBranch) + return &passer{kvs: kvs, w: w, logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{}, now: func() time.Time { return at }} } @@ -301,8 +300,8 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)}) } p := &passer{w: w, reads: &enginereads.Keeper{Call: call, Fallback: w.recordReads, Sync: true}} - on, _ := presenceValue(false) - w.put(buckets.Presence, "ace", on) + on := false + w.heardPresence("ace", !on, time.Now()) n = w.nodes(nil, nil) p.engineReads(n) if !slices.Equal(n.Reads["ace"], []string{"form:new"}) || len(n.Reads["g14"]) != 0 || asked["ace"] != 1 { @@ -312,8 +311,8 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { t.Errorf("the statements are %+v", s) } p.engineReads(w.nodes(nil, nil)) - on, _ = presenceValue(true) - w.put(buckets.Presence, "ace", on) + on = true + w.heardPresence("ace", !on, time.Now()) p.engineReads(w.nodes(nil, nil)) if asked["ace"] != 2 { t.Errorf("ace, back online, was asked %d times, want 2", asked["ace"]) @@ -337,16 +336,17 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { Keys: func() ([]string, error) { return slices.Collect(maps.Keys(held)), nil }}, reads: &enginereads.Keeper{Call: silent, Sync: true, Now: func() time.Time { return clock }}} key := mcc.Key(mcc.ScopeCall, enginereads.CallID("ace"), mcc.KindCallUnanswered) + w.heardPresence("g14", true, time.Now()) // g14 away throughout: a machine is present unless the watchman said it away (§389) before := asked["ace"] q.engineReads(w.nodes(nil, nil)) clock = clock.Add(6 * time.Minute) - off, _ := presenceValue(false) - w.put(buckets.Presence, "ace", off) + off := false + w.heardPresence("ace", !off, time.Now()) q.engineReads(w.nodes(nil, nil)) if len(held) != 0 { t.Errorf("a finding was written while ace is away: %v", slices.Collect(maps.Keys(held))) } - w.put(buckets.Presence, "ace", on) + w.heardPresence("ace", !on, time.Now()) q.engineReads(w.nodes(nil, nil)) if _, ok := held[key]; !ok || asked["ace"]-before < 3 { t.Errorf("ace unanswered six minutes, heard: findings %v, asked %d times", slices.Collect(maps.Keys(held)), asked["ace"]-before) @@ -560,18 +560,14 @@ func putOfTheBus(t *testing.T) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)}) putModuleRaw(p.w, trunkBranch, "nats", mod) p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}} - bkey, _ := builds.BuilderBuildKey("nats", fp) - built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) - world(buckets.Builds, bkey, built) + endDone(t, p.w, "nats", fp, mdigest) hkey, _ := nodeengineclient.HaveKey("novox", "nats") before, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) world(buckets.Have, hkey, before) - buildKV := mem() - buildKV.values[bkey] = built // Issue 1966: newest is asked of the indexer's first view of modules, never read from its bus state. indexer := &fakeIndexer{views: map[string]map[string][]byte{"mesh-delivery-indexer.modules": {"mesh-catalog": mod, "nats": mod}}} - s := newPutTools(memStore{putsState: putsKV, builds.BuilderModule + ".builds": buildKV}, func(key string, body any) (json.RawMessage, error) { + s := newPutTools(memStore{putsState: putsKV}, func(key string, body any) (json.RawMessage, error) { if raw, ok, err := indexer.answer(key, body); ok { return raw, err } @@ -582,10 +578,10 @@ func putOfTheBus(t *testing.T) { answerWithin = 0 defer func() { answerWithin = was }() if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil { - t.Error("a prefix of a fingerprint the builder never built was taken") + t.Error("a prefix of a fingerprint was taken: a put names the whole fingerprint (A-181)") } - if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp { - t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err) + if _, err := s.version("nats", "fd497ab80000"); err == nil { + t.Error("the prefix of the fingerprint built was read: a put names the whole fingerprint (A-181)") } if got, err := s.version("nats", "newest"); err != nil || got != fp || indexer.asks["mesh-delivery-indexer.modules"] != 2 { t.Errorf("newest reads %q, %v, asking %d pages of the indexer's first view (want its 2)", got, err, indexer.asks["mesh-delivery-indexer.modules"]) @@ -811,14 +807,14 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) { p := newPasser(kvSet{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC)) var logged []string p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } - notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}} + notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is being built"}}} wanted := assign.Outcome{Wants: map[string]mwc.Want{"shanks": {Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}} for i, c := range []struct { before map[string]mwc.Want out assign.Outcome line string // the line said in this pass, "" for none }{ - {nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"}, + {nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is being built"}, {nil, notDone, ""}, {map[string]mwc.Want{"shanks": {Removed: true}}, notDone, ""}, {nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"}, @@ -903,8 +899,8 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) putModuleRaw(p.w, trunkBranch, "web", mod) placeOn(t, p.w, "g14", "web") - on, _ := presenceValue(true) - p.w.put(buckets.Presence, "g14", on) + on := true + p.w.heardPresence("g14", !on, time.Now()) have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) p.w.put(buckets.Have, "g14.web", have) key, _ := mwc.WantKey("g14", "web") @@ -941,8 +937,8 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) putModuleRaw(p.w, trunkBranch, "systemd-resolved", mod) placeOn(t, p.w, "shanks", "systemd-resolved") - on, _ := presenceValue(true) - p.w.put(buckets.Presence, "shanks", on) + on := true + p.w.heardPresence("shanks", !on, time.Now()) have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}) p.w.put(buckets.Have, "shanks.systemd-resolved", have) hkey, _ := mwc.HeldKey("systemd-resolved", fpTest) @@ -1168,70 +1164,29 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing aReleasesApproveAnswersTheMoveOfItsBuildAlone(t) } -// ADR 0601 §1: a placed node's have waiting for a yes to a move of the build alone, to a build an approved release carries, is -// given its yes by that Approve, the want naming the release, and no record ask is raised; a move that changes a setting or a -// secret too, one at the build it runs, and one no approved release carries are asked of a person as before. +// Design 47 G §431: no release's Approve is a yes (the assigner reads no `merges`): a placed node's have waiting for a yes to a move +// of the build alone is asked of a person, the want naming no release and no yes. func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) { fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64) commit := strings.Repeat("1", 40) - for _, c := range []struct { - name string - alone bool - at string // the fingerprint the have runs - mergedAt string // the commit the release merged web at; "" for no merge of the release - fpSrc int64 // the source revision at which web's fingerprint last changed - yes, asked bool - }{ - {"the build alone, its release approved", true, fpOld, commit, 7, true, false}, - {"a setting changed too", false, fpOld, commit, 7, false, true}, - {"a setting changed at the build it runs", true, fpTest, commit, 7, false, true}, - {"no approved release carries it", true, fpOld, "", 7, false, true}, - {"the release merged another commit", true, fpOld, strings.Repeat("2", 40), 7, false, true}, - // Its fingerprint last moved at another release's revision, which merged nothing of web: that Approve answers nothing. - {"a fingerprint another release moved", true, fpOld, commit, 8, false, true}, - // Review rounds 1 and 2: moved by a recompute after the release (a toolchain, a package, a base) with no merge, the - // indexer writes fp-src 0, not known, though Src is still the release's: asked of a person. - {"a fingerprint moved by a recompute after the release", true, fpOld, commit, 0, false, true}, - } { - asksKV := &memKV{values: map[string][]byte{}} - p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now()) - p.logf = t.Logf - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: c.fpSrc, Path: meshcatalogclient.RootPath}) - putModuleRaw(p.w, trunkBranch, "web", mod) - src, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/web", Heads: map[string]string{"novox/web": commit}}) - p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), src, 7, time.Now()) - later, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}}) - p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), later, 8, time.Now()) // a later merge leaves revision 7's release - if c.mergedAt != "" { - merge, err := mdc.EncodeMerge(mdseat.MergesByGroupValue{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []mdseat.MergesByGroupValuePullsItem{ - {Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40), State: string(mdc.MergeMerged), Commit: c.mergedAt}}}) - if err != nil { - t.Fatal(err) - } - p.w.put(buckets.Merges, "feat/web", merge) - other, err := mdc.EncodeMerge(mdseat.MergesByGroupValue{Group: "feat/other", Ask: "merge.feat_other.0123456789ab", Pulls: []mdseat.MergesByGroupValuePullsItem{ - {Repository: "novox/x", Number: 4, Head: strings.Repeat("4", 40), State: string(mdc.MergeMerged), Commit: commit}}}) - if err != nil { - t.Fatal(err) - } - p.w.put(buckets.Merges, "feat/other", other) - } - have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: c.at, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK, - Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone}) - p.w.put(buckets.Have, "g14.web", have) - m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true }, - Release: func(fp string) (string, bool) { return p.w.release("web", fp) }} - out := assign.Outcome{Wants: map[string]mwc.Want{"g14": {Fp: fpTest, Self: true}}} - p.yeses(context.Background(), "web", m, map[string]mwc.Want{"g14": {Fp: fpOld, Self: true}}, &out) - w := out.Wants["g14"] - if got := w.Yes == waiting && w.Release == "feat/web"; got != c.yes || (!c.yes && (w.Yes != "" || w.Release != "")) { - t.Errorf("%s: the want is %+v", c.name, w) - } - akey, _ := asks.RecordAskKey("g14", "web", waiting) - if _, there := asksKV.values[akey]; there != c.asked { - t.Errorf("%s: a record ask raised %v, want %v", c.name, there, c.asked) - } + asksKV := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now()) + p.logf = t.Logf + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: 7, Path: meshcatalogclient.RootPath}) + putModuleRaw(p.w, trunkBranch, "web", mod) + have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK, + Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: true}) + p.w.put(buckets.Have, "g14.web", have) + m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true }} + out := assign.Outcome{Wants: map[string]mwc.Want{"g14": {Fp: fpTest, Self: true}}} + p.yeses(context.Background(), "web", m, map[string]mwc.Want{"g14": {Fp: fpOld, Self: true}}, &out) + if w := out.Wants["g14"]; w.Yes != "" || w.Release != "" { + t.Errorf("the want is %+v: no release's Approve is a yes", w) + } + akey, _ := asks.RecordAskKey("g14", "web", waiting) + if _, there := asksKV.values[akey]; !there { + t.Error("no record ask was raised for the move of the build alone") } } @@ -1265,8 +1220,8 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) { placeOn(t, p.w, "g14", "web") pl, _ := mmac.EncodeAssignments(mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}}) put(plc.AssignmentsBucket, "assignments.g14", pl) - on, _ := presenceValue(true) - put(buckets.Presence, "g14", on) + on := true + p.w.heardPresence("g14", !on, time.Now()) have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) put(buckets.Have, "g14.web", have) wkey, _ := mwc.WantKey("g14", "web") @@ -1407,9 +1362,7 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) { t.Fatal(err) } p.w.manifests[digest] = parsed{module: "server", manifest: parsedServer} - build, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: digest}) - key, _ := builds.BuilderBuildKey("server", fp("f")) - p.w.put(buckets.Builds, key, build) + endDone(t, p.w, "server", fp("f"), digest) running := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} at := func(f string) nodeengine.HaveByNodeModuleValue { h := running; h.Fp = f; return h } served := p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ @@ -1450,11 +1403,6 @@ func mustAskID(a asks.Ask) string { return ask.ID } -// presenceValue is a node's presence as the watchman writes it. -func presenceValue(online bool) ([]byte, error) { - return json.Marshal(wdc.NodePresence{V: 1, Online: online}) -} - // Issue 1966 (nats on novox): a yes given for a record ask was lost because the same pass took the ask away as stale before the // want named the yes, and an ask taken away and written again was never asked again because it carried the id the router kept. // Given a record ask of a placed node, then (a) a have mid-move, or running with the operator's yes unwritten, leaves the ask, diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins.go new file mode 100644 index 00000000..fa6055ac --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins.go @@ -0,0 +1,538 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + + mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-assigner/internal/assign" + "mesh-delivery-assigner/internal/assigngraph" + "mesh-delivery-assigner/internal/assignment" +) + +// The pins of the assignment graph revisions (design 47 G §153, §178, §216, §579; A-169): the gate says on the seat +// mesh-delivery which revisions a delivery record starts (`pinned`) and stops (`freed`) pointing at, and once at each of its +// starts `pins-stated`. The assigner hears them on its durable consumer of the events stream and keeps, in its old `state` +// section `pins` (4 MiB), one key `trunk.` per assignment graph revision whose value is the list of the records pinning +// it, added or removed by compare-and-set before the event is acknowledged; a record heard twice pins once. Two more keys: +// `made`, written once when the state is first used, and `pins-stated`, the `at` of the last pins-stated applied. A revision +// is removed, its object and its key and the entry objects no kept revision lists and no want or running build names +// (§179), when its list is empty and it is not current (the newest the assigner wrote), only while the consumer is caught up +// and once a pins-stated said after `made` was applied; judged on a freed, on a revision ceasing to be current, on being +// caught up, and at start once both hold. Never on a timer, and never by reading the gate's state. + +const ( + pinsState = "pins" + pinsBucket = assignment.Module + "_" + pinsState + pinsTool = "assigner_pins" + unpinTool = "assigner_unpin" + pinsMadeKey = "made" + pinsStatedKey = "pins-stated" + pinsCapBytes = 4 << 20 + pinMissingKind = "pin-missing" +) + +// PinsValue is one value of the state pins: the records that pin the revision, and whether the revision was no longer held +// when a record pinned it. +type PinsValue struct { + Records []string `json:"records"` + Missing bool `json:"missing,omitempty"` +} + +// pinKey is the key of an assignment graph revision, its object's own name (A-169). +func pinKey(revision int64) string { return assigngraph.TrunkObject(revision) } + +// pinRevision is the revision a key of the state names, and whether it names one. +func pinRevision(key string) (int64, bool) { + n, ok := strings.CutPrefix(key, "trunk.") + if !ok { + return 0, false + } + r, err := strconv.ParseInt(n, 10, 64) + return r, err == nil && r > 0 +} + +// pinObjects is the object bucket of the revisions as the pins remove from it. +type pinObjects interface { + assigngraph.Objects + Delete(ctx context.Context, name string) error +} + +// pinKeeper keeps the pins and removes what nothing pins. +type pinKeeper struct { + ctx context.Context + st endStore + objects pinObjects + // inUse says a want or a running build names the module's fingerprint (§179). + inUse func(module, fp string) bool + findings assign.Findings + now func() time.Time + logf func(string, ...any) + mu sync.Mutex + // caught is whether the last event applied left none of the consumer's messages pending. + caught bool +} + +// edit applies change to the value at key by compare-and-set, again while another write comes between. +func (k *pinKeeper) edit(key string, change func(v PinsValue, found bool) (PinsValue, bool)) error { + for { + raw, rev, err := k.st.get(key) + if err != nil { + return err + } + var v PinsValue + if raw != nil { + if err := json.Unmarshal(raw, &v); err != nil { + return fmt.Errorf("the pins of %s do not read: %w", key, err) + } + } + next, write := change(v, raw != nil) + if !write { + return nil + } + value, _ := json.Marshal(next) + if err := k.st.swap(key, value, rev); errors.Is(err, errSwapped) { + continue + } else { + return err + } + } +} + +// held says the revision's object is kept. +func (k *pinKeeper) held(revision int64) (bool, error) { + _, err := k.objects.Get(k.ctx, assigngraph.TrunkObject(revision)) + if errors.Is(err, assigngraph.ErrNoObject) { + return false, nil + } + return err == nil, err +} + +// pin adds record to each assignment graph revision of p; a revision no longer held is written missing, with the finding +// pin-missing naming the record. +func (k *pinKeeper) pin(p mdseat.Pins) error { + for _, r := range p.Assignment { + held, err := k.held(r) + if err != nil { + return err + } + err = k.edit(pinKey(r), func(v PinsValue, _ bool) (PinsValue, bool) { + if slices.Contains(v.Records, p.Record) && (held || v.Missing) { + return v, false + } + if !slices.Contains(v.Records, p.Record) { + v.Records = append(v.Records, p.Record) + slices.Sort(v.Records) + } + v.Missing = v.Missing || !held + return v, true + }) + if err != nil { + return err + } + if !held { + k.missing(r, p.Record) + } + } + return nil +} + +// free removes record from each assignment graph revision of p; a freed applied twice changes nothing. +func (k *pinKeeper) free(p mdseat.Pins) error { + for _, r := range p.Assignment { + err := k.edit(pinKey(r), func(v PinsValue, found bool) (PinsValue, bool) { + i := slices.Index(v.Records, p.Record) + if !found || i < 0 { + return v, false + } + v.Records = slices.Delete(v.Records, i, i+1) + return v, true + }) + if err != nil { + return err + } + } + return nil +} + +// missing writes the finding pin-missing, scope the assigner, id the revision. +func (k *pinKeeper) missing(revision int64, record string) { + if k.findings.Put == nil { + k.logf("the revision %d the record %s pins is not held, and no finding writer is wired to say so", revision, record) + return + } + o := assign.Observed{Scope: assignment.Module, ID: strconv.FormatInt(revision, 10), Kind: pinMissingKind, + Summary: fmt.Sprintf("the delivery record %s pins the assignment graph revision %d, which the assigner no longer holds", record, revision), + Headline: "A delivery record pins a revision the assigner no longer holds", + Explanation: fmt.Sprintf("The record %s pinned revision %d after it was removed; %s clears it.", record, revision, unpinTool), + Source: assignment.Module} + if err := k.findings.Put(o.Finding(k.now())); err != nil { + k.logf("the finding %s of revision %d is not written: %v", pinMissingKind, revision, err) + } +} + +// made writes the key made once, create-only; one there already stays. +func (k *pinKeeper) made() error { + raw, _, err := k.st.get(pinsMadeKey) + if err != nil || raw != nil { + return err + } + value, _ := json.Marshal(k.now().UTC()) + if err := k.st.swap(pinsMadeKey, value, 0); err != nil && !errors.Is(err, errSwapped) { + return err + } + return nil +} + +// timeAt is the time a key holds; false when it holds none. +func (k *pinKeeper) timeAt(key string) (time.Time, bool, error) { + raw, _, err := k.st.get(key) + if err != nil || raw == nil { + return time.Time{}, false, err + } + var t time.Time + if err := json.Unmarshal(raw, &t); err != nil { + return t, false, fmt.Errorf("the key %s does not read: %w", key, err) + } + return t, true, nil +} + +// stated keeps the at of a pins-stated applied. +func (k *pinKeeper) stated(at time.Time) error { + for { + raw, rev, err := k.st.get(pinsStatedKey) + if err != nil { + return err + } + var was time.Time + if raw != nil && json.Unmarshal(raw, &was) == nil && !at.After(was) { + return nil + } + value, _ := json.Marshal(at.UTC()) + if err := k.st.swap(pinsStatedKey, value, rev); errors.Is(err, errSwapped) { + continue + } else { + return err + } + } +} + +// statedSinceMade says a pins-stated said after the state was made was applied. +func (k *pinKeeper) statedSinceMade() (bool, error) { + made, ok, err := k.timeAt(pinsMadeKey) + if err != nil || !ok { + return false, err + } + at, ok, err := k.timeAt(pinsStatedKey) + if err != nil || !ok { + return false, err + } + return at.After(made), nil +} + +// hear applies one of the seat mesh-delivery's events and acknowledges it only once applied. +func (k *pinKeeper) hear(m jetstream.Msg) { + err := k.apply(m.Subject(), m.Data()) + if err != nil { + k.logf("%s: %v; it is heard again", m.Subject(), err) + _ = m.Nak() + return + } + _ = m.Ack() + caught := false + if meta, merr := m.Metadata(); merr == nil { + caught = meta.NumPending == 0 + } + k.mu.Lock() + was := k.caught + k.caught = caught + k.mu.Unlock() + if caught && (!was || m.Subject() != mdseat.SubjectPinned) { + k.judge() // on a freed, on a candidate's heads replaced, on pins-stated, and on being caught up + } +} + +// apply takes one event's body. +func (k *pinKeeper) apply(subject string, data []byte) error { + switch subject { + case mdseat.SubjectPinned, mdseat.SubjectFreed: + var p mdseat.Pins + if err := json.Unmarshal(data, &p); err != nil || p.Record == "" { + k.logf("%s holds no pins naming a record, and is not taken: %v", subject, err) + return nil + } + if subject == mdseat.SubjectPinned { + return k.pin(p) + } + return k.free(p) + case mdseat.SubjectPinsStated: + var s mdseat.PinsStatedEvent + if err := json.Unmarshal(data, &s); err != nil || s.At.IsZero() { + k.logf("%s holds no at, and is not taken: %v", subject, err) + return nil + } + return k.stated(s.At) + } + return nil // heads-replaced: a candidate's revision stops being current, judged after +} + +// judge removes every revision neither pinned nor current, when the rule allows a removal at all. +func (k *pinKeeper) judge() { + k.mu.Lock() + caught := k.caught + k.mu.Unlock() + if !caught { + return + } + ok, err := k.statedSinceMade() + if err != nil { + k.logf("the pins are not judged: %v", err) + return + } + if !ok { + return // no pins-stated applied since the state was made: a pin said before may not be heard yet + } + if err := k.remove(); err != nil { + k.logf("the revisions nothing pins are not removed: %v", err) + } +} + +// remove removes each kept revision that is not current and that no record pins, and then the entries left unlisted. +func (k *pinKeeper) remove() error { + names, err := k.objects.Names(k.ctx) + if err != nil { + return err + } + var kept []int64 + for _, n := range names { + if r, ok := pinRevision(n); ok { + kept = append(kept, r) + } + } + if len(kept) == 0 { + return nil + } + current := slices.Max(kept) + listed := map[string]bool{} + removed := 0 + for _, r := range kept { + raw, _, err := k.st.get(pinKey(r)) + if err != nil { + return err + } + var v PinsValue + if raw != nil { + if err := json.Unmarshal(raw, &v); err != nil { + return fmt.Errorf("the pins of %d do not read: %w", r, err) + } + } + if r == current || len(v.Records) > 0 { + rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r) + if err != nil { + return err + } + for _, e := range rev.Entries { + listed[e.Digest] = true + } + continue + } + if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil { + return fmt.Errorf("revision %d: %w", r, err) + } + if raw != nil { + if err := k.st.drop(pinKey(r)); err != nil { + return fmt.Errorf("the pins of %d: %w", r, err) + } + } + removed++ + } + if removed == 0 { + return nil + } + // A run's candidate kept now lists its entries too, until its group-ended drops it (A-201, A-235). + for _, n := range names { + group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n) + if !ok { + continue + } + rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads) + if err != nil { + return err + } + for _, e := range rev.Entries { + listed[e.Digest] = true + } + } + for _, n := range names { + digest, ok := strings.CutPrefix(n, "entry.") + if !ok || listed[digest] { + continue + } + e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest) + if err != nil { + return err + } + if k.inUse != nil && k.inUse(e.Module, e.Fp) { + continue + } + if err := k.objects.Delete(k.ctx, n); err != nil { + return fmt.Errorf("the entry %s: %w", digest, err) + } + } + k.logf("%d assignment graph revisions no record pins and not current are removed", removed) + return nil +} + +// hearPins registers the seat mesh-delivery's four events on the assigner's hub. +func hearPins(hub *process.EventHub, k *pinKeeper) error { + for _, s := range []string{mdseat.SubjectPinned, mdseat.SubjectFreed, mdseat.SubjectHeadsReplaced, mdseat.SubjectPinsStated} { + if err := hub.Handle(s, k.hear); err != nil { + return fmt.Errorf("%s: %w", s, err) + } + } + return nil +} + +// pinsTools are assigner_pins and assigner_unpin over the runtime's states. +func pinsTools(st store, ask func(string, any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{{Name: pinsTool, Description: "the assignment graph revisions the delivery records pin (design 47 G §579): per revision " + + "the records that pin it and whether it was missing when pinned, the newest revision kept (current, never removed), when the state " + + "was made, the last pins-stated applied, and the state's fill against its cap of 4 MiB.", + Run: func(map[string]any) (any, error) { return readPins(st) }}, + {Name: unpinTool, Description: "Remove one record from a revision's pins, for a freed that was lost (design 47 G §579); written in the " + + "hand-act log first. The removal rule then applies.", + Input: schema(map[string]string{"revision": "the assignment graph revision's number", "record": "the delivery record's id", + "why": "why it is removed by hand"}, []string{"revision", "record", "why"}), + Run: func(a map[string]any) (any, error) { + record, _ := a["record"].(string) + why, _ := a["why"].(string) + return unpin(st, ask, fmt.Sprint(a["revision"]), record, why, stdio.CallerOf(a)) + }}} +} + +// readPins is what assigner_pins answers. +func readPins(st store) (any, error) { + ks, err := st.keys(pinsState) + if err != nil { + return nil, err + } + sort.Strings(ks) + revisions := map[string]PinsValue{} + out := map[string]any{"revisions": revisions} + var newest int64 + bytes := 0 + for _, key := range ks { + raw, _, found, err := st.get(pinsState, key) + if err != nil || !found { + continue + } + bytes += len(key) + len(raw) + if r, ok := pinRevision(key); ok { + var v PinsValue + if json.Unmarshal(raw, &v) == nil { + revisions[key] = v + } + newest = max(newest, r) + continue + } + out[key] = json.RawMessage(raw) + } + out["fill"] = meshwants.Fill{Store: pinsState, Bytes: int64(bytes), Cap: pinsCapBytes} + if newest > 0 { + out["newest-pinned"] = newest + } + return out, nil +} + +// unpin removes one record from one revision's list, the hand act recorded first. +func unpin(st store, ask func(string, any) (json.RawMessage, error), revision, record, why, by string) (any, error) { + r, err := strconv.ParseInt(strings.TrimPrefix(revision, "trunk."), 10, 64) + switch { + case err != nil || r <= 0: + return nil, fmt.Errorf("revision %q is no assignment graph revision's number", revision) + case record == "" || why == "": + return nil, errors.New("name the record and say why") + case by == "" || by == stdio.Unproven: + return nil, errors.New("an unpin names who asks it, and the runtime proved no caller") + } + key := pinKey(r) + raw, _, found, err := st.get(pinsState, key) + if err != nil { + return nil, err + } + var v PinsValue + if found { + if err := json.Unmarshal(raw, &v); err != nil { + return nil, err + } + } + i := slices.Index(v.Records, record) + if i < 0 { + return nil, fmt.Errorf("the record %s does not pin revision %d: %s lists its pins", record, r, pinsTool) + } + taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: unpinTool + " " + key + " " + record, + Why: why + " (" + by + ")", Cause: "unpin"}) + if err != nil { + return nil, fmt.Errorf("the hand act is not recorded, so nothing is removed: %w", err) + } + v.Records = slices.Delete(v.Records, i, i+1) + value, _ := json.Marshal(v) + if err := st.put(pinsState, key, value); err != nil { + return nil, err + } + return map[string]any{"revision": r, "unpinned": record, "records": v.Records, "hand-act": taken.Log}, nil +} + +// watchUnpins judges again when a revision's pins change in the state itself, as assigner_unpin writes them from the tool +// process: the assigner's own state, heard as it changes, never on a timer. +func (k *pinKeeper) watchUnpins(kv jetstream.KeyValue) { + w, err := kv.WatchAll(k.ctx, jetstream.UpdatesOnly()) + if err != nil { + k.logf("the state %s is not watched, so a hand unpin is judged at the next freed or revision: %v", pinsState, err) + return + } + go func() { + defer func() { _ = w.Stop() }() + for e := range w.Updates() { + if e == nil { + continue + } + if _, ok := pinRevision(e.Key()); ok && e.Operation() == jetstream.KeyValuePut { + k.judge() + } + } + }() +} + +// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now. +func inUseOf(w *world) func(module, fp string) bool { + return func(module, fp string) bool { + for _, want := range w.wantsOf()[module] { + if want.Fp == fp { + return true + } + } + for _, mods := range w.runningNow() { + if r, ok := mods[module]; ok && r.Fingerprint == fp { + return true + } + } + return false + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins_test.go new file mode 100644 index 00000000..de2a17d8 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins_test.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + "time" + + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + + "mesh-delivery-assigner/internal/assign" + "mesh-delivery-assigner/internal/assigngraph" +) + +func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil } + +// The pins of the assignment graph revisions (design 47 G §153, §178, §579; A-169): a revision two records pin stays until +// both free it; a record pinning twice pins once; the current revision is never removed; nothing is removed before a +// pins-stated said after the state was made, nor while the consumer is behind; a revision no longer held is pinned missing, +// with the finding pin-missing; an entry another kept revision lists, or a want or a running build names, stays. +func TestPinsKeepARevisionWhileARecordPinsIt(t *testing.T) { + objects := memObjects{} + entry := func(module, fp string) string { + e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp} + d := assigngraph.EntryDigest(e) + raw, _ := json.Marshal(e) + objects[assigngraph.EntryObject(d)] = raw + return d + } + shared, old, running := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1") + revision := func(n int64, digests ...string) { + rev := meshwants.AssignmentRevision{Revision: n} + for _, d := range digests { + rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d}) + } + raw, _ := json.Marshal(rev) + objects[assigngraph.TrunkObject(n)] = raw + } + // A run's candidate lists an entry no trunk revision lists (A-201): it stays while the candidate is kept. + cand := entry("d", "fp-d2") + candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1", + Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}}) + objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw + revision(1, shared, old, running) + revision(2, shared, old) + revision(3, shared) + var found []mcseat.Finding + clock := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC) + k := &pinKeeper{ctx: context.Background(), st: newMemEnds(), objects: objects, now: func() time.Time { return clock }, + logf: t.Logf, + inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" }, + findings: assign.Findings{Put: func(f mcseat.Finding) error { found = append(found, f); return nil }}} + if err := k.made(); err != nil { + t.Fatal(err) + } + say := func(subject string, body any) { + t.Helper() + raw, _ := json.Marshal(body) + if err := k.apply(subject, raw); err != nil { + t.Fatal(err) + } + k.judge() + } + pins := func(record string, revs ...int64) mdseat.Pins { return mdseat.Pins{Record: record, Assignment: revs} } + has := func(n int64) bool { _, ok := objects[assigngraph.TrunkObject(n)]; return ok } + + k.caught = true + say(mdseat.SubjectPinned, pins("r1", 1)) + say(mdseat.SubjectPinned, pins("r1", 1)) + say(mdseat.SubjectPinned, pins("r2", 1)) + say(mdseat.SubjectFreed, pins("r9", 2)) // nothing pins 2, but no pins-stated since made: nothing removed + if !has(2) { + t.Fatal("revision 2 was removed before a pins-stated said after the state was made") + } + k.caught = false + say(mdseat.SubjectPinsStated, mdseat.PinsStatedEvent{At: clock.Add(time.Minute)}) + if !has(2) { + t.Fatal("revision 2 was removed while the consumer was behind") + } + k.caught = true + say(mdseat.SubjectFreed, pins("r1", 1)) + if has(2) { + t.Error("revision 2, neither pinned nor current, is kept") + } + if !has(1) || !has(3) { + t.Error("revision 1 (pinned by r2) or 3 (current) was removed") + } + raw, _, _ := k.st.get(pinKey(1)) + if string(raw) != `{"records":["r2"]}` { + t.Errorf("the pins of 1 = %s, want r2 alone (r1 pinned twice pins once, and freed once)", raw) + } + say(mdseat.SubjectFreed, pins("r2", 1)) + if has(1) { + t.Error("revision 1 is kept once both records freed it") + } + if _, ok := objects[assigngraph.EntryObject(old)]; ok { + t.Error("the entry no kept revision lists and nothing names is kept") + } + if _, ok := objects[assigngraph.EntryObject(running)]; !ok { + t.Error("the entry of a running build is removed") + } + if _, ok := objects[assigngraph.EntryObject(shared)]; !ok { + t.Error("the entry the current revision lists is removed") + } + if _, ok := objects[assigngraph.EntryObject(cand)]; !ok { + t.Error("the entry a run's kept candidate lists is removed") + } + say(mdseat.SubjectHeadsReplaced, mdseat.HeadsReplacedEvent{}) + if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; !ok { + t.Error("the gate's heads-replaced dropped a run's candidate") + } + say(mdseat.SubjectPinned, pins("r3", 2)) + raw, _, _ = k.st.get(pinKey(2)) + if string(raw) != `{"records":["r3"],"missing":true}` || len(found) != 1 || found[0].Kind != pinMissingKind || found[0].ID != "2" { + t.Errorf("a pin of a revision no longer held = %s, findings %+v", raw, found) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 63951fc7..cedaedfa 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -151,8 +151,12 @@ type pipelineEnds struct { // firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design // 47 G §14). firsts map[string]string + // runEnded is told a run on request's group-ended, its group `run-`, to drop the run's candidate (A-235); nil drops nothing. + runEnded func(group string) error // onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195). onBranches func([]string) + // trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none. + trunk func() string } func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds { @@ -225,7 +229,10 @@ func (e *pipelineEnds) branchOf(module, placed string) string { if b := e.firsts[module]; b != "" { return b } - return trunkBranch + if e.trunk != nil { + return e.trunk() + } + return "" } // hearPipeline takes a pipeline-ended. A pipeline of a group writes nothing on its own end, and a run on request delivers @@ -246,7 +253,8 @@ func (e *pipelineEnds) hearPipeline(m jetstream.Msg) { _ = m.Ack() } -// hearGroup takes a group-ended: each member's end, passed only if the group passed. +// hearGroup takes a group-ended: each member's end, passed only if the group passed. A run on request's (group `run-`) delivers +// nothing: it drops the run's candidate assignment graph (A-235). func (e *pipelineEnds) hearGroup(m jetstream.Msg) { g, err := mprc.GroupEnded(m.Data()) if err != nil { @@ -254,6 +262,17 @@ func (e *pipelineEnds) hearGroup(m jetstream.Msg) { _ = m.Ack() return } + if strings.HasPrefix(g.Group, runGroupPrefix) { + if e.runEnded != nil { + if err := e.runEnded(g.Group); err != nil { + e.logf("%s: %v; it is heard again", m.Subject(), err) + _ = m.Nak() + return + } + } + _ = m.Ack() + return + } for _, p := range g.Pipelines { if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil { e.logf("%s: %v; it is heard again", m.Subject(), err) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index 19a2d4a4..1a1d594a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -20,8 +20,8 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" @@ -55,6 +55,19 @@ func (m *memEnds) swap(key string, v []byte, rev uint64) error { return nil } +// heardMsg is one message as a handler of the hub is handed it, saying whether it was acknowledged or given back. +type heardMsg struct { + jetstream.Msg + data []byte + subject string + acked, naked bool +} + +func (m *heardMsg) Data() []byte { return m.data } +func (m *heardMsg) Subject() string { return m.subject } +func (m *heardMsg) Ack() error { m.acked = true; return nil } +func (m *heardMsg) Nak() error { m.naked = true; return nil } + func pipe(module, branch string, source, count int64, fp, outcome string, steps ...mpr.StepResult) mpr.PipelineEnded { return mpr.PipelineEnded{Module: module, Branch: branch, Source: source, EndCount: count, Fingerprint: fp, Outcome: outcome, Steps: steps, Pipeline: branch + "." + module, Commit: "c"} @@ -112,6 +125,35 @@ func TestAWantFollowsAPassedPipeline(t *testing.T) { t.Errorf("%s: may want = %v, want %v", c.name, got, c.may) } } + // A run on request's group-ended (group run-, A-235) writes no want and drops the run's candidate; a drop that fails is + // heard again. + { + e := newPipelineEnds(newMemEnds(), func() {}, t.Logf) + e.on.Store(true) + var dropped []string + fail := false + e.runEnded = func(group string) error { + dropped = append(dropped, group) + if fail { + return fmt.Errorf("the bucket is not open") + } + return nil + } + raw, _ := json.Marshal(mpr.GroupEnded{V: 1, Branch: "main", Group: "run-7", Outcome: mpr.GroupEndedOutcomePassed, + Pipelines: []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Run = "7"; return p }()}}) + m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.group-ended.x"} + e.hearGroup(m) + if e.mayWant("web", fp1, []string{"main"}, nil) || !slices.Equal(dropped, []string{"run-7"}) || !m.acked { + t.Errorf("a run's group-ended: may want %v, dropped %v, acked %v; want no want, run-7 dropped, acked", + e.mayWant("web", fp1, []string{"main"}, nil), dropped, m.acked) + } + fail = true + m = &heardMsg{data: raw, subject: m.subject} + e.hearGroup(m) + if !m.naked { + t.Error("a run's candidate not dropped is not heard again") + } + } // A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it). e := newPipelineEnds(newMemEnds(), func() {}, t.Logf) e.on.Store(true) @@ -154,9 +196,7 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) putModuleRaw(p.w, trunkBranch, "web", mod) p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} - bkey, _ := builds.BuilderBuildKey("web", fp) - built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) - p.w.put(buckets.Builds, bkey, built) + endDone(t, p.w, "web", fp, mdigest) gkey, _ := grantsstate.PartGrantKey("web", fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) p.w.put(buckets.Grants, gkey, grant) @@ -165,10 +205,11 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { p.w.put(buckets.Have, "g14.web", have) composes(t, p.w, "g14", digest) placeOn(t, p.w, "g14", "web") - on, _ := presenceValue(true) - p.w.put(buckets.Presence, "g14", on) + on := true + p.w.heardPresence("g14", !on, time.Now()) if c.follow { p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.trunk = p.w.trunkName p.ends.on.Store(true) if c.passed { _ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true) @@ -196,6 +237,7 @@ func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) { st := newMemEnds() woken := 0 e := newPipelineEnds(st, func() { woken++ }, t.Logf) + e.trunk = func() string { return trunkBranch } _ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false) _ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true) if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) { @@ -225,12 +267,13 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { p := newPasser(kvSet{buckets.Wants: wantsKV, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now()) p.say = func(oc.Condition) error { return nil } p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.trunk = p.w.trunkName p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false) digest := "sha256:" + strings.Repeat("c", 64) - on, _ := presenceValue(true) + on := true for _, node := range []string{"g14", "ace"} { composes(t, p.w, node, digest) - p.w.put(buckets.Presence, node, on) + p.w.heardPresence(node, !on, time.Now()) } for node, branch := range map[string]string{"g14": "", "ace": "beta"} { pl := mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}} @@ -251,9 +294,7 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) putModuleRaw(p.w, branch, "web", mod) p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} - bkey, _ := builds.BuilderBuildKey("web", f) - built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) - p.w.put(buckets.Builds, bkey, built) + endDone(t, p.w, "web", f, mdigest) gkey, _ := grantsstate.PartGrantKey("web", f) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) p.w.put(buckets.Grants, gkey, grant) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index dab41719..30f5d9e3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -14,7 +14,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" @@ -107,6 +106,7 @@ func (p *passer) puts(ctx context.Context) { } p.placementPuts(ctx) all := p.putsByModule() + defer func() { p.putWaits(ctx, p.putsByModule()) }() if len(all) == 0 { return } @@ -184,14 +184,8 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue. w.Assigned = true } } - if bkey, err := builds.BuilderBuildKey(module, put.FP); err == nil { - if raw := p.w.value(buckets.Builds, bkey); raw != nil { - if b, err := builds.DecodeBuilderBuild(raw); err == nil { - w.Build, w.BuildWords = b.State, b.Error - } else { - w.Build = "unreadable: " + err.Error() - } - } + if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state + w.Build, w.BuildWords = e.Outcome, e.Error } w.Composed, w.Granted = p.composedFor(node, module, put.FP) if d := w.Composed; d != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/putwaits.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/putwaits.go new file mode 100644 index 00000000..5aca048a --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/putwaits.go @@ -0,0 +1,116 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "time" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + + "mesh-delivery-assigner/internal/assignment" + "mesh-delivery-assigner/internal/buckets" + "mesh-delivery-assigner/internal/puts" +) + +// A put without hold that stands is said as a wait (design 47 G §498, main file *A put without hold that stands*): when its +// want is written the assigner says `wait-held` on the seat mesh-wants, step put, with the node, the module, the fingerprint +// and since when, bound 30 minutes; and `wait-ended` when it ends: ran when its node runs it, released by assigner_release, +// superseded by a put asked again under its key. The watchman times it; a held put waits on nobody and is said as none. + +// putWaitBound is the bound of a put without hold's wait. +const putWaitBound = 30 * time.Minute + +// Wait-ended's how of a put's wait. +const ( + putWaitRan = "ran" + putWaitReleased = "released" + putWaitSuperseded = "superseded" +) + +// putWaits says whether p is a put that waits: one without hold whose want is written and not yet run. +func putWaits(p puts.Put) bool { + return !p.Hold && p.Step == puts.Wanted +} + +// putWaitBody is the wait of the put p of module on node. +func putWaitBody(node, module string, p puts.Put) meshwants.Wait { + since := p.StandsSince().UTC() + return meshwants.Wait{Step: meshwants.WaitStepPut, Owner: assignment.Module, Node: node, Module: module, Fingerprint: p.FP, + Since: since, ClockFrom: &since, Bound: putWaitBound.String(), + Words: fmt.Sprintf("the put of %s at %s on %s waits for %s to run it", module, shortFP(p.FP), node, node)} +} + +// putWaiting is a put whose wait-held was said, as the state put-waits keeps it under the put's key (design 47 G A-204, +// A-211): its identity and the put as it was then. +type putWaiting struct { + ID string `json:"id"` + Put puts.Put `json:"put"` +} + +// saidPutWaits is every put wait the state put-waits holds, by key; a value that does not decode is said and left. +func (p *passer) saidPutWaits() map[string]putWaiting { + out := map[string]putWaiting{} + for key, value := range p.w.read(buckets.PutWaits) { + var w putWaiting + if err := json.Unmarshal(value, &w); err != nil || w.ID == "" { + p.logf("%s %s holds no put wait, left: %v", buckets.PutWaits, key, err) + continue + } + out[key] = w + } + return out +} + +// putWaits says wait-ended of each put whose said wait the state put-waits keeps and that no longer waits, whether it ended +// while this process ran or while the assigner was stopped (A-204), and wait-held of each put that waits and was not said; +// one not said, or not kept, is said at the next pass. Without the state no put's wait is said. +func (p *passer) putWaits(ctx context.Context, all map[string]map[string]puts.Put) { + if p.kvs[buckets.PutWaits] == nil { + return + } + waiting := p.saidPutWaits() + for key, was := range waiting { + node, module, err := puts.ParseKey(key) + if err != nil { + continue + } + cur, there := all[module][node] + same := there && identity(key, cur) == was.ID + if same && putWaits(cur) { + continue + } + how := putWaitSuperseded + if same && cur.Step == puts.Done { + how = putWaitRan + } else if same && cur.Step == puts.Released { + how = putWaitReleased + } + body := putWaitBody(node, module, was.Put) + body.How = how + if err := p.wantsSay(meshwants.EventWaitEnded, body, was.ID+".put."+how); err != nil { + p.logf("the end of the put %s's wait is not said, and is said at the next pass: %v", key, err) + continue + } + p.remove(ctx, buckets.PutWaits, key) + delete(waiting, key) + } + for module, byNode := range all { + for node, put := range byNode { + key, err := puts.Key(node, module) + if err != nil || !putWaits(put) { + continue + } + id := identity(key, put) + if w, said := waiting[key]; said && w.ID == id { + continue + } + if err := p.wantsSay(meshwants.EventWaitHeld, putWaitBody(node, module, put), id+".put"); err != nil { + p.logf("the put %s's wait is not said, and is said at the next pass: %v", key, err) + continue + } + kept := putWaiting{ID: id, Put: put} + p.put(ctx, buckets.PutWaits, key, func() ([]byte, error) { return json.Marshal(kept) }) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/putwaits_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/putwaits_test.go new file mode 100644 index 00000000..1106d2a7 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/putwaits_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "testing" + "time" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + + "mesh-delivery-assigner/internal/buckets" + "mesh-delivery-assigner/internal/puts" +) + +// Design 47 G §498 and the main file's *A put without hold that stands*: a put without hold whose want is written is said as +// wait-held, step put, with its node, module, fingerprint, since and a 30-minute bound, and wait-ended when it runs, is +// released or is put again; a held put is said as no wait; assigner_puts answers since and bound. A-204, A-211: the said +// waits are kept in the state put-waits, so a put that waits at the start was said before and is not said again, and one +// that ran while the assigner was stopped gets its wait-ended at the next start. +func TestAPutWithoutHoldIsSaidAsAWait(t *testing.T) { + at := time.Date(2026, 11, 12, 8, 0, 0, 0, time.UTC) + kept := &memKV{values: map[string][]byte{}} + var events []string + var bodies []meshwants.Wait + start := func() *passer { // a process's start: a fresh world that hands what the state holds + p := newPasser(kvSet{buckets.PutWaits: kept}, at) + for k, v := range kept.values { + p.w.put(buckets.PutWaits, k, v) + } + p.acts = newPutActs(&fakeBus{calls: make(chan string, 8)}, func() {}, func(string, ...any) {}) + p.publishWants = func(event string, body any, id string) error { + events, bodies = append(events, event), append(bodies, body.(meshwants.Wait)) + return nil + } + return p + } + ctx := context.Background() + asked := puts.Put{FP: fpTest, Why: "a try", At: at}.Said(at, puts.Asked, "asked") + wanted := asked.Said(at.Add(time.Minute), puts.Wanted, "the want is written").Said(at.Add(2*time.Minute), puts.Wanted, "its check failed") + held := wanted + held.Hold = true + p := start() + p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}}) + if len(events) != 1 || events[0] != meshwants.EventWaitHeld || kept.values["ace.nats"] == nil { + t.Fatalf("a put that waits said %v, kept %q", events, kept.values) + } + events, bodies = nil, nil + // At the next start the put still waits: it was said before, nothing is said of it. + p = start() + p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}}) + if len(events) != 0 { + t.Fatalf("a put waiting at the start was said again: %v", events) + } + // It ran while the assigner was stopped: at the next start, wait-ended, ran, and the state forgets it. + p = start() + p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted.Said(at.Add(3*time.Minute), puts.Done, "runs")}}) + if len(events) != 1 || events[0] != meshwants.EventWaitEnded || bodies[0].How != putWaitRan || bodies[0].Step != meshwants.WaitStepPut || + kept.values["ace.nats"] != nil { + t.Fatalf("a put that ran while stopped said %v %+v, kept %q", events, bodies, kept.values) + } + events, bodies = nil, nil + // A new put's want is written: wait-held, since the want was written, bound 30m; a held put says nothing. + again := wanted + again.At = at.Add(time.Hour) + p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}}) + if len(events) != 1 || events[0] != meshwants.EventWaitHeld { + t.Fatalf("a put's written want said %v", events) + } + b := bodies[0] + if b.Node != "ace" || b.Module != "nats" || b.Fingerprint != fpTest || !b.Since.Equal(at.Add(time.Minute)) || b.ClockFrom == nil || + !b.ClockFrom.Equal(b.Since) || b.Bound != "30m0s" || b.Owner != "mesh-delivery-assigner" { + t.Errorf("the put's wait is %+v", b) + } + // Said once. + p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}}) + if len(events) != 1 { + t.Errorf("a put's wait was said again: %v", events) + } + // Put again under its key: the first wait ends superseded, the new one is held. + newer := asked + newer.At = at.Add(2 * time.Hour) + p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": newer}}) + if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != putWaitSuperseded || bodies[1].Fingerprint != fpTest { + t.Errorf("a put asked again said %v %+v", events, bodies) + } + // The answer of assigner_puts. + if a := answerOf("ace", "nats", wanted); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "30m0s" { + t.Errorf("a put that waits answers since %v, bound %q", a.Since, a.Bound) + } + if a := answerOf("ace", "web", held); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "" { + t.Errorf("a held put answers since %v, bound %q", a.Since, a.Bound) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 7664c4fb..1911af8f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -17,9 +17,9 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/seatlist" @@ -72,14 +72,11 @@ func (s *putTools) hear(key string, value []byte) { s.mu.Unlock() } -// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits. -var ( - fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`) - prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`) -) +// fullFP is a fingerprint of the new form, whole. +var fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`) -// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start -// of one the builder built, which must name one build alone. +// version reads the version a put names: newest (the indexer's fingerprint at the trunk) or a whole fingerprint (design 47 G A-181: +// a put names the whole fingerprint, the builder's builds being no bucket to list). func (s *putTools) version(module, version string) (string, error) { v := strings.ToLower(strings.TrimSpace(version)) switch { @@ -103,29 +100,8 @@ func (s *putTools) version(module, version string) (string, error) { return m.Fp2, nil case fullFP.MatchString(v): return v, nil - case prefix.MatchString(v): - start := strings.TrimPrefix(v, meshcatalogclient.SourcePrefix2) - keys, err := s.st.keys(builds.BuilderModule + ".builds") - if err != nil { - return "", fmt.Errorf("the builder's builds cannot be listed: %w", err) - } - var found []string - for _, k := range keys { - m, fp, err := builds.ParseBuilderBuildKey(k) - if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2), start) { - found = append(found, fp) - } - } - switch len(found) { - case 0: - return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start) - case 1: - return found[0], nil - } - sort.Strings(found) - return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", ")) } - return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version) + return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) whole", version) } // putAnswer is what assigner_put and assigner_puts answer of one put. @@ -137,6 +113,19 @@ type putAnswer struct { // Older says a put that has not ended stands at an older version than the newest build, and which that is: it pins the // node's want until it ends or is released. Older string `json:"older-than-newest,omitempty"` + // Since is when the put came to stand at its step, held or not; Bound is the bound of a put without hold whose want is + // written, the wait the watchman times and seeds from here (design 47 G §498, main file *A put without hold that stands*). + Since time.Time `json:"since"` + Bound string `json:"bound,omitempty"` +} + +// answerOf is what assigner_put and assigner_puts answer of the put p of module on node. +func answerOf(node, module string, p puts.Put) putAnswer { + a := putAnswer{Node: node, Module: module, Put: p, Since: p.StandsSince()} + if putWaits(p) { + a.Bound = putWaitBound.String() + } + return a } // doPut is the tool assigner_put. @@ -168,7 +157,9 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) { } else if found { if before, err := puts.Decode(raw); err == nil && !puts.Ended(before.Step) { if before.FP == fp && before.Hold == hold { - return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil + a := answerOf(node, module, before) + a.Follow = followPut(node, module) + return a, nil } words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step) } @@ -182,7 +173,9 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) { return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err) } s.hear(key, data) - return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil + a := answerOf(node, module, s.await(key, p)) + a.Follow = followPut(node, module) + return a, nil } // await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin. @@ -217,6 +210,12 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) { if arg(args, "how") != "" { return s.doReleaseWait(args, by) // a wait on an outside cause (outside_tools.go, design 47 G §706) } + if step := arg(args, "step"); step != "" { + if step != meshwants.WaitStepDelivery { + return nil, fmt.Errorf("step is %s, the one wait released by its step, not %q", meshwants.WaitStepDelivery, step) + } + return s.doReleaseDelivery(args, by) // a dependent's delivery wait (delivered.go, design 47 G §376, A-246) + } node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why") key, err := puts.Key(node, module) if err != nil { @@ -248,7 +247,7 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) { return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err) } s.hear(key, data) - return putAnswer{Node: node, Module: module, Put: p}, nil + return answerOf(node, module, p), nil } // doPuts is the tool assigner_puts. @@ -274,7 +273,7 @@ func (s *putTools) doPuts(args map[string]any) (any, error) { if err != nil { continue } - a := putAnswer{Node: node, Module: module, Put: p} + a := answerOf(node, module, p) if !puts.Ended(p.Step) { if !asked { asked = true @@ -349,10 +348,12 @@ func (s *putTools) tools() []stdio.Tool { "rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.", Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put", "how": "with how, the wait of the delivery on an outside cause (assigner_waits) is released, not a put: go-on (the module counts healthy there), " + - "failed (the failing check is the delivery's) or rejudge (judge it again now); written in the hand-act log first"}, - []string{"node", "module", "why"}), + "failed (the failing check is the delivery's) or rejudge (judge it again now); written in the hand-act log first", + "step": "delivery: the module's wait for its dependency's delivery (assigner_waits) is released, its want written now; no machine is named; written in the hand-act log first"}, + []string{"module", "why"}), Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }}, s.waitsTool(), + s.deliveredAgainTool(), {Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " + "its time and words. Narrowed by machine or module. Read-only.", Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil), @@ -417,6 +418,7 @@ func serveTools() error { // What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it. tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now)) tools = append(tools, writtenTools(runtimeStore{})...) + tools = append(tools, pinsTools(runtimeStore{}, stdio.Ask)...) tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go index ec642a9c..35a3c35f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go @@ -1,7 +1,6 @@ package main import ( - "mesh-delivery-assigner/internal/buckets" "slices" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" @@ -94,22 +93,14 @@ func (p *passer) servesOf(module, fp string) ([]string, bool) { return graph.ServesOf(m), true } -// manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests), -// and whether it could be read. +// manifestOf is the manifest of module's build at fp (the build's end names its digest, A-181, which the indexer's manifests +// hold), and whether it could be read. func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) { - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { + e, ok := p.w.ends.endOf(module, fp) + if !ok || e.Outcome != builds.BuildDone || e.Manifest == "" { return catalogue.Manifest{}, false } - raw := p.w.value(buckets.Builds, key) - if raw == nil { - return catalogue.Manifest{}, false - } - b, err := builds.DecodeBuilderBuild(raw) - if err != nil || b.Manifest == "" { - return catalogue.Manifest{}, false - } - got := p.w.manifest(b.Manifest) + got := p.w.manifest(e.Manifest) if got.err != "" { return catalogue.Manifest{}, false } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go index f52e96e7..4eb76b26 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go @@ -3,15 +3,19 @@ package main import ( "context" "encoding/json" + "fmt" "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" "slices" + "sync/atomic" "time" "github.com/nats-io/nats.go/jetstream" mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/process" @@ -122,6 +126,12 @@ type catalogViews struct { hub *process.EventHub h *halves + // candidates is told each candidate's catalog-revision (A-201), once the passer that keeps its assignment graph is made; + // before that a candidate's revision is said not taken. + candidates atomic.Pointer[func(context.Context, meshcatalog.CatalogRevisionEvent) error] + // answers is the picture of the teller's answers (design 47 G A-175, A-178): its first view through teller_read_answers, + // then its views family; nil when it is not followed. + answers *views.Picture } // firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait @@ -136,7 +146,24 @@ func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settle reader.Findings = func() views.WireWriter { return iv.writer } } iv.h = &halves{w: w, call: iv.call, reader: reader, wake: wake, logf: iv.logf, ctx: ctx, settled: settled} - iv.h.follow(trunkBranch) + if err := iv.h.follow(); err != nil { + iv.logf("the catalog half is not followed, so no module is placed until the assigner starts again: %v", err) + } + // Each candidate's catalog graph revision, heard on the reader's one subscription of catalog-revision (A-201, A-185). + if err := mcc.WireCandidatesProcess(reader, func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error { + if f := iv.candidates.Load(); f != nil { + return (*f)(ctx, ev) + } + return fmt.Errorf("the candidate %s's revision %d is heard before the assigner can keep its assignment graph", ev.Group, ev.Revision) + }); err != nil { + iv.logf("no candidate's catalog graph revision is heard, so no run's assignment graph is kept until the assigner starts again: %v", err) + } + // The teller's answers to the assigner's asks, followed as every owner's state is (A-175), never its bucket. + if pic, err := reader.Follow(buckets.TellerModule, buckets.AnswersState, buckets.TellerModule+"."+buckets.AnswersReadTool); err != nil { + iv.logf("the teller's answers are not followed, so no ask is answered until the assigner starts again: %v", err) + } else { + iv.answers = pic + } reader.Start(ctx, iv.js, iv.account, iv.status) return 1 } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index e591751b..4f65c6d4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -12,13 +12,11 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - wdc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" @@ -45,12 +43,19 @@ type world struct { ends builtEnds // released are the catalog's version-released heard, by module and source revision (buildended.go). released releasedVersions + // delivered are the assigner's own version-delivered heard and said (delivered.go). + delivered deliveredVersions + // edges are, by watched branch and module, the edges into the module's artifacts its catalog entry names (§374's kinds). + edges map[string]map[string][]meshcatalog.Edge // assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk // mmac.Keys; research 072 row 37): the one reader of its keys in the assigner. assignmentKeys *mmac.Keys - // sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its - // history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1). - sourceGroups map[uint64]string + // first is the first watched branch the catalog half is followed of, the trunk's: the branch of the trunk-only want, read from the + // branches the registrations name, before the first registrations from those the gate's `source` answers (design 47 G §507, §532), + // and never fixed in code (§7). + first string + // presence is, by machine, the last machine-away or machine-back the watchman said of it (design 47 G §389). + presence map[string]presenceSaid // running is, by `.`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's // durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound. running map[string]nodeengine.Running @@ -77,7 +82,7 @@ type parsed struct { func newWorld() *world { return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}, meta: map[string]entryMeta{}, assignmentKeys: mmac.NewKeys(), - sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{}, + running: map[string]nodeengine.Running{}, modules: map[string]map[string]meshcatalog.ModulesByModuleValue{}, graphs: map[string]map[string]graph.Node{}} } @@ -94,44 +99,6 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t if bucket == plc.AssignmentsBucket { _, _ = w.assignmentKeys.Put(key, value, revision, false) // put took it already; this is its revision } - if bucket == buckets.Source && key == mdc.SourceKey(trunkBranch) { - if s, err := mdc.DecodeSource(value); err == nil && s.Group != "" { - w.sourceGroups[revision] = s.Group - } - } -} - -// release is the release whose Approve carries module's build fp (ADR 0601 §1), and whether one does: the trunk's value of -// the module names fp, the source revision at which that fingerprint last changed (FPSrc, never the Src of its last merge, -// which a recompute with no merge keeps) names the release that moved it, and the gate's merge of that release, which it -// writes on the operator's Approve, holds the module's own trunk merge among its merged pull requests. Any of them -// unknown: none, and the move is asked of a person. -func (w *world) release(module, fp string) (string, bool) { - t, ok := w.trunk()[module] - if !ok || fp == "" || t.Fp2 != fp || t.FpSrc == 0 || t.Commit == "" { - return "", false - } - w.mu.Lock() - group := w.sourceGroups[uint64(t.FpSrc)] - w.mu.Unlock() - key, err := mdc.MergesKey(group) - if group == "" || err != nil { - return "", false - } - raw := w.value(buckets.Merges, key) - if raw == nil { - return "", false - } - merge, err := mdc.DecodeMerge(raw) - if err != nil { - return "", false - } - for _, pull := range merge.Pulls { - if pull.State == string(mdc.MergeMerged) && pull.Commit == t.Commit && trunk.SameRepository(pull.Repository, t.Repository) { - return group, true - } - } - return "", false } // metaOf is the revision and time of the value a bucket holds at key, as the watch said them. @@ -267,8 +234,8 @@ func (w *world) branches() []string { w.mu.Lock() defer w.mu.Unlock() out := slices.Sorted(maps.Keys(w.modules)) - if i := slices.Index(out, trunkBranch); i > 0 { - out = slices.Insert(slices.Delete(out, i, i+1), 0, trunkBranch) + if i := slices.Index(out, w.first); i > 0 { + out = slices.Insert(slices.Delete(out, i, i+1), 0, w.first) } return out } @@ -307,6 +274,27 @@ func (w *world) dropModule(branch, module string) { defer w.mu.Unlock() delete(w.modules[branch], module) delete(w.graphs[branch], module) + delete(w.edges[branch], module) +} + +// putEdges keeps the edges into module's artifacts that branch's catalog entry names. +func (w *world) putEdges(branch, module string, edges []meshcatalog.Edge) { + w.mu.Lock() + defer w.mu.Unlock() + if w.edges == nil { + w.edges = map[string]map[string][]meshcatalog.Edge{} + } + if w.edges[branch] == nil { + w.edges[branch] = map[string][]meshcatalog.Edge{} + } + w.edges[branch][module] = edges +} + +// edgesOf is the edges into module's artifacts in branch's catalog half. +func (w *world) edgesOf(branch, module string) []meshcatalog.Edge { + w.mu.Lock() + defer w.mu.Unlock() + return w.edges[branch][module] } // holdsBranch says whether the catalog half of branch has been taken in (even empty). @@ -332,18 +320,37 @@ func (w *world) records() map[string]nodeengine.NodeRecord { return out } -// presenceOf reads one key of the watchman's presence: a node's key (``) with the node's online or not; the keys of -// the parts (`part..`) are not a node's and are none here. -func presenceOf(key string, value []byte) (node string, online, ok bool) { - node, part, _, err := wdc.ParsePresenceKey(key) - if err != nil || node == "" || part != "" { - return "", false, false +// presenceSaid is the last of the watchman's machine-away and machine-back heard of a machine (design 47 G §389). +type presenceSaid struct { + away bool + at time.Time +} + +// heardPresence takes one machine-away (away) or machine-back of machine said at at: the later of the two holds, so the two +// streams, read whole in either order, leave each machine as its last event says. +func (w *world) heardPresence(machine string, away bool, at time.Time) { + w.mu.Lock() + defer w.mu.Unlock() + if w.presence == nil { + w.presence = map[string]presenceSaid{} } - p, err := wdc.DecodeNodePresence(value) - if err != nil { - return "", false, false + if was, ok := w.presence[machine]; ok && at.Before(was.at) { + return } - return node, p.Online, true + w.presence[machine] = presenceSaid{away: away, at: at} +} + +// awayNow is every machine whose last presence event heard is machine-away. +func (w *world) awayNow() map[string]bool { + w.mu.Lock() + defer w.mu.Unlock() + out := map[string]bool{} + for m, p := range w.presence { + if p.away { + out[m] = true + } + } + return out } // seatHeard says a machine holding seat (a seat held once) is online as the watchman holds it: the callee of a call to @@ -364,13 +371,13 @@ func (w *world) seatHeard(seat string) bool { return false } -// online is, by node, whether the watchman holds the node online. +// online is, by node the assigner hears of, whether it is present: a machine is away from the watchman's machine-away until its +// machine-back, and present otherwise (design 47 G §389); the watchman's state is not read (ADR 0596 §2). func (w *world) online() map[string]bool { + away := w.awayNow() out := map[string]bool{} - for key, value := range w.read(buckets.Presence) { - if node, online, ok := presenceOf(key, value); ok { - out[node] = online - } + for _, node := range w.knownNodes() { + out[node] = !away[node] } return out } @@ -399,11 +406,7 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want { func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes { n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{}, Reads: map[string][]string{}} - for key, value := range w.read(buckets.Presence) { - if node, online, ok := presenceOf(key, value); ok { - n.Online[node] = online - } - } + n.Online = w.online() for key, value := range w.read(buckets.Have) { node, module, isNode, err := nodeengineclient.ParseHaveKey(key) if err != nil { @@ -459,34 +462,28 @@ func (w *world) has(bucket, key string) bool { return ok } -// built says whether the builder's build of a module at a fingerprint is done. +// built says whether the builder's build of a module at a fingerprint is done, from its `build.ended` heard (design 47 G A-181). func (w *world) built(module, fp string) bool { - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return false - } - w.mu.Lock() - value, ok := w.values[buckets.Builds][key] - w.mu.Unlock() - if !ok { - return false - } - b, err := builds.DecodeBuilderBuild(value) - return err == nil && b.State == builds.BuildDone + e, ok := w.ends.endOf(module, fp) + return ok && e.Outcome == builds.BuildDone } -// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build key's needs, ADR -// 0618 §1): nil when it is not done or names none. +// buildStanding says whether the builder's `build.ended` of module at fp said the build failed, and its error whole (design 47 G +// §575, A-162): the words of a hold under rule 1. +func (w *world) buildStanding(module, fp string) (failed bool, words string) { + if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildFailed { + return true, e.Error + } + return false, "" +} + +// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build's end says them, ADR +// 0618 §1, A-181): nil when it is not done or names none. func (w *world) buildNeeds(module, fp string) []string { - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return nil + if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildDone { + return e.Needs } - b, err := builds.DecodeBuilderBuild(w.value(buckets.Builds, key)) - if err != nil || b.State != builds.BuildDone { - return nil - } - return b.Needs + return nil } // recordReads is what node's record `.node` states its engine reads (ADR 0614 §4), and whether the assigner holds @@ -496,7 +493,7 @@ func (w *world) recordReads(node string) ([]string, bool) { return r.ManifestReads, ok } -// knownNodes is every node the assigner hears of: placed, with a record, or with a presence. +// knownNodes is every node the assigner hears of: placed, with a record, or named by machine-away or machine-back. func (w *world) knownNodes() []string { set := map[string]bool{} for node := range w.placedNodes() { @@ -505,9 +502,11 @@ func (w *world) knownNodes() []string { for node := range w.records() { set[node] = true } - for node := range w.online() { + w.mu.Lock() + for node := range w.presence { set[node] = true } + w.mu.Unlock() return slices.Sorted(maps.Keys(set)) } @@ -582,8 +581,19 @@ func (w *world) unplacedNodes() []string { return out } -// trunkBranch is the branch of the trunk-only want, kept until the first registrations (design 47 G §95, §353). -const trunkBranch = "main" +// trunkName is the first watched branch, the trunk's (design 47 G §507, §353): "" until the first view of the catalog half is adopted. +func (w *world) trunkName() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.first +} + +// setTrunk says which watched branch is the first. +func (w *world) setTrunk(branch string) { + w.mu.Lock() + defer w.mu.Unlock() + w.first = branch +} // placedBranches is, by module and node, the watched branch each placement names ("" when it names none). func (w *world) placedBranches() map[string]map[string]string { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go index 7cf9ea22..bc013a53 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go @@ -3,6 +3,8 @@ package main import ( "context" "mesh-delivery-assigner/internal/buckets" + "os" + "path/filepath" "strings" "testing" "time" @@ -81,3 +83,32 @@ func putModuleRaw(w *world, branch, module string, raw []byte) { } w.putModule(branch, module, v, graph.Node{FP: v.Fp2, Repository: v.Repository}, nil) } + +// Design 47 G §507, §532, §7: the first watched branch the catalog half is followed of is the trunk's, whatever its name, and no part of the +// assigner's code names a branch: the world's trunk comes first, and no non-test source holds the string "main". +func TestTheTrunkIsTheFirstWatchedBranchAndNoCodeNamesOne(t *testing.T) { + w := newWorld() + w.setTrunk("develop") + for _, b := range []string{"beta", "develop", "alpha"} { + w.putModule(b, "web", meshcatalog.ModulesByModuleValue{Fp2: "fp"}, graph.Node{FP: "fp"}, nil) + } + if got := w.branches(); strings.Join(got, ",") != "develop,alpha,beta" { + t.Errorf("branches = %v, want the first watched first, then by name", got) + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(src), `"main"`) { + t.Errorf("%s names the branch main in code (design 47 G §507)", f) + } + } +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index d3cb0742..058770c8 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -26,11 +26,15 @@ import ( // trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller // issued its credential for F, its manifest at F, and the nodes it is assigned to. type Module struct { - Name string - FP string - Built bool - Held bool - Granted bool + Name string + FP string + Built bool + // BuildFailed and BuildError say a build that ended failed, with the error as the builder's `build.ended` carries it whole + // (design 47 G §575, A-162): the words of a hold under rule 1. + BuildFailed bool + BuildError string + Held bool + Granted bool // GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node // that runs an older one names it (issue 1945). Nil: none. GrantedAt func(fp string) bool @@ -69,10 +73,6 @@ type Module struct { // Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition // with this digest (ADR 0539 §14). Nil: none did. Allowed func(node, digest string) bool - // Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads, - // read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no - // approved release is known to carry it. Nil: none is read. - Release func(fp string) (string, bool) // Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed. Released bool // Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back @@ -159,9 +159,6 @@ func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool { return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK } -// Healthy says whether a have runs fp healthy. -func Healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool { return healthy(h, fp) } - // Failed says whether a have's health check failed once the node is done moving. func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) } @@ -191,7 +188,10 @@ func Holds(m Module, node string, n Nodes) (bool, int, string) { func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { switch { case !m.Built: - return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP)) + if m.BuildFailed { + return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s failed: %s", m.Name, short(m.FP), m.BuildError) + } + return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is being built", m.Name, short(m.FP)) case m.Held: return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP)) case m.MachineHeld != nil && m.MachineHeld(node): @@ -202,7 +202,7 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { switch { case canary == "": return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name) - case node != canary && !healthy(n.Haves[canary][m.Name], m.FP): + case node != canary && !n.RunsHealthy(canary, m.Name, m.FP): return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP)) } } @@ -322,15 +322,19 @@ func (m Module) MissingProvisions(node string) []string { return out } -// runsHealthy says whether module runs healthy on node: from G the node-engine seat's last `running` of it names a build and -// does not say it stopped; with the seat's events not heard, the have's phase and health. +// runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it +// stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy. func (n Nodes) runsHealthy(node, module string) bool { - if n.Running != nil { - r, ok := n.Running[node][module] - return ok && !r.Stopped && r.Fingerprint != "" - } - h, ok := n.Haves[node][module] - return ok && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK + r, ok := n.Running[node][module] + return ok && !r.Stopped && r.Fingerprint != "" && (r.Healthy == nil || *r.Healthy) +} + +// RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone +// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a +// module of which nothing was heard counts as not healthy. The have is never read for it. +func (n Nodes) RunsHealthy(node, module, fp string) bool { + r, ok := n.Running[node][module] + return ok && n.runsHealthy(node, module) && r.Fingerprint == fp } // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned @@ -702,7 +706,7 @@ func healthyElsewhere(m Module, n Nodes, node string) []string { for _, other := range slices.Sorted(slices.Values(m.Nodes)) { // A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs // healthy now. - if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) { + if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) { out = append(out, other) } } @@ -750,7 +754,7 @@ func allRunHealthy(m Module, n Nodes) bool { if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) { continue } - if !healthy(h, m.FP) { + if !n.RunsHealthy(node, m.Name, m.FP) { return false } ran++ diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index d0f3be9f..28b7f3a0 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -60,7 +60,7 @@ func wantedOld() map[string]mwc.Want { // g14 follows at once: step 0, ADR 0528 clause 3). func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) { m, n := ready() - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) for _, node := range []string{"ace", "g14"} { if out.Wants[node].Fp != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) { t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew) @@ -89,7 +89,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { } { m, n := ready() breakIt(&m, &n) - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if w, ok := out.Wants["g14"]; ok && w.Fp == fNew { t.Errorf("rule %d broken: g14's want moved to F", rule) } @@ -110,7 +110,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { }, nil} { m, n := ready() m.Provisions = all - if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].Fp != fNew { + if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["g14"].Fp != fNew { t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words) } } @@ -161,7 +161,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m, n := ready() m.Name, m.Needs, n.Reads, m.FormNeeds = c.name, c.needs, c.reads, c.forms n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if moved := out.Wants["g14"].Fp == fNew; moved == c.held { t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"]) } @@ -208,7 +208,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if c.holds == graph.BusState { words = graph.BusStateWaitWords("mail", "g14") } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if c.holds == "" { if out.Wants["g14"].Fp != fNew { t.Errorf("%s: held %+v", c.name, out.Rules["g14"]) @@ -235,7 +235,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { return map[string]meshprovisionerclient.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state}, node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}} } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if got := out.Wants["g14"].Fp == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) { t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words) } @@ -280,7 +280,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m.Provisions = func(string) map[string]meshprovisionerclient.Provision { return map[string]meshprovisionerclient.Provision{} } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if c.missing == "" { if out.Wants["g14"].Fp != fNew || len(m.MissingProvisions("g14")) != 0 { t.Errorf("%s: held %+v", c.name, out.Rules["g14"]) @@ -307,7 +307,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m.Provisions = func(string) map[string]meshprovisionerclient.Provision { return map[string]meshprovisionerclient.Provision{} } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if got := out.Wants["g14"].Fp == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != mwc.RuleProvisions || !strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) { t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"]) @@ -342,7 +342,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if c.have != nil { n.Haves["novox"][c.module] = *c.have } - out := Wants(m, n, c.before, nil) + out := Wants(m, heardOfHaves(n), c.before, nil) w, written := out.Wants["novox"] if !written { w = c.before["novox"] @@ -431,7 +431,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if c.name == "placed" { before["g14"] = mwc.Want{Fp: fNew, Composed: dig(fNew)} } - out := Wants(m, n, before, nil) + out := Wants(m, heardOfHaves(n), before, nil) if got := out.Wants["g14"]; !reflect.DeepEqual(got, c.want) { t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want) } @@ -453,7 +453,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m, n := ready() grant := grantsstate.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}} m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["ace"]; w.Fp != fNew { t.Errorf("ace, whose account the grant names, wants %+v", w) } @@ -486,7 +486,7 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) { if c := Canary(m.Nodes, n.Online); c != "g14" { t.Fatalf("canary = %q, want g14", c) } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if out.Wants["g14"].Fp != fNew { t.Errorf("the canary g14's want = %+v", out.Wants["g14"]) } @@ -500,7 +500,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"} m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC) - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld}) // 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words. if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" { t.Fatalf("held = %+v", out.Held) @@ -516,7 +516,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { // back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes). n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} before := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}} - out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld}) + out = Wants(m, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld}) if out.Held != nil || len(out.Wants) != 0 { t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants) } @@ -534,7 +534,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { n.Online["novox"] = true n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}} before["novox"] = mwc.Want{Fp: fOld, Composed: dig(fOld)} - out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) if _, alone := out.NodeFailed["g14"]; !alone { t.Errorf("g14 does not fail alone: %+v", out.NodeFailed) } @@ -547,7 +547,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { // A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945). n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} delete(before, "novox") - out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) if w := out.Wants["novox"]; w.Fp != fNew || len(out.Neither) != 0 { t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither) } @@ -569,7 +569,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} } before["novox"] = mwc.Want{Fp: fNew, Composed: dig(fNew)} - out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 { t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed) } @@ -584,7 +584,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) { m, n := ready() m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}} n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) if out.Held == nil { t.Fatal("F was not held") } @@ -601,7 +601,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) { func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) if len(out.Restore) != 0 { t.Errorf("restore asks = %v", out.Restore) } @@ -612,7 +612,7 @@ func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) { func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) { m, n := ready() m.Nodes = []string{"ace"} - out := Wants(m, n, map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil) if w := out.Wants["g14"]; !w.Removed { t.Errorf("g14's want = %+v, want removed", w) } @@ -632,7 +632,7 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) { } return composedEverywhere(node, fp) } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) for node, w := range out.Wants { if w.Fp == fNew { t.Errorf("%s's want moved to F before its composition exists: %+v", node, w) @@ -642,11 +642,11 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) { t.Errorf("ace is held back by %+v, want rule 7 naming the node", r) } // and a node that never had a want gets none - if out := Wants(m, n, nil, nil); len(out.Wants) != 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Wants) != 0 { t.Errorf("wants without a composition: %+v", out.Wants) } have["ace|"+fNew] = true - out = Wants(m, n, wantedOld(), nil) + out = Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("ace's want = %+v, want F naming the composition's digest", w) } @@ -664,14 +664,14 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { return ComposedDigest{Digest: revised, Judgement: JudgedNeither}, true } running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}} - out := Wants(m, n, running, nil) + out := Wants(m, heardOfHaves(n), running, nil) for _, node := range []string{"ace", "g14"} { if w := out.Wants[node]; w.Fp != fNew || w.Composed != revised { t.Errorf("%s's want = %+v, want F naming the revised digest", node, w) } } m.Granted = false - out = Wants(m, n, running, nil) + out = Wants(m, heardOfHaves(n), running, nil) if w, ok := out.Wants["ace"]; ok && w.Composed == revised { t.Errorf("a revision was written while a rule holds: %+v", w) } @@ -679,7 +679,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { // A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the // controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while // the rules still hold F; for a record module whose revision recreates a container, only after the ask. - out = Wants(m, n, wantedOld(), nil) + out = Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["g14"]; w.Fp != fOld || w.Composed != revised || w.Newest != fNew { t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w) } @@ -687,7 +687,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { return ComposedDigest{Digest: revised, Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord, Against: dig(fOld), Words: "recreates"}, true } - out = Wants(m, n, wantedOld(), nil) + out = Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["g14"]; w.Composed == revised { t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w) } @@ -707,12 +707,12 @@ func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) { } return composedEverywhere(node, fp) } - out := Wants(m, n, running, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}) if w := out.Wants["ace"]; w.Fp == fOld { t.Errorf("set back to a fingerprint whose composition is gone: %+v", w) } m.Composed = composedEverywhere - out = Wants(m, n, running, map[string]string{"ace": fOld}) + out = Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}) if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { t.Errorf("set-back want = %+v", w) } @@ -741,7 +741,7 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) for _, j := range []Judgement{JudgedRecreates, JudgedMovesData, JudgedCannot} { m, n, wanted := movingTo(j) recording(&m) - out := Wants(m, n, wanted, nil) + out := Wants(m, heardOfHaves(n), wanted, nil) if w, ok := out.Wants["ace"]; ok { t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w) } @@ -756,11 +756,11 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) } // a new revision at the same fingerprint is asked too same := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}} - if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) { + if out := Wants(m, heardOfHaves(n), same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) { t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out) } m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) } - out = Wants(m, n, wanted, nil) + out = Wants(m, heardOfHaves(n), wanted, nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("%s: after the yes ace's want = %+v", j, w) } @@ -772,25 +772,25 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) { m, n, wanted := movingTo(JudgedNeither) recording(&m) - if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { + if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("neither: %+v", out) } m, n, _ = movingTo(JudgedRecreates) recording(&m) m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone delete(n.Haves["ace"], "mail") - if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("a node that never ran it: %+v", out) } // but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below) m, n, _ = movingTo(JudgedCannot) recording(&m) n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 { t.Errorf("a node that runs it with no want: %+v", out) } m, n, wanted = movingTo(JudgedRecreates) - if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { + if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("a rolled module: %+v", out) } } @@ -802,11 +802,11 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} // ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node // backs its streams up before it applies. - if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 { + if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 { t.Errorf("the bus did not move by the rules: %+v", out) } running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}, "g14": {Fp: fNew, Composed: dig(fOld)}} - out := Wants(m, n, running, nil) + out := Wants(m, heardOfHaves(n), running, nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("a user-list revision of the bus: ace's want = %+v", w) } @@ -835,7 +835,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T if c.step == "" { m.Puts = nil } - out := Wants(m, n, c.wanted, nil) + out := Wants(m, heardOfHaves(n), c.wanted, nil) got, wrote := out.Wants["ace"] switch { case c.want == nil && wrote: @@ -852,7 +852,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T held := put held.Step = puts.Holding m.Puts = map[string]puts.Put{"ace": held} - if w := Wants(m, n, map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold { + if w := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold { t.Errorf("a held bus did not take its user-list revision as a put: %+v", w) } m.Puts = nil @@ -983,14 +983,14 @@ func stepsOfAPut(t *testing.T) { func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) { m, n := ready() running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew), Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}} - if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok { + if w, ok := Wants(m, heardOfHaves(n), running, nil).Wants["ace"]; ok { t.Errorf("the answer was erased: %+v", w) } // A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its // have names is not in the want, so a want kept for its digest would erase the answer at every pass). m.Self = func(string) bool { return true } placed := map[string]mwc.Want{"ace": {Fp: fNew, Self: true, Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}} - if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok { + if w, ok := Wants(m, heardOfHaves(n), placed, nil).Wants["ace"]; ok { t.Errorf("the answer in a placed node's want was erased: %+v", w) } } @@ -1001,7 +1001,7 @@ func TestAReleasedHoldIsNotHeldAgain(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"} m.Released = true - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld}) if out.Held != nil { t.Errorf("a released fingerprint was held again: %+v", out.Held) } @@ -1018,11 +1018,11 @@ func TestARestoreAnswerSetsADataModuleBack(t *testing.T) { n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}} m.Restored = func(node string) bool { return false } - if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld { + if w, ok := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld { t.Errorf("set back without the operator's restore: %+v", w) } m.Restored = func(node string) bool { return node == "ace" } - if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { + if w := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { t.Errorf("after restore ace's want = %+v", w) } } @@ -1040,22 +1040,22 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) { } } with(catalogue.PolicyRecord, "person") // a roll module the operator set to record - if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew { t.Errorf("the operator's record policy was not honoured: %+v", out) } recording(&m) // the manifest says record, the operator chose roll with("roll", "person") - if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { t.Errorf("the operator's roll policy was not honoured: %+v", out) } with("", "") // made by hand: the manifest decides - if out := Wants(m, n, wanted, nil); len(out.Record) == 0 { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 { t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out) } // 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition. with(catalogue.PolicyRecord, "person") m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"} - if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out) } } @@ -1066,11 +1066,11 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) { m, n, _ := movingTo(JudgedNeither) m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew { + if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew { t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok) } n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew { + if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew { t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok) } } @@ -1096,7 +1096,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test "a want at the fingerprint it runs": {"ace": {Fp: fOld, Composed: dig(fOld)}}, "no readable want": nil, } { - out := Wants(m, n, wanted, nil) + out := Wants(m, heardOfHaves(n), wanted, nil) if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != userList { t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w) } @@ -1106,7 +1106,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test } // the same flag on a module that is not the bus changes nothing m.Manifest.Provides = nil - if out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList { + if out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList { t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"]) } } @@ -1129,7 +1129,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { } m.Composed = unchanged n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} // no composition named - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w) } @@ -1146,7 +1146,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} } n2.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out := Wants(m2, n2, nil, nil) + out := Wants(m2, heardOfHaves(n2), nil, nil) if j == JudgedNeither { if out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("%s, %s: %+v", name, what, out) @@ -1162,12 +1162,12 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "judged"}, true } n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 { t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out) } // a new fingerprint still asks (the bus is barred instead) n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out = Wants(m, n, nil, nil) + out = Wants(m, heardOfHaves(n), nil, nil) if w, ok := out.Wants["ace"]; ok { t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w) } @@ -1181,7 +1181,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld}) if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" { t.Fatalf("held = %+v", out.Held) } @@ -1198,7 +1198,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { return m, n } m, n := everywhere() - out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}) + out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}) if !out.Lifted || out.Held != nil { t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held) } @@ -1210,7 +1210,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { m, n = everywhere() delete(n.Haves["g14"], "mail") - if out := Wants(m, n, wantedOld(), nil); !out.Lifted { + if out := Wants(m, heardOfHaves(n), wantedOld(), nil); !out.Lifted { t.Error("a node that does not run the module kept the hold") } @@ -1227,7 +1227,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { } { m, n := everywhere() change(&m, &n) - if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted { + if out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted { t.Errorf("%s: the hold was lifted", name) } } @@ -1258,7 +1258,7 @@ func x11() (Module, Nodes) { // composition, never the newest, and says the newest it is held back from; the canary is never passed over. func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) { m, n := x11() - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) // 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said if w := out.Wants["g14"]; w.Fp != fNew || out.Unreached["g14"] == "" { t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"]) @@ -1273,13 +1273,13 @@ func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) m, n = x11() n.Haves["g14"]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"] - if w := Wants(m, n, nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary { + if w := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary { t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w) } // a node already running F gets its want at F; a removed want is no want either m, n = x11() n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - out = Wants(m, n, map[string]mwc.Want{"g14": {Removed: true}}, nil) + out = Wants(m, heardOfHaves(n), map[string]mwc.Want{"g14": {Removed: true}}, nil) if w := out.Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Removed { t.Errorf("g14 running F with a removed want = %+v, want F", w) } @@ -1301,7 +1301,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { } { m, n := x11() change(&m, &n) - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if what == "an old-form build" { // the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945) if w, ok := out.Wants["shanks"]; ok { @@ -1322,7 +1322,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { } m, n := x11() before := map[string]mwc.Want{"g14": {Fp: fOld, Composed: dig(fOld)}, "shanks": {Fp: fOld, Composed: dig(fOld)}} - out := Wants(m, n, before, nil) + out := Wants(m, heardOfHaves(n), before, nil) if len(out.Neither) != 0 { t.Errorf("nodes holding a want named on neither list: %+v", out.Neither) } @@ -1341,7 +1341,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { m, n := x11() delete(n.Haves["shanks"], "screen-lock") - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) // 1960: the canary orders moves, never a first want: the new assignment is wanted at F if w := out.Wants["shanks"]; w.Fp != fNew { t.Errorf("a new assignment while the canary does not run F: %+v, want F", w) @@ -1350,7 +1350,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { t.Errorf("a new assignment is named on neither list: %+v", out.Neither) } m.Built = false - if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone { + if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone { t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither) } // 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run @@ -1364,12 +1364,12 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { }{ {"no key: nothing written", nil, map[string]mwc.Want{}}, {"a removed want: it says why", map[string]mwc.Want{"shanks": gone}, - map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}}, - {"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}, + map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}}}, + {"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}}, map[string]mwc.Want{}}, {"a node that runs it: on neither list, its removed want left", map[string]mwc.Want{"g14": gone}, map[string]mwc.Want{}}, } { - out := Wants(m, n, c.before, nil) + out := Wants(m, heardOfHaves(n), c.before, nil) if len(out.Wants) != len(c.want) { t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want) continue @@ -1381,7 +1381,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { } } m.FP = "" - if out := Wants(m, n, nil, nil); len(out.Neither) != 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 0 { t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither) } } @@ -1404,7 +1404,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { providersHealthy(n) oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if w := out.Wants["shanks"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 { t.Errorf("canary healthy on F: shanks's want = %+v, want F", w) } @@ -1416,7 +1416,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { providersHealthy(n) oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - out = Wants(m, n, nil, nil) + out = Wants(m, heardOfHaves(n), nil, nil) if w, ok := out.Wants["shanks"]; ok { t.Errorf("canary not healthy on F: shanks got a want: %+v", w) } @@ -1427,7 +1427,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { m, n = x11() oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out = Wants(m, n, nil, nil) + out = Wants(m, heardOfHaves(n), nil, nil) if w := out.Wants["shanks"]; w.Fp != fNew || out.Unreached["shanks"] == "" { t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"]) } @@ -1440,14 +1440,14 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} m.Granted = false - if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok { + if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; ok { t.Errorf("F not granted: shanks got a want: %+v", w) } // the old-form node is the canary itself: F once its providers run healthy m, n = x11() providersHealthy(n) oldForm(n, "g14") - if w := Wants(m, n, nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) { + if w := Wants(m, heardOfHaves(n), nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("old-form canary: g14's want = %+v, want F", w) } } @@ -1462,7 +1462,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) { n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}} before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}} - if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil { + if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held != nil { t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held) } } @@ -1472,7 +1472,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) { n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}} before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}} - if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" { + if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" { t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held) } } @@ -1491,7 +1491,7 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) { } for _, phase := range []string{nodeengineclient.PhaseRemoved, nodeengineclient.PhaseHeld, nodeengineclient.PhaseApplying, nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseChecking} { m, n := setup(phase, fOld, nodeengineclient.HealthUnknown) - if out := Wants(m, n, nil, nil); !out.Lifted { + if out := Wants(m, heardOfHaves(n), nil, nil); !out.Lifted { t.Errorf("g14's have in phase %s kept the hold up", phase) } } @@ -1500,14 +1500,14 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) { "failed": {Fp: fNew, Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed}, } { m, n := setup(h.Phase, h.Fp, h.Health) - if out := Wants(m, n, nil, nil); out.Lifted { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Lifted { t.Errorf("g14 %s: the hold was lifted", what) } } // a failed health on a removed have holds nothing m, n := setup(nodeengineclient.PhaseRemoved, fNew, nodeengineclient.HealthFailed) m.Held = false - if out := Wants(m, n, nil, nil); out.Held != nil { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Held != nil { t.Errorf("a removed have with a failed health held F: %+v", out.Held) } } @@ -1520,7 +1520,7 @@ func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) { m.Composed = func(_, fp string) (ComposedDigest, bool) { return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord}, true } - out := Wants(m, n, map[string]mwc.Want{"shanks": {Removed: true}}, nil) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"shanks": {Removed: true}}, nil) if w, ok := out.Wants["shanks"]; ok { t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w) } @@ -1550,7 +1550,7 @@ func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) { } { m, n := x11() change(&m, &n) - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if len(out.Neither) != 0 { t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither) } @@ -1602,9 +1602,79 @@ func TestAWantNamesAKeptBuildsPartsAndNodeForANodeThatReadsThem(t *testing.T) { m, n := ready() n.Reads = map[string][]string{"ace": row.reads, "g14": row.reads} m.KeptBuilds = row.kept - w := Wants(m, n, wantedOld(), nil).Wants["ace"] + w := Wants(m, heardOfHaves(n), wantedOld(), nil).Wants["ace"] if w.Fp != fNew || (w.KeptOn == "g14") != row.wanted || (w.Version == "1.2.3") != row.wanted || (len(w.Parts) == 1) != row.wanted { t.Errorf("%s: ace's want = %+v", name, w) } } } + +// heardOfHaves gives each have that runs the `running` the node-engine seat says of it (design 47 G §520: health is read from +// `running` alone), for a fixture written as haves; a fixture that names its own `running` is left as it is. +func heardOfHaves(n Nodes) Nodes { + if n.Running != nil { + return n + } + n.Running = map[string]map[string]nodeengine.Running{} + for node, byModule := range n.Haves { + n.Running[node] = map[string]nodeengine.Running{} + for module, h := range byModule { + if h.Phase == nodeengineclient.PhaseRunning && h.Fp != "" { + ok := h.Health == nodeengineclient.HealthOK + n.Running[node][module] = nodeengine.Running{Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok} + } + } + } + return n +} + +// §520: health is read from the node-engine seat's `running` alone, whatever the have says, and a build of which nothing was heard +// is not healthy. +func TestHealthIsReadFromRunningAlone(t *testing.T) { + yes, no := true, false + say := func(node, module, fp string, healthy *bool) nodeengine.Running { + return nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Healthy: healthy} + } + for _, c := range []struct { + name string + running map[string]map[string]nodeengine.Running + rule int + }{ + {"all healthy by running", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &yes)}}, 0}, + {"a running with no healthy reads healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, nil), "store": say("ace", "store", fpOf('s'), nil)}}, 0}, + {"the canary says unhealthy, its have says healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &no), "store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary}, + {"nothing heard of the canary", map[string]map[string]nodeengine.Running{"ace": {"store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary}, + } { + m, n := ready() + n.Running = c.running + if got := Wants(m, n, wantedOld(), nil).Rules["g14"].Rule; got != c.rule { + t.Errorf("%s: g14 held by rule %d, want %d", c.name, got, c.rule) + } + } + // rule 5: a provider that says unhealthy, or that nothing was heard of, holds no want but leaves the consumer unreached + m, n := ready() + n.Running = map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &no)}} + if out := Wants(m, n, wantedOld(), nil); len(out.Unreached) == 0 { + t.Errorf("a provider saying unhealthy: no module unreached, %+v", out) + } +} + +// §575: a hold under rule 1 says how the build stands: failed with the error whole, or being built. +func TestRuleOneSaysHowTheBuildStands(t *testing.T) { + for _, c := range []struct { + name string + failed bool + err string + want string + }{ + {"a build that runs", false, "", "the build of mail at " + short(fNew) + " is being built"}, + {"a failed build", true, "transport fault: no responders at mesh-build-go", "the build of mail at " + short(fNew) + " failed: transport fault: no responders at mesh-build-go"}, + } { + m, n := ready() + m.Built, m.BuildFailed, m.BuildError = false, c.failed, c.err + held, rule, words := Holds(m, "g14", n) + if held || rule != mwc.RuleBuildDone || words != c.want { + t.Errorf("%s: held %v rule %d words %q, want %q", c.name, held, rule, words, c.want) + } + } +} diff --git a/modules/mesh-delivery-assigner/internal/assigngraph/candidate.go b/modules/mesh-delivery-assigner/internal/assigngraph/candidate.go new file mode 100644 index 00000000..1aafe268 --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assigngraph/candidate.go @@ -0,0 +1,138 @@ +package assigngraph + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" +) + +// A candidate's assignment graph (design 47 G §189, §190, A-201, A-235): computed per run on request from the candidate's catalog +// graph revision, kept apart from trunk's under the seat's key form candidate.., the group `run-`, and dropped on +// the runner's group-ended of that run; it writes no want, and the gate's heads-replaced drops nothing. + +// CandidateObject is the object name of a candidate's revision (the seat's key form candidate..). +func CandidateObject(group, heads string) string { + return meshwants.AssignmentRevisionsByCandidateGroupHeadsKey(group, heads) +} + +// KeepCandidate keeps c as the candidate's revision of group and heads, numbered after the one kept of it, unless that one lists +// the same entries from the same inputs; its entry objects are written first, an entry trunk shares stored once. +func KeepCandidate(ctx context.Context, o Objects, c Computed, group, heads, cause string, at time.Time) (meshwants.AssignmentRevision, bool, error) { + if group == "" || heads == "" { + return meshwants.AssignmentRevision{}, false, fmt.Errorf("a candidate's revision names its group and its heads, not %q and %q", group, heads) + } + was, err := readCandidateObject(ctx, o, CandidateObject(group, heads)) + has := err == nil + if err != nil && !errors.Is(err, ErrNoObject) { + return was, false, err + } + if has && Same(was, c.Revision) { + return was, false, nil + } + rev := c.Revision + rev.Kind, rev.Group, rev.Heads, rev.Cause = meshcatalogclient.KindCandidate, group, heads, cause + rev.Revision = was.Revision + 1 + t := at.UTC() + rev.At = &t + if err := writeEntries(ctx, o, c, rev); err != nil { + return was, false, err + } + list, err := json.Marshal(rev) + if err != nil { + return was, false, err + } + if err := o.Put(ctx, CandidateObject(group, heads), list); err != nil { + return was, false, fmt.Errorf("the candidate %s's revision in %s: %w", group, RevisionsObjects, err) + } + return rev, true, nil +} + +// ReadCandidate is the candidate's revision kept of group, of heads when named, else the newest kept of the group; ErrNoCandidate +// when none is kept. +func ReadCandidate(ctx context.Context, o Objects, group, heads string) (meshwants.AssignmentRevision, error) { + if heads != "" { + rev, err := readCandidateObject(ctx, o, CandidateObject(group, heads)) + if errors.Is(err, ErrNoObject) { + err = fmt.Errorf("%w (group %q, heads %q)", ErrNoCandidate, group, heads) + } + return rev, err + } + names, err := CandidateNames(ctx, o, group) + if err != nil { + return meshwants.AssignmentRevision{}, err + } + var newest meshwants.AssignmentRevision + found := false + for _, n := range names { + rev, err := readCandidateObject(ctx, o, n) + if err != nil { + return rev, err + } + if !found || rev.At != nil && newest.At != nil && rev.At.After(*newest.At) { + newest, found = rev, true + } + } + if !found { + return newest, fmt.Errorf("%w (group %q)", ErrNoCandidate, group) + } + return newest, nil +} + +// CandidateNames is the object name of every revision kept of the candidate group; every candidate's when group is "". +func CandidateNames(ctx context.Context, o Objects, group string) ([]string, error) { + names, err := o.Names(ctx) + if err != nil { + return nil, fmt.Errorf("the objects %s: %w", RevisionsObjects, err) + } + var out []string + for _, n := range names { + g, _, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n) + if ok && (group == "" || g == group) { + out = append(out, n) + } + } + return out, nil +} + +// Remover is an object bucket that also removes an object. +type Remover interface { + Objects + // Delete removes the object of that name; one not held is no error. + Delete(ctx context.Context, name string) error +} + +// DropCandidate removes every revision kept of the candidate group (its run's group-ended, A-235); the entries it alone listed are +// removed with the next revisions nothing pins. It says how many it removed. +func DropCandidate(ctx context.Context, o Remover, group string) (int, error) { + if strings.TrimSpace(group) == "" { + return 0, errors.New("a candidate to drop names its group") + } + names, err := CandidateNames(ctx, o, group) + if err != nil { + return 0, err + } + for i, n := range names { + if err := o.Delete(ctx, n); err != nil { + return i, fmt.Errorf("the candidate %s's revision %s: %w", group, n, err) + } + } + return len(names), nil +} + +func readCandidateObject(ctx context.Context, o Objects, name string) (meshwants.AssignmentRevision, error) { + var rev meshwants.AssignmentRevision + data, err := o.Get(ctx, name) + if err != nil { + return rev, err + } + if err := json.Unmarshal(data, &rev); err != nil { + return rev, fmt.Errorf("the candidate's revision %s: %w", name, err) + } + return rev, nil +} diff --git a/modules/mesh-delivery-assigner/internal/assigngraph/compute.go b/modules/mesh-delivery-assigner/internal/assigngraph/compute.go index 2825d6d5..fca2b7fb 100644 --- a/modules/mesh-delivery-assigner/internal/assigngraph/compute.go +++ b/modules/mesh-delivery-assigner/internal/assigngraph/compute.go @@ -58,13 +58,16 @@ type placedAt struct { entry meshwants.AssignmentEntry langs map[string]bool acts []actingRule + // provides is what its catalog entry provides; open its requirements its placement neither binds nor its node pins. + provides, open []string } type actingRule struct{ seat, step, on, bound string } // Compute is the assignment graph revision of in: one entry per module placed on a node whose branch's catalog graph // holds it, its version the build the placement names or else the catalog entry's fingerprint (design 47 lines 909-914), -// with the edges of provisioning to the providers its placement binds or its node pins, and of acting from each acting +// with the edges of provisioning to the providers its placement binds or its node pins, else the provider on its node or +// the only provider (an entry held naming the candidates where none is), and of acting from each acting // part's version to each version its rule names (design 47 *An acting part's new version acts first*: "the assigner // gives an edge from the acting part's version to each version its rule names"). func Compute(in Inputs) Computed { @@ -115,6 +118,7 @@ func Compute(in Inputs) Computed { for _, r := range ce.Acts { p.acts = append(p.acts, actingRule{seat: r.Seat, step: r.Step, on: r.On, bound: r.Bound}) } + p.provides = ce.Provides for _, r := range ce.Requires { prov, ok := m.Bindings[r] if !ok { @@ -122,12 +126,42 @@ func Compute(in Inputs) Computed { } if ok { p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeProvisioning, Node: prov.Node, Module: prov.Module, Provision: r}) + } else { + p.open = append(p.open, r) } } placed = append(placed, p) branches[branch] = true } } + // A requirement neither bound nor pinned points, by design 27's order (design 47 G §357), to the provider on the + // consumer's node, else to the only provider; otherwise the entry is held, naming the candidates, and nothing is guessed. + for _, p := range placed { + for _, r := range p.open { + var here, all []*placedAt + for _, q := range placed { + if q != p && slices.Contains(q.provides, r) { + all = append(all, q) + if q.node == p.node { + here = append(here, q) + } + } + } + switch { + case len(here) == 1: + p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeProvisioning, Node: here[0].node, Module: here[0].placed.Module, Provision: r}) + case len(here) == 0 && len(all) == 1: + p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeProvisioning, Node: all[0].node, Module: all[0].placed.Module, Provision: r}) + default: + h := meshwants.Held{Provision: r} + for _, q := range all { + h.Candidates = append(h.Candidates, meshwants.Edge{Kind: EdgeProvisioning, Node: q.node, Module: q.placed.Module, Fp: q.entry.Fp, Provision: r}) + } + sortEdges(h.Candidates) + p.entry.Held = append(p.entry.Held, h) + } + } + } at := map[string]*placedAt{} for _, p := range placed { at[p.node+"/"+p.placed.Module] = p diff --git a/modules/mesh-delivery-assigner/internal/assigngraph/graph_test.go b/modules/mesh-delivery-assigner/internal/assigngraph/graph_test.go index 767c88e6..8e8b3589 100644 --- a/modules/mesh-delivery-assigner/internal/assigngraph/graph_test.go +++ b/modules/mesh-delivery-assigner/internal/assigngraph/graph_test.go @@ -91,11 +91,16 @@ func TestTheAssignmentGraphIsComputedFromItsInputsAndKept(t *testing.T) { if fmt.Sprint(webA.Edges) != fmt.Sprint(want) { t.Errorf("web on anchor's edges:\n got %+v\nwant %+v", webA.Edges, want) } - // laptop's web: its own node's node-engine composes it, the build module placed once builds it, and no binding or pin - // names a provider there. - if e := webL.Edges; len(e) != 2 || e[0].Module != "build-go" || e[0].Node != "anchor" || e[1].Module != "node-engine" || e[1].Node != "laptop" { + // laptop's web: its own node's node-engine composes it, the build module placed once builds it, and with no binding or + // pin its database is the only provider's (design 47 G §357). + if e := webL.Edges; len(e) != 3 || e[0].Module != "build-go" || e[0].Node != "anchor" || e[1].Module != "node-engine" || e[1].Node != "laptop" || + e[2] != (meshwants.Edge{Kind: EdgeProvisioning, Node: "anchor", Module: "postgres", Fp: "src2:p1", Provision: "database"}) { t.Errorf("web on laptop's edges: %+v", e) } + // Its secrets no placed module provides: held on both nodes, naming no candidate. + if h := webA.Held; len(h) != 1 || h[0].Provision != "secrets" || len(h[0].Candidates) != 0 { + t.Errorf("web on anchor is held on %+v", h) + } if e := byNode["anchor/postgres"].Edges; len(e) != 1 || e[0].Module != "node-engine" { t.Errorf("postgres builds no Go, so only the node-engine acts on it: %+v", e) } @@ -149,6 +154,63 @@ func TestTheAssignmentGraphIsComputedFromItsInputsAndKept(t *testing.T) { } } +// Design 47 G §357 and A-167: a requirement neither bound nor pinned points to the provider on the consumer's node, else +// to the only provider; otherwise the entry is held, naming the candidates, and nothing is guessed. +func TestARequirementPointsByTheOrderOrIsHeld(t *testing.T) { + h := half("main", 3, + meshcatalog.CatalogEntry{Module: "kv", Fp: "src2:k1", Provides: []string{"cache"}}, + meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1", Requires: []string{"cache"}}) + for _, c := range []struct { + name string + kv []string // the nodes kv is placed on + pin bool // web's node pins kv on laptop + edge string // the node of the provisioning edge, "" for none + held string // the candidates' nodes of the hold, "-" for no hold + }{ + {name: "the provider on the consumer's node", kv: []string{"anchor", "laptop"}, edge: "anchor", held: "-"}, + {name: "the only provider", kv: []string{"laptop"}, edge: "laptop", held: "-"}, + {name: "a pin before the node's own", kv: []string{"anchor", "laptop"}, pin: true, edge: "laptop", held: "-"}, + {name: "two providers elsewhere: held", kv: []string{"laptop", "server"}, held: "laptop,server"}, + {name: "none: held naming none", held: ""}, + } { + t.Run(c.name, func(t *testing.T) { + pl := map[string]mma.Assignments{"anchor": {Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}}} + if c.pin { + pl["anchor"] = mma.Assignments{Node: "anchor", Modules: pl["anchor"].Modules, Pins: map[string]mma.Provider{"cache": {Node: "laptop", Module: "kv"}}} + } + for _, n := range c.kv { + a := pl[n] + a.Node, a.Modules = n, append(a.Modules, mma.PlacedModule{Module: "kv"}) + pl[n] = a + } + out := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": h}, Placements: pl, FirstBranch: func(string) string { return "main" }}) + var web meshwants.AssignmentEntry + for _, ref := range out.Revision.Entries { + if ref.Node == "anchor" && ref.Module == "web" { + web = out.Entries[ref.Digest] + } + } + edge := "" + for _, e := range web.Edges { + if e.Kind == EdgeProvisioning && e.Provision == "cache" { + edge = e.Node + } + } + held := "-" + if len(web.Held) == 1 && web.Held[0].Provision == "cache" { + var nodes []string + for _, cand := range web.Held[0].Candidates { + nodes = append(nodes, cand.Node) + } + held = strings.Join(nodes, ",") + } + if edge != c.edge || held != c.held || len(web.Held) > 1 { + t.Errorf("web points to %q, held on %q (%+v); want %q, %q", edge, held, web.Held, c.edge, c.held) + } + }) + } +} + // Design 47 line 1385: a whole assignment graph revision of about 120 modules on four nodes is under the bus's 256 KiB // value limit. func TestARevisionOf120ModulesOnFourNodesFitsOneValue(t *testing.T) { @@ -174,3 +236,77 @@ func TestARevisionOf120ModulesOnFourNodesFitsOneValue(t *testing.T) { t.Errorf("%d entries in %d bytes", len(c.Revision.Entries), len(raw)) } } + +func (m memObjects) Delete(_ context.Context, name string) error { + delete(m, name) + return nil +} + +// A-201, A-235, §186, §189: a run's candidate assignment graph is kept apart from trunk's under candidate.., served by +// revision, entry and difference against trunk, and dropped by its group alone. +func TestARunsCandidateGraphIsKeptApartServedAndDropped(t *testing.T) { + ctx := context.Background() + at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) + placed := map[string]mma.Assignments{"ace": {Modules: []mma.PlacedModule{{Module: "web"}}}} + first := func(string) string { return "main" } + o := memObjects{} + trunk := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": half("main", 3, meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1"})}, + Placements: placed, FirstBranch: first}) + if _, _, err := Keep(ctx, o, trunk, "catalog-revision", at); err != nil { + t.Fatal(err) + } + cand := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": half("main", 4, meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w2"})}, + Placements: placed, FirstBranch: first}) + if _, _, err := KeepCandidate(ctx, o, cand, "", "h1", "catalog-revision", at); err == nil { + t.Error("a candidate naming no group was kept") + } + rev, fresh, err := KeepCandidate(ctx, o, cand, "run-7", "h1", "catalog-revision", at) + if err != nil || !fresh || rev.Kind != meshcatalogclient.KindCandidate || rev.Group != "run-7" || rev.Revision != 1 { + t.Fatalf("kept %+v fresh %v: %v", rev, fresh, err) + } + if _, ok := o[CandidateObject("run-7", "h1")]; !ok { + t.Fatalf("the candidate is not under candidate.run-7.h1: %v", o) + } + if _, fresh, _ := KeepCandidate(ctx, o, cand, "run-7", "h1", "catalog-revision", at); fresh { + t.Error("the same candidate kept again is a new revision") + } + newest, has, err := Newest(ctx, o) + if err != nil || !has || newest.Kind != meshcatalogclient.KindTrunk || newest.Revision != 1 { + t.Errorf("trunk's newest is %+v (%v, %v): the candidate is not kept apart", newest, has, err) + } + for _, in := range []meshwants.RevisionInput{{Group: "run-7"}, {Group: "run-7", Heads: "h1"}} { + a, err := Revision(ctx, o, in) + if err != nil || a.Revision.Group != "run-7" { + t.Errorf("revision %+v answered %+v: %v", in, a.Revision, err) + } + } + if _, err := Revision(ctx, o, meshwants.RevisionInput{Group: "run-8"}); !errors.Is(err, ErrNoCandidate) { + t.Errorf("an unkept candidate answered %v", err) + } + e, err := Entry(ctx, o, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"}) + if err != nil || e.Entry.Fp != "src2:w2" { + t.Errorf("the candidate's entry is %+v: %v", e.Entry, err) + } + d, err := Difference(ctx, o, meshwants.DifferenceInput{Group: "run-7"}) + if err != nil || len(d.Changes) != 1 || d.Changes[0].From != "src2:w1" || d.Changes[0].To != "src2:w2" { + t.Errorf("the candidate against trunk is %+v: %v", d, err) + } + if _, err := Difference(ctx, o, meshwants.DifferenceInput{}); err == nil { + t.Error("difference with none of from, to and group is not refused") + } + if _, _, err := KeepCandidate(ctx, o, cand, "run-9", "h2", "catalog-revision", at); err != nil { + t.Fatal(err) + } + if n, err := DropCandidate(ctx, o, "run-7"); err != nil || n != 1 { + t.Errorf("dropped %d: %v", n, err) + } + if _, err := ReadCandidate(ctx, o, "run-7", ""); !errors.Is(err, ErrNoCandidate) { + t.Errorf("the dropped candidate reads: %v", err) + } + if _, err := ReadCandidate(ctx, o, "run-9", ""); err != nil { + t.Errorf("another run's candidate went with it: %v", err) + } + if _, has, _ := Newest(ctx, o); !has { + t.Error("dropping a candidate dropped trunk") + } +} diff --git a/modules/mesh-delivery-assigner/internal/assigngraph/keep.go b/modules/mesh-delivery-assigner/internal/assigngraph/keep.go index 7ce8600e..743d346e 100644 --- a/modules/mesh-delivery-assigner/internal/assigngraph/keep.go +++ b/modules/mesh-delivery-assigner/internal/assigngraph/keep.go @@ -103,36 +103,13 @@ func Keep(ctx context.Context, o Objects, c Computed, cause string, at time.Time if has && Same(newest, c.Revision) { return newest, false, nil } - names, err := o.Names(ctx) - if err != nil { - return newest, false, fmt.Errorf("the objects %s: %w", RevisionsObjects, err) - } - held := make(map[string]bool, len(names)) - for _, n := range names { - held[n] = true - } rev := c.Revision rev.Revision = newest.Revision + 1 rev.Cause = cause t := at.UTC() rev.At = &t - for _, ref := range rev.Entries { - name := EntryObject(ref.Digest) - if held[name] { - continue - } - e, ok := c.Entries[ref.Digest] - if !ok { - return newest, false, fmt.Errorf("revision %d lists the entry of %s/%s (%s), which it does not carry", rev.Revision, ref.Node, ref.Module, ref.Digest) - } - data, err := json.Marshal(e) - if err != nil { - return newest, false, err - } - if err := o.Put(ctx, name, data); err != nil { - return newest, false, fmt.Errorf("the entry of %s/%s in %s: %w", ref.Node, ref.Module, RevisionsObjects, err) - } - held[name] = true + if err := writeEntries(ctx, o, c, rev); err != nil { + return newest, false, err } list, err := json.Marshal(rev) if err != nil { @@ -144,6 +121,37 @@ func Keep(ctx context.Context, o Objects, c Computed, cause string, at time.Time return rev, true, nil } +// writeEntries writes each entry object rev lists that the bucket does not hold yet, from c, before its list is kept. +func writeEntries(ctx context.Context, o Objects, c Computed, rev meshwants.AssignmentRevision) error { + names, err := o.Names(ctx) + if err != nil { + return fmt.Errorf("the objects %s: %w", RevisionsObjects, err) + } + held := make(map[string]bool, len(names)) + for _, n := range names { + held[n] = true + } + for _, ref := range rev.Entries { + name := EntryObject(ref.Digest) + if held[name] { + continue + } + e, ok := c.Entries[ref.Digest] + if !ok { + return fmt.Errorf("revision %d lists the entry of %s/%s (%s), which it does not carry", rev.Revision, ref.Node, ref.Module, ref.Digest) + } + data, err := json.Marshal(e) + if err != nil { + return err + } + if err := o.Put(ctx, name, data); err != nil { + return fmt.Errorf("the entry of %s/%s in %s: %w", ref.Node, ref.Module, RevisionsObjects, err) + } + held[name] = true + } + return nil +} + // Event is the seat's event assignment-revision of a kept revision. func Event(rev meshwants.AssignmentRevision) meshwants.AssignmentRevisionEvent { return meshwants.AssignmentRevisionEvent{V: rev.V, Revision: rev.Revision, Kind: rev.Kind, Cause: rev.Cause, Group: rev.Group, diff --git a/modules/mesh-delivery-assigner/internal/assigngraph/verbs.go b/modules/mesh-delivery-assigner/internal/assigngraph/verbs.go index 3b7d2d6c..525fee58 100644 --- a/modules/mesh-delivery-assigner/internal/assigngraph/verbs.go +++ b/modules/mesh-delivery-assigner/internal/assigngraph/verbs.go @@ -10,16 +10,23 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) -// ErrNoCandidate answers a verb asked of a candidate's revision: the assigner keeps none (the candidate's graph and the -// pins from the gate are milestone 47/G's W5.9, held with research 086's fold). -var ErrNoCandidate = errors.New("the assigner keeps no candidate's assignment graph revision") +// ErrNoCandidate answers a verb asked of a candidate's revision the assigner keeps none of: a run's candidate is kept from its +// catalog graph revision until its run's group-ended (A-201, A-235). +var ErrNoCandidate = errors.New("the assigner keeps no such candidate's assignment graph revision") // ErrNoRevision answers a verb asked while no trunk revision is kept yet. var ErrNoRevision = errors.New("no assignment graph revision is kept yet") func revisionOf(ctx context.Context, o Objects, revision int64, group, heads string) (meshwants.AssignmentRevision, error) { if group != "" || heads != "" { - return meshwants.AssignmentRevision{}, fmt.Errorf("%w (group %q, heads %q)", ErrNoCandidate, group, heads) + if group == "" { + return meshwants.AssignmentRevision{}, fmt.Errorf("a candidate's revision is asked by its group, with or without its heads, not by heads %q alone", heads) + } + rev, err := ReadCandidate(ctx, o, group, heads) + if err == nil && revision > 0 && rev.Revision != revision { + err = fmt.Errorf("%w (group %q, revision %d: the one kept is %d)", ErrNoCandidate, group, revision, rev.Revision) + } + return rev, err } if revision > 0 { return ReadRevision(ctx, o, revision) @@ -66,7 +73,16 @@ func Entry(ctx context.Context, o Objects, in meshwants.EntryInput) (meshwants.E // differs, with the edges that moved, and the nodes it touches. func Difference(ctx context.Context, o Objects, in meshwants.DifferenceInput) (meshwants.Difference, error) { if in.Group != "" || in.Heads != "" { - return meshwants.Difference{}, fmt.Errorf("%w (group %q, heads %q)", ErrNoCandidate, in.Group, in.Heads) + // A candidate against trunk (§186, §187): from the trunk revision named, else the newest kept. + to, err := revisionOf(ctx, o, 0, in.Group, in.Heads) + if err != nil { + return meshwants.Difference{}, err + } + from, err := revisionOf(ctx, o, in.From, "", "") + if err != nil { + return meshwants.Difference{}, err + } + return Compare(ctx, o, from, to) } if in.From <= 0 || in.To <= 0 { return meshwants.Difference{}, errors.New("difference takes the two kept revisions it compares, from and to") diff --git a/modules/mesh-delivery-assigner/internal/buckets/buckets.go b/modules/mesh-delivery-assigner/internal/buckets/buckets.go index 0cf5be2d..2b2674c6 100644 --- a/modules/mesh-delivery-assigner/internal/buckets/buckets.go +++ b/modules/mesh-delivery-assigner/internal/buckets/buckets.go @@ -6,10 +6,8 @@ package buckets import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - watchdogs "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" @@ -25,6 +23,18 @@ const ( // Waits holds the waits on an outside cause, one key `..` (design 47 G §706). WaitsState = "waits" Waits = plc.Module + "_" + WaitsState + // DeliveredAgain holds each ask of assigner_delivered_again, one key ``, its value the version, who asked and why, and + // why it does not hold yet (design 47 G §376, A-250). + DeliveredAgainState = "delivered-again" + DeliveredAgain = plc.Module + "_" + DeliveredAgainState + // DeliveryWaits holds each dependent's delivery wait the assigner said, one key ``, its value the wait-held said + // (design 47 G §376, §498, A-246). + DeliveryWaitsState = "delivery-waits" + DeliveryWaits = plc.Module + "_" + DeliveryWaitsState + // PutWaits holds each put wait the assigner said, one key `.`, its value the wait's id and the put as said + // (design 47 G A-204, A-211), so a put that ends while the assigner is stopped gets its wait-ended at the next start. + PutWaitsState = "put-waits" + PutWaits = plc.Module + "_" + PutWaitsState // Inputs holds the inputs of the composition steps, `