diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go index 7cfce906..7e8466e2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go @@ -146,7 +146,7 @@ func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesI // sayWanted says `wanted` for a want written or removed now, from the revision the pass keeps (A-61, A-67); with none kept, it is // said in the log that it was not said. fp is the want's, and for a removal the fingerprint last wanted there. -func (p *passer) sayWanted(node, module, fp string, removed bool) { +func (p *passer) sayWanted(node, module, fp, pipeline string, removed bool) { if p.publishWants == nil { return } @@ -156,7 +156,7 @@ func (p *passer) sayWanted(node, module, fp string, removed bool) { return } at := p.clock().UTC() - body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Removed: removed, Revision: g.kept.Revision, Source: g.source, At: &at} + body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Pipeline: pipeline, Removed: removed, Revision: g.kept.Revision, Source: g.source, At: &at} event := strings.TrimPrefix(meshwants.SubjectWanted(keys.EscapeName(node), keys.EscapeName(module)), "mesh.seat."+meshwants.Seat+".event.") id := fmt.Sprintf("wanted.%s.%s.%s.%t.%d.%s", keys.EscapeName(node), keys.EscapeName(module), fp, removed, g.kept.Revision, at.Format(time.RFC3339Nano)) if err := p.wantsSay(event, body, id); err != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index f8c0a2f6..5d48d16e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -586,6 +586,12 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}, reads: reads, seats: gate, askForFile: getenv("MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE")} p.askOperatorFor() // the setting ask-operator-for said once at the start (design 47 G A-425a) + // The setting tier-bound (design 47 H §751) and the one timer that wakes the pass at a tier's deadline (H-A-56). + tierBound, err := tierBoundSetting(getenv) + if err != nil { + return err + } + p.tierBound, p.tierWake = tierBound, wake // The wait on an outside cause (outside.go, design 47 G §706): the keeper's three events say when to judge again, and the // seat mesh-wants' wait-held and wait-ended are said; registered before the hub starts. if bus != nil && kvs[buckets.Waits()] != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 88575bf5..835ceccd 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -110,6 +110,11 @@ type passer struct { // entries are the catalog entries by fingerprint asked of the seat mesh-catalog's verb `entry` (design 47 G §614, A-217): a // held revert's running entry and its want's (revert.go); nil: no revert is asked about. entries *meshcatalogclient.Entries + // tierBound is the setting tier-bound (design 47 H §751; zero: its default), and tierTimer the one timer that wakes the pass at + // the earliest tier's deadline (tiers.go), made with tierWake; nil tierWake sets no timer. + tierBound time.Duration + tierWake func() + tierTimer *time.Timer } // wantsSay says one event of the seat mesh-wants; with no bus to say it on, nothing is said. @@ -267,6 +272,16 @@ func (p *passer) wants(ctx context.Context) { } } wanted := p.w.wantsOf() + // The tier progress moves before the wants are written, so a tier that runs has the next tier's wants written in this pass + // (design 47 H §751). + p.tierDeadline(p.tiers(nodes, func(module, branch string) []string { + for _, g := range p.groups(module, nodesOf[module], assignedBranches[module]) { + if g.branch == branch { + return g.nodes + } + } + return nil + })) chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket())) defer p.unplacedFindings() defer func() { p.seatHolders(ctx, viewOf(p.w.trunkName()).manifests) }() @@ -354,7 +369,7 @@ func (p *passer) putWant(ctx context.Context, node, module string, want mwc.Want } } if p.written(ctx, buckets.Wants(), key, func() ([]byte, error) { return mwc.EncodeWant(want) }) { - p.sayWanted(node, module, fp, want.Removed) + p.sayWanted(node, module, fp, want.Pipeline, want.Removed) } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 8e93cecb..0ca00286 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -151,6 +151,8 @@ type pipelineEnds struct { onBranches func([]string) // trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none. trunk func() string + // now is the clock of the tier progress (tiers.go); nil is time.Now. + now func() time.Time } func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds { @@ -165,6 +167,12 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { return false, nil } changed := false + var kept []tierMover + defer func() { + if err := e.startTiers(p.Pipeline, kept); err != nil { + e.logf("the tier progress of %s is not written: %v", p.Pipeline, err) + } + }() for _, m := range p.Movers { if m.Version == "" || m.Fingerprint == "" || !moverPassed(m) { continue @@ -191,6 +199,7 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { return changed, err } changed = true + kept = append(kept, tierMover{Module: m.Module, Fp: m.Fingerprint, Tier: max(m.Tier, 1)}) break } } @@ -211,10 +220,11 @@ func (e *pipelineEnds) endOf(module, branch string) (End, bool) { } // target is the build module's wants on branch name (design 47 H §750): the newest version released there whose pipeline -// passed, with the pipeline and the mover's rank; false when none is kept. +// passed, with the pipeline and the mover's rank; false when none is kept, or its pipeline's tier progress has not reached its +// rank (§751). func (e *pipelineEnds) target(module, branch string) (assign.Target, bool) { end, ok := e.endOf(module, e.branchOf(module, branch)) - if !ok { + if !ok || !e.tierWritten(end) { return assign.Target{}, false } return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank}, true @@ -286,7 +296,7 @@ func (e *pipelineEnds) branchOf(module, placed string) string { // writtenTools is the tool assigner_written: every key of the state written, with its end. func writtenTools(st store) []stdio.Tool { - return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the source revision and end count of the newest end heard, whether it passed and whether the wants were written from it; with why, why each want is what it is (as assigner_why).", + return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the newest version released whose pipeline passed, with its pipeline and rank; per pipeline whose wants are written tier by tier, its tier progress (tier, since, movers); with why, why each want is what it is (as assigner_why).", Input: schema(map[string]string{"why": "true: why each want is what it is", "module": "with why: a module's name", "node": "with why: a node's name"}, nil, "why"), Run: func(args map[string]any) (any, error) { if args["why"] == true || args["why"] == "true" { @@ -297,7 +307,7 @@ func writtenTools(st store) []stdio.Tool { return nil, err } sort.Strings(ks) - out := map[string]End{} + out := map[string]any{} for _, k := range ks { if strings.HasPrefix(k, whyPrefix) { continue // a want's why (why.go), read by assigner_why @@ -306,6 +316,13 @@ func writtenTools(st store) []stdio.Tool { if err != nil || !ok { continue } + if isTierKey(k) { // a pipeline's tier progress (tiers.go) + var t tierProgress + if json.Unmarshal(raw, &t) == nil { + out[k] = t + } + continue + } var e End if json.Unmarshal(raw, &e) == nil { out[k] = e @@ -336,7 +353,7 @@ func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) { prefix := keys.EscapeName(reg.Module) + "." for _, k := range kept { rest, ok := strings.CutPrefix(k, prefix) - if !ok { + if !ok || isTierKey(k) || strings.HasPrefix(k, whyPrefix) { continue } branch, err := keys.UnescapeName(rest) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index 9d7ff722..53de5242 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -188,15 +188,23 @@ func TestARegistrationNamesTheDefaultBranch(t *testing.T) { _, _ = e.take(ended("main", 5, pass, mover("web", "fp1", "0.1.1", 1, mpr.StepResultOutcomePassed))) _, _ = e.take(ended("beta", 5, pass, mover("web", "fp2", "0.1.2", 1, mpr.StepResultOutcomePassed))) e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false) - if woken == 0 || len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch { - t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", len(st.vals), e.branchOf("web", "")) + endsKept := func() (n int) { // a pipeline's tier progress is not a module's end + for k := range st.vals { + if !isTierKey(k) { + n++ + } + } + return n + } + if woken == 0 || endsKept() != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch { + t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", endsKept(), e.branchOf("web", "")) } if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" { t.Errorf("the default branch's target = %+v %v", tg, ok) } e.registered(mpcat.Registration{Module: "web"}, true) - if len(st.vals) != 0 { - t.Errorf("a registration gone left %d ends", len(st.vals)) + if endsKept() != 0 { + t.Errorf("a registration gone left %d ends", endsKept()) } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go new file mode 100644 index 00000000..f8a040e4 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go @@ -0,0 +1,304 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" + + "mesh-delivery-assigner/internal/assign" +) + +// One pipeline's wants are written tier by tier (design 47 H §751, H-A-10, H-A-56): on a passed pipeline-ended the assigner +// writes the wants of its first tier's movers; once each of them runs at the mover's fingerprint (the node-engine seat's +// `running`, H-A-40) on every present placing machine, it writes the next tier's, and so on. The tier progress is the key +// `pipeline..` of the state written, `{tier, since, movers}`; a tier counts placed movers only, and a machine away +// is not waited for. A tier whose movers do not all run within `tier-bound` ends the delivery loudly: no later tier is written, +// the finding `tier-late` (scope delivery, id the pipeline) is raised naming the movers and machines not running, and +// `pipeline-wanted` says outcome tier-late; once every tier runs it says outcome running and the key goes (H-A-75). tier-late +// clears when the late movers run or a later passed pipeline moves them (H-A-56). + +// tierPrefix starts the key of a pipeline's tier progress in the state written. +const tierPrefix = "pipeline." + +// kindTierLate is the finding of a tier not running within tier-bound (design 47 H §751). +const kindTierLate = "tier-late" + +// defaultTierBound is tier-bound's default, 30 minutes (design 47 H §751). +const defaultTierBound = 30 * time.Minute + +// tierMover is one mover of a passed pipeline whose version the state written keeps: its module, fingerprint and tier. +type tierMover struct { + Module string `json:"module"` + Fp string `json:"fp"` + Tier int64 `json:"tier"` +} + +// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, and the movers. +type tierProgress struct { + Tier int64 `json:"tier"` + Since time.Time `json:"since"` + Movers []tierMover `json:"movers"` +} + +// tierKey is the key of pipeline's tier progress; pipeline is the runner's key `.`, its branch escaped. +func tierKey(pipeline string) string { return tierPrefix + pipeline } + +// isTierKey says whether a key of the state written is a pipeline's tier progress: three parts, the first `pipeline` (a module's +// end has two). +func isTierKey(key string) bool { + return strings.HasPrefix(key, tierPrefix) && strings.Count(key, ".") == 2 +} + +// startTiers writes the tier progress of a passed pipeline whose movers' versions are now kept, its first tier the lowest of +// theirs, since now. A pipeline heard again writes nothing. +func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error { + if len(movers) == 0 { + return nil + } + first := movers[0].Tier + for _, m := range movers { + first = min(first, m.Tier) + } + value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers}) + if err != nil { + return err + } + if err := e.st.swap(tierKey(pipeline), value, 0); err != nil && !errors.Is(err, errSwapped) { + return err + } + return nil +} + +func (e *pipelineEnds) clock() time.Time { + if e.now != nil { + return e.now() + } + return time.Now() +} + +// tiersOf is the tier progress of pipeline, with its revision; false when none stands. +func (e *pipelineEnds) tiersOf(pipeline string) (tierProgress, uint64, bool) { + raw, rev, err := e.st.get(tierKey(pipeline)) + var t tierProgress + if err != nil || raw == nil || json.Unmarshal(raw, &t) != nil { + return tierProgress{}, 0, false + } + return t, rev, true +} + +// tierWritten says whether the want of an end may be written: its pipeline's tier progress reached the end's rank, or no +// progress stands for its pipeline (every tier ran). +func (e *pipelineEnds) tierWritten(end End) bool { + t, _, ok := e.tiersOf(end.Pipeline) + return !ok || end.Rank <= t.Tier +} + +// tierBound is the setting tier-bound; zero or less is its default. +func (p *passer) tierBoundOf() time.Duration { + if p.tierBound > 0 { + return p.tierBound + } + return defaultTierBound +} + +// tiers moves each pipeline's tier progress (design 47 H §751): a tier whose placed movers all run on every present placing machine +// gives way to the next, the last to the end said `running`; a tier past tier-bound is late. placedOn is, by module, the nodes +// placing it on branch. It returns the earliest deadline still to come (zero: none), for one timer to wake the pass at. +func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []string) time.Time { + if p.ends == nil { + return time.Time{} + } + ks, err := p.ends.st.keys() + if err != nil { + p.logf("the tier progress cannot be listed: %v", err) + return time.Time{} + } + sort.Strings(ks) + now := p.clock().UTC() + bound := p.tierBoundOf() + var next time.Time + for _, key := range ks { + if !isTierKey(key) { + continue + } + pipeline := strings.TrimPrefix(key, tierPrefix) + branchEsc, source, _ := strings.Cut(pipeline, ".") + branch, err := keys.UnescapeName(branchEsc) + if err != nil { + continue + } + for { + t, rev, ok := p.ends.tiersOf(pipeline) + if !ok { + break + } + notRunning := p.notRunning(n, placedOn, pipeline, branch, t, t.Tier) + late := !now.Before(t.Since.Add(bound)) + if late { + p.tierLate(pipeline, branch, source, t, notRunning) + break + } + if len(notRunning) > 0 { + if d := t.Since.Add(bound); next.IsZero() || d.Before(next) { + next = d + } + break + } + // The tier runs: the next tier's wants are written, or every tier ran. + later := int64(0) + for _, m := range t.Movers { + if m.Tier > t.Tier && (later == 0 || m.Tier < later) { + later = m.Tier + } + } + if later == 0 { + if err := p.ends.st.drop(key); err != nil { + p.logf("the tier progress %s is not deleted: %v", key, err) + break + } + p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeRunning, nil) + break + } + value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers}) + if err == nil { + err = p.ends.st.swap(key, value, rev) + } + if errors.Is(err, errSwapped) { + continue + } + if err != nil { + p.logf("the tier progress %s is not moved to tier %d: %v", key, later, err) + break + } + p.logf("pipeline %s: tier %d runs on every present placing machine, so tier %d's wants are written", pipeline, t.Tier, later) + } + } + return next +} + +// notRunning is, of tier's movers in t whose kept end still names pipeline, those not running at their fingerprint on a present +// placing machine, with those machines. A mover a later pipeline moved counts as run; one placed nowhere counts as run. +func (p *passer) notRunning(n assign.Nodes, placedOn func(module, branch string) []string, pipeline, branch string, t tierProgress, tier int64) []meshwants.NotRunning { + var out []meshwants.NotRunning + for _, m := range t.Movers { + if m.Tier != tier { + continue + } + if end, ok := p.ends.endOf(m.Module, branch); !ok || end.Pipeline != pipeline { + continue // a later pipeline moved it + } + var machines []string + for _, node := range placedOn(m.Module, branch) { + if online, known := n.Online[node]; known && !online { + continue // a machine away is not waited for + } + r, ok := n.Running[node][m.Module] + if !ok || r.Stopped || r.Fingerprint != m.Fp { + machines = append(machines, node) + } + } + if len(machines) > 0 { + sort.Strings(machines) + out = append(out, meshwants.NotRunning{Module: m.Module, Machines: machines}) + } + } + return out +} + +// tierLate ends a pipeline's delivery at a tier past tier-bound: the finding tier-late is raised and pipeline-wanted says tier-late, +// once each; the finding clears once the late movers run or a later pipeline moved them. The progress stays, so no later tier is +// written, until no later tier's mover is kept at this pipeline any more. +func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRunning []meshwants.NotRunning) { + o := tierLateFinding(pipeline, t.Tier, notRunning) + key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind) + if len(notRunning) > 0 { + if !p.said[key+".raised"] { + if p.findings.Put != nil { + if err := p.findings.Put(o.Finding(p.clock())); err != nil { + p.logf("the finding %s is not written: %v", key, err) + return + } + } + p.said[key+".raised"] = true + delete(p.said, key+".cleared") + p.logf("pipeline %s: tier %d is late (tier-bound %s): %s; no later tier is written", pipeline, t.Tier, p.tierBoundOf(), o.Summary) + p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeTierLate, notRunning) + } + } else if !p.said[key+".cleared"] { + if p.findings.Delete != nil { + if err := p.findings.Delete(key); err != nil { + p.logf("the finding %s is not deleted: %v", key, err) + return + } + } + p.said[key+".cleared"] = true + delete(p.said, key+".raised") + } + for _, m := range t.Movers { + if m.Tier <= t.Tier { + continue + } + if end, ok := p.ends.endOf(m.Module, branch); ok && end.Pipeline == pipeline { + return // a later tier's mover still kept at this pipeline: its want stays unwritten + } + } + if len(notRunning) == 0 { + if err := p.ends.st.drop(tierKey(pipeline)); err != nil { + p.logf("the tier progress of %s is not deleted: %v", pipeline, err) + } + } +} + +// tierLateFinding is the finding tier-late of pipeline's tier, naming the movers and machines not running (design 47 H §751). +func tierLateFinding(pipeline string, tier int64, notRunning []meshwants.NotRunning) assign.Observed { + var named []string + for _, nr := range notRunning { + named = append(named, fmt.Sprintf("%s on %s", nr.Module, strings.Join(nr.Machines, ", "))) + } + summary := fmt.Sprintf("pipeline %s: tier %d does not run within tier-bound: %s", pipeline, tier, strings.Join(named, "; ")) + return assign.Observed{Scope: mcc.ScopeDelivery, ID: pipeline, Kind: kindTierLate, Severity: mcseat.FindingSeverityWarning, + Summary: summary, Headline: "A delivery's tier does not run in time", + Explanation: summary + ". The assigner writes no later tier of this pipeline; the delivery ends here.", + Source: "assigner." + kindTierLate} +} + +// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome). +func (p *passer) sayPipelineWanted(pipeline, branch, source, outcome string, notRunning []meshwants.NotRunning) { + var src int64 + _, _ = fmt.Sscan(source, &src) + body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: src, Outcome: outcome, NotRunning: notRunning} + event := meshwants.EventPipelineWanted + "." + pipeline + if err := p.wantsSay(event, body, "pipeline-wanted."+pipeline+"."+outcome); err != nil { + p.logf("pipeline-wanted of %s is not said: %v", pipeline, err) + } +} + +// tierDeadline sets the one timer that wakes the pass at the earliest tier's deadline (H-A-56); zero stops it. +func (p *passer) tierDeadline(at time.Time) { + if p.tierWake == nil { + return + } + if p.tierTimer != nil { + p.tierTimer.Stop() + } + if at.IsZero() { + return + } + p.tierTimer = time.AfterFunc(max(at.Sub(p.clock()), 0), p.tierWake) +} + +// tierBoundSetting is the setting tier-bound in minutes (design 47 H §751), read at the start from the file the manifest declares +// with the content ${config:tier-bound}; zero, its default, for a file missing or empty. +func tierBoundSetting(getenv func(string) string) (time.Duration, error) { + minutes, err := settingFile(getenv("MESH_ASSIGNER_TIER_BOUND_FILE")) + return time.Duration(minutes) * time.Minute, err +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go new file mode 100644 index 00000000..39d9bb39 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/views" + + "mesh-delivery-assigner/internal/assign" + "mesh-delivery-assigner/internal/buckets" +) + +// A7/AC1, A7/AC2 (design 47 H §751, H-A-56): a pipeline of a provider (tier 1) and its consumer (tier 2) writes the provider's +// wants first and the consumer's only once the provider runs at its fingerprint on every present placing machine (one machine +// away is not waited for), then says pipeline-wanted running and drops its progress; a provider that never runs is late at +// tier-bound: the finding tier-late names it and its machine, pipeline-wanted says tier-late, and no consumer want is written. +func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { + db, web := meshcatalogclient.SourcePrefix2+strings.Repeat("d", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64) + for _, runs := range []bool{true, false} { + start := time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC) + now := start + wantsKV := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, start) + p.now = func() time.Time { return now } + p.say = func(oc.Condition) error { return nil } + findings := map[string]mcseat.Finding{} + p.findings = assign.Findings{Put: func(f mcseat.Finding) error { findings[views.FindingKey(f.Scope, f.ID, f.Kind)] = f; return nil }, + Delete: func(key string) error { delete(findings, key); return nil }} + var said []meshwants.PipelineWanted + p.publishWants = func(event string, body any, _ string) error { + if b, ok := body.(meshwants.PipelineWantedEvent); ok { + if event != "pipeline-wanted."+trunkBranch+".5" { + t.Errorf("pipeline-wanted said as %s", event) + } + said = append(said, b) + } + return nil + } + for i, m := range []struct{ name, fp string }{{"db", db}, {"web", web}} { + digest := strings.Repeat(string(rune('e'+i)), 64) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.fp, Manifest: digest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) + putModuleRaw(p.w, trunkBranch, m.name, mod) + p.w.manifests[digest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}} + } + placeOn(t, p.w, "g14", "db", "web") + placeOn(t, p.w, "ace", "db", "web") + p.w.heardPresence("ace", true, start) // away: not waited for + p.w.mu.Lock() + p.w.runningHeard = true + p.w.mu.Unlock() + p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now } + if _, err := p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed), + mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))); err != nil { + t.Fatal(err) + } + wanted := func(node, module string) bool { + key, _ := mwc.WantKey(node, module) + _, ok := wantsKV.values[key] + return ok + } + p.wants(context.Background()) + if !wanted("g14", "db") || !wanted("ace", "db") || wanted("g14", "web") || wanted("ace", "web") { + t.Fatalf("runs %v: tier 1 alone is not what is wanted first: %v", runs, wantsKV.values) + } + if runs { + raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: "db", Fingerprint: db}) + if err := p.w.hearRunning("running.g14.db", raw); err != nil { + t.Fatal(err) + } + now = start.Add(time.Minute) + p.wants(context.Background()) + if !wanted("g14", "web") || !wanted("ace", "web") { + t.Fatalf("the consumer's want is not written once its provider runs: %v", wantsKV.values) + } + raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web}) + _ = p.w.hearRunning("running.g14.web", raw) + p.wants(context.Background()) + if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || len(findings) != 0 { + t.Errorf("every tier runs: said %+v, findings %v", said, findings) + } + if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok { + t.Error("the tier progress stands after every tier ran") + } + continue + } + now = start.Add(29 * time.Minute) + p.wants(context.Background()) + if len(findings) != 0 || len(said) != 0 { + t.Fatalf("late before tier-bound: %v, %+v", findings, said) + } + now = start.Add(30 * time.Minute) + p.wants(context.Background()) + p.wants(context.Background()) + f, ok := findings["delivery."+assign.KeyID(trunkBranch+".5")+".tier-late"] + if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") { + t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings) + } + if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" { + t.Errorf("pipeline-wanted tier-late is not said once: %+v", said) + } + if wanted("g14", "web") || wanted("ace", "web") { + t.Error("a consumer's want is written after its provider's tier was late") + } + } +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index a98c65bd..fc5e108d 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -166,7 +166,7 @@ func TestManifest(t *testing.T) { } file, _ := os.ReadFile("module.json") if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) || - raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 8 || + raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 9 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" || raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) { t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err) @@ -181,6 +181,7 @@ func TestManifest(t *testing.T) { {"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE", "${dir:state}/composition-step-bound", "composition-step-bound"}, {"MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE", "${dir:state}/run-untaken-bound", "run-untaken-bound"}, {"MESH_ASSIGNER_BLAME_TRIES_FILE", "${dir:state}/blame-tries", "blame-tries"}, + {"MESH_ASSIGNER_TIER_BOUND_FILE", "${dir:state}/tier-bound", "tier-bound"}, // design 47 H §751 } { if !strings.Contains(string(file), `"`+f.env+`": "`+f.path+`"`) || !strings.Contains(string(file), `"path": "`+f.path+`"`) || !strings.Contains(string(file), `"content": "${config:`+f.key+`}"`) { diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 9bcb6689..a807397e 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -178,6 +178,14 @@ "trusted": false, "default": "take-off,remove-node,stops" }, + "tier-bound": { + "type": "number", + "description": "minutes each tier of a pipeline's movers has to run on every present placing machine before the assigner writes the next tier's wants; a tier that does not run within it ends the delivery with the finding tier-late and writes no later tier (design 47 H section 751).", + "minimum": 1, + "maximum": 10080, + "trusted": false, + "default": 30 + }, "composition-runs-cap-mib": { "type": "number", "description": "the cap of the state composition-runs in MiB, the process refusing a new composition run at it with its steps ending could not run naming the fill (design 47 G section 719, A-469d); its maximum is the manifest's max-mib.", @@ -260,6 +268,14 @@ "mode": "0600", "content": "${config:ask-operator-for}" }, + { + "id": "tier-bound", + "type": "file", + "path": "${dir:state}/tier-bound", + "owner": "mesh-delivery-assigner", + "mode": "0600", + "content": "${config:tier-bound}" + }, { "id": "composition-runs-cap-mib", "type": "file", @@ -313,6 +329,7 @@ "MESH_ASSIGNER_OUTSIDE_WAIT_FILE": "${dir:state}/outside-wait-minutes", "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": "${dir:state}/outside-settle-seconds", "MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE": "${dir:state}/ask-operator-for", + "MESH_ASSIGNER_TIER_BOUND_FILE": "${dir:state}/tier-bound", "MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "${dir:state}/composition-runs-cap-mib", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "${dir:state}/composition-runs-kept", "MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "${dir:state}/composition-step-bound",