diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go index 5c45c24f4..6d8e4a9a5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sort" "strings" "time" @@ -62,23 +63,47 @@ func isTierKey(key string) bool { } // startTiers writes the tier progress of a passed pipeline whose movers' versions are now kept, its first tier the lowest of -// theirs, since now. A pipeline heard again writes nothing. +// theirs, since now. A pipeline heard again adds to the progress that stands each mover it does not hold, at the mover's tier +// (H-A-270, H-A-273): its want is written when the progress reaches that tier, or at once when that tier ran already, never out +// of tier order. A mover the progress holds writes nothing. func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMover) error { if len(movers) == 0 { return nil } - first := movers[0].Tier - for _, m := range movers { - first = min(first, m.Tier) - } - value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers, Source: source}) - if err != nil { + for { + t, rev, ok := e.tiersOf(pipeline) + if !ok { + first := movers[0].Tier + for _, m := range movers { + first = min(first, m.Tier) + } + t = tierProgress{Tier: first, Since: e.clock().UTC(), Source: source} + } + added := false + for _, m := range movers { + if !slices.ContainsFunc(t.Movers, func(h tierMover) bool { return h.Module == m.Module }) { + t.Movers, added = append(t.Movers, m), true + } + } + if !added { + return nil + } + value, err := json.Marshal(t) + if err != nil { + return err + } + err = e.st.swap(tierKey(pipeline), value, rev) + if errors.Is(err, errSwapped) { + if _, _, held := e.tiersOf(pipeline); !ok && !held { + return nil // a key that stands but does not read is left as it is + } + continue + } + if err == nil && ok { + e.logf("pipeline %s: a mover whose version came after its end joins the tier progress at its tier", pipeline) + } return err } - if err := e.st.swap(tierKey(pipeline), value, 0); err != nil && !errors.Is(err, errSwapped) { - return err - } - return nil } func (e *pipelineEnds) clock() time.Time { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go index 9937f7754..c40e74c49 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go @@ -125,3 +125,74 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { } } } + +// H-A-270, H-A-273: a mover whose version came after its pipeline's end joins the tier progress at its tier when the end is taken +// again. A late consumer (tier 2) is wanted only once its provider (tier 1) runs on every placing machine, and the tier above it +// (3) only once it runs: never out of tier order. A late mover of a tier already run is wanted at once. +func TestALateMoverJoinsTheTierProgressAtItsTier(t *testing.T) { + fps := map[string]string{} + for i, m := range []string{"db", "web", "api", "cache"} { + fps[m] = meshcatalogclient.SourcePrefix2 + strings.Repeat(string(rune('a'+i)), 64) + } + start := time.Date(2026, 11, 21, 4, 0, 0, 0, time.UTC) + now := start + wantsKV := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, start) + p.now = func() time.Time { return now } + p.say = func(oc.Condition) error { return nil } + p.publishWants = func(string, any, string) error { return nil } + for i, m := range []string{"db", "web", "api", "cache"} { + digest := strings.Repeat(string(rune('1'+i)), 64) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fps[m], Manifest: digest, Repository: "novox/" + m, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) + putModuleRaw(p.w, trunkBranch, m, mod) + p.w.manifests[digest] = parsed{module: m, manifest: catalogue.Manifest{Module: m}} + } + placeOn(t, p.w, "g14", "db", "web", "api", "cache") + p.w.mu.Lock() + p.w.runningHeard = true + p.w.mu.Unlock() + p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now } + pass := mpr.StepResultOutcomePassed + take := func(webVersion, cacheVersion string) { + t.Helper() + e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", fps["db"], "0.2.0", 1, pass), + mover("web", fps["web"], webVersion, 2, pass), mover("api", fps["api"], "0.3.0", 3, pass), mover("cache", fps["cache"], cacheVersion, 1, pass)) + if _, err := p.ends.take(e); err != nil { + t.Fatal(err) + } + p.wants(context.Background()) + } + runs := func(module string) { + t.Helper() + raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: module, Fingerprint: fps[module]}) + if err := p.w.hearRunning("running.g14."+module, raw); err != nil { + t.Fatal(err) + } + now = now.Add(time.Minute) + p.wants(context.Background()) + } + wanted := func(want ...string) { + t.Helper() + var got []string + for _, m := range []string{"db", "web", "api", "cache"} { + key, _ := mwc.WantKey("g14", m) + if _, ok := wantsKV.values[key]; ok { + got = append(got, m) + } + } + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("wanted %v, want %v", got, want) + } + } + take("", "") + wanted("db") + take("0.1.2", "") // web's version came after the end: it joins tier 2 + wanted("db") + runs("db") + wanted("db", "web") + runs("web") + wanted("db", "web", "api") + take("0.1.2", "0.4.0") // cache's tier (1) ran already: wanted at once + wanted("db", "web", "api", "cache") +}