diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go index bd7519586..7cb0c03ba 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -28,6 +28,8 @@ package main import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -110,8 +112,8 @@ type Composer func(ctx context.Context, node string, keptAt map[string]string) C // Composition is a node's plans at its wanted builds and at its newest, and where the time went. type Composition struct { AtWants, Newest HeldPlans - // Unpublished is each module whose composition at the build it is read at is not published yet: the node's picture - // is not whole, so no retirement is judged on it (issue 940). + // Unpublished is each module whose composition at the build it is read at is not published yet: its own plan and keys + // are kept as they were, and nothing else on the node waits for it (issue 940). Unpublished []string // UnpublishedNewest is each module whose composition at its newest build (which is not its wanted build) is not // published yet: the newest build's needs are unread, so no key only that build requires is judged (issue 1966). @@ -1102,6 +1104,25 @@ const askHeldWithin = time.Minute // askWaits is what a retiring key's why says while its ask waits for an open one to end. const askWaits = "its ask waits until fewer than 3 of mesh-provisioner's asks stand open, the router showing at most 3 per asker" +// askRefusedNote opens what a retiring key's why says once its ask is refused: the ask is not raised, and the key says +// so here, and provisioner_health names the key with those words. +const askRefusedNote = "its ask is refused: " + +// askID is a retiring ask's id: letters, digits and - only, of bounded length (the router refuses over 64), made from the +// key by its digest and the time, so one key's asks differ and no key's length matters. +func askID(key string, at time.Time) string { + sum := sha256.Sum256([]byte(key)) + return "retire-" + hex.EncodeToString(sum[:8]) + "-" + fmt.Sprint(at.Unix()) +} + +// withoutAskNotes is why without what ask() said of its ask (askWaits, askRefusedNote). +func withoutAskNotes(why string) string { + if i := strings.Index(why, askRefusedNote); i >= 0 { + why = why[:i] + } + return strings.TrimSuffix(strings.TrimSuffix(why, askWaits), "; ") +} + // errAskWaits is an ask held back by askOpenAtMost: no fault, raised once one ends. var errAskWaits = errors.New(askWaits) @@ -1127,21 +1148,26 @@ func (h *Holder) ask(p *meshprovisionerclient.Provision, key string) error { } if h.openAsks() >= askOpenAtMost { if !strings.Contains(p.Why, askWaits) { - p.Why = strings.TrimPrefix(p.Why+"; "+askWaits, "; ") + p.Why = strings.TrimPrefix(withoutAskNotes(p.Why)+"; "+askWaits, "; ") } return errAskWaits } now := h.Now().UTC() - id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix()) + id := askID(key, now) a := retiringAsk(key, *p, now, id) - if err := occ.Check(a, now); err != nil { + // A refused ask is said in the key's why, where the operator reads why it stands, and on provisioner_health. + refused := func(err error) error { + p.Why = strings.TrimPrefix(withoutAskNotes(p.Why)+"; "+askRefusedNote+err.Error(), "; ") return err } + if err := occ.Check(a, now); err != nil { + return refused(err) + } if err := h.Asks.Publish(oc.AcceptSubjectAsk(Module), a, id); err != nil { - return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err) + return refused(fmt.Errorf("the retiring ask of %s could not be published: %w", key, err)) } p.Ask = id - p.Why = strings.TrimSuffix(strings.TrimSuffix(p.Why, askWaits), "; ") + p.Why = withoutAskNotes(p.Why) return nil } @@ -1372,6 +1398,22 @@ func (h *Holder) Faults() map[string]string { return out } +// RefusedAsks is each retiring key whose ask the router's rules refuse, as its key with its why: read from the keys +// themselves, so a restart keeps it (ask). +func (h *Holder) RefusedAsks() map[string]string { + out := map[string]string{} + keys, _ := h.Store.Keys(ProvisionsState) + for _, k := range keys { + if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil { + if p, err := meshprovisionerclient.Decode(e.Value); err == nil && p.State == meshprovisionerclient.Retiring && + p.Ask == "" && strings.Contains(p.Why, askRefusedNote) { + out[k] = p.Why + } + } + } + return out +} + // LastingHolds is each module held unpublished at one build for longer than unpublishedFor, as "." with // the hold's words, which name its build, and since when. func (h *Holder) LastingHolds() map[string]string { @@ -1546,8 +1588,8 @@ func orSaid(words ...string) string { // builds a module is read at the fingerprint keptAt names, else at the indexer's newest (FP2); at the newest, each placed // module at FP2. A key whose engine refused the module holds it with the engine's words; a key not published yet holds // it at its wanted build (its last plan kept, as for a refusal) and at its newest too (issue 1966), whose build the engine -// writes once it is done (ADR 0593 §6), and says it unpublished, so the pass judges no retirement on the node (issue -// 940). A placed module no build of which is known is held too. A plan absent from a key is a module that hands the +// writes once it is done (ADR 0593 §6), and says it unpublished, so only that module's own plan and keys are kept as they were and nothing else on the +// node waits for it (issue 940). A placed module no build of which is known is held too. A plan absent from a key is a module that hands the // provisioner nothing. func FromCompositions(s Store) Composer { return func(_ context.Context, node string, keptAt map[string]string) Composition { diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go index f1d67b485..63376d196 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" assignerclient "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" meshseat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" @@ -35,6 +36,8 @@ type router struct { cancelled []string // failCancel is a router that refuses a cancel. failCancel bool + // refuses is a router whose rules refuse every ask, with these words. + refuses string } func (r *router) Publish(subject string, body any, id string) error { @@ -49,6 +52,9 @@ func (r *router) Publish(subject string, body any, id string) error { delete(r.records, c.ID) return nil } + if r.refuses != "" { + return errors.New(r.refuses) + } r.asked[id] = body.(oc.Ask) if !r.drop { r.records[id], _ = json.Marshal(map[string]any{"state": "open", "ask": body}) @@ -870,3 +876,33 @@ func TestAnAskTheRouterDoesNotHoldIsRaisedAgain(t *testing.T) { } } } + +// Issue 1966, U7 (design 47 fix-unpublished-placed-module): a retiring ask's id is bounded and made from the key, so a +// long key is asked; an ask the router refuses is said in the key's why and in provisioner_health, and the key is +// asked once the refusal ends. +func TestARetiringAskIsBoundedAndARefusedOneIsSaid(t *testing.T) { + m := newMesh() + long := "claude-licence-manager-ab" // with this key the old id "retire-g14-...-" is 65 characters + key := "g14." + long + ".postgres-database" + m.atWant["g14"] = map[string]catalogue.Plan{long: planOf(pgDB), "dunst": planOf(oidc)} + m.compose(t, "g14") + m.atWant["g14"] = map[string]catalogue.Plan{"dunst": planOf(oidc)} + m.r.refuses = "its id is refused" + c := m.compose(t, "g14") + p := m.state(t, key) + if p.Ask != "" || !strings.Contains(p.Why, askRefusedNote) || !strings.Contains(p.Why, "its id is refused") || !strings.Contains(strings.Join(c.Problems, " "), "its id is refused") { + t.Errorf("a refused ask: ask %q, why %q, problems %v", p.Ask, p.Why, c.Problems) + } + if a := m.h.health(); a.Healthy || !strings.Contains(a.Why, key) || !strings.Contains(a.Why, "its id is refused") { + t.Errorf("health with a refused ask: %+v", a) + } + m.r.refuses = "" + m.compose(t, "g14") + p = m.state(t, key) + if len(p.Ask) == 0 || len(p.Ask) > 64 || !occ.UsableID(p.Ask) || strings.Contains(p.Why, askRefusedNote) { + t.Errorf("the ask once the refusal ends: %q, why %q", p.Ask, p.Why) + } + if a := m.h.health(); !a.Healthy { + t.Errorf("health once the ask is raised: %+v", a) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go index 778e488f2..227695642 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go @@ -91,17 +91,17 @@ func holderTools(h *Holder) []stdio.Tool { "a pass held back a retirement of most of a node's consumers at once as a fault of the mesh (issue 940), " + "naming the node, the consumers and the likely cause, or while a module's composition has stayed " + "unpublished at one build, wanted or newest, for longer than 45 minutes (longer than a build takes), naming the module, " + - "the build and since when.", + "the build and since when, or while a retiring provision's ask is refused by the operator channel's rules, naming the key and the refusal.", Run: func(map[string]any) (any, error) { return h.health(), nil }}, } } // health is the holder's runtime health: unhealthy while a retirement is held back as a fault of the mesh. func (h *Holder) health() health.Answer { - faults, holds := h.Faults(), h.LastingHolds() - if len(faults)+len(holds) == 0 { + faults, holds, refused := h.Faults(), h.LastingHolds(), h.RefusedAsks() + if len(faults)+len(holds)+len(refused) == 0 { return health.Healthy("No pass holds back a retirement of most of a node's consumers, and no module's " + - "composition has stayed unpublished at one build for longer than " + unpublishedFor.String() + ".") + "composition has stayed unpublished at one build for longer than " + unpublishedFor.String() + ", and no retiring ask is refused.") } var said []string for _, n := range sortedKeys(faults) { @@ -110,6 +110,9 @@ func (h *Holder) health() health.Answer { for _, k := range sortedKeys(holds) { said = append(said, k+": "+holds[k]+"; its plan and its provisions are kept as they were.") } + for _, k := range sortedKeys(refused) { + said = append(said, k+" is retiring and "+refused[k]+".") + } return health.Unhealthy(strings.Join(said, " ")) }