diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index dc673330..202020ac 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -253,7 +253,7 @@ func main() { return err })...) tools = append(tools, index.PinsTools(keptRead{stdio.State(index.PinsState)}, revisions, asks, time.Now, func(what, why string) error { - _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.UnpinTool, Verb: "unpin"}) + _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.PinDropTool, Verb: "pin-drop"}) return err })...) tools = append(tools, index.HeldTools(keptRead{stdio.State(index.HeldState)}, asks, time.Now, func(what, why string) error { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 477cd3cc..f27fc75e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -20,7 +20,7 @@ type takerIn interface { RecomputeByHand(ctx context.Context, by, why string) index.TakenIn DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn - Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn + PinDrop(ctx context.Context, key, by, why string) index.TakenIn DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn // HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744). @@ -66,8 +66,8 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.NamedFingerprint != "" && a.Drop { asked, r = "drop of a member of a fingerprint's names", ix.NamedDrop(ctx, a.NamedFingerprint, a.NamedMember, a.By, a.Why) a.Module = r.Module - } else if a.UnpinRevision != "" { - asked, r = "unpin of a record from a catalog graph revision", ix.Unpin(ctx, a.UnpinRevision, a.UnpinRecord, a.By, a.Why) + } else if a.PinDrop != "" { + asked, r = "drop of a pin of the catalog graph revisions", ix.PinDrop(ctx, a.PinDrop, a.By, a.Why) a.Module = r.Module } else if a.RunDrop != "" { asked, r = "drop of a run on request", ix.DropRunByHand(ctx, a.RunDrop, a.By, a.Why) diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index c5a59372..ac2fcdd9 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -36,6 +36,11 @@ type HeldTakeIn struct { Since time.Time `json:"since"` } +// PendingAnswer is indexer_pending's answer (H-A-102): every branch's held take-in, in the state held's form (since in RFC 3339). +type PendingAnswer struct { + Held []HeldTakeIn `json:"held"` +} + // hold writes the branch's held take-in and says take-in.pending, once per source: a source already held is neither written nor // said again, so its since stays. func (ix *Indexer) hold(ctx context.Context, e meshdelivery.SourceMovedEvent, why string) { @@ -120,7 +125,10 @@ func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error) "could not take in (the catalog store full, a read of the forge unanswered), why and since when; it stays unacknowledged " + "until the indexer's room wakes it. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return ReadHeld(context.Background(), held) }}, + Run: func(map[string]any) (any, error) { + h, err := ReadHeld(context.Background(), held) + return PendingAnswer{Held: h}, err + }}, {Name: TakeInPendingTool, Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " + "answers whether it was taken in. indexer_pending names the held branches.", diff --git a/modules/mesh-delivery-indexer/internal/index/held_test.go b/modules/mesh-delivery-indexer/internal/index/held_test.go new file mode 100644 index 00000000..5e90dfd0 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/held_test.go @@ -0,0 +1,66 @@ +package index + +import ( + "encoding/json" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Design 47 H §744, §765, H-A-97, H-A-57: the hand tools of the held take-in and the pin drop write asks the indexer reads +// back, each recorded in the hand-act log before it is asked where it is a hand act. +func TestTheHeldAndPinToolsAskWhatTheIndexerReads(t *testing.T) { + var asked []TakeInAsk + var recorded []string + ask := func(_ string, value []byte) (TakeInAsk, error) { + a, err := DecodeTakeIn(value) + if err != nil { + t.Errorf("the indexer cannot read the ask %s: %v", value, err) + } + asked = append(asked, a) + return a, nil + } + record := func(what, _ string) error { recorded = append(recorded, what); return nil } + tools := map[string]stdio.Tool{} + for _, tl := range append(HeldTools(newMem(), ask, time.Now, record), PinsTools(newMem(), newMemObjects(0), ask, time.Now, record)...) { + tools[tl.Name] = tl + } + for _, c := range []struct { + tool string + args map[string]any + want func(TakeInAsk) bool + }{ + {TakeInPendingTool, map[string]any{"branch": "main"}, func(a TakeInAsk) bool { return a.Held == "main" && !a.Skip }}, + {TakeInSkipTool, map[string]any{"branch": "main", "source": float64(4), "why": "stale"}, func(a TakeInAsk) bool { return a.Held == "main" && a.HeldSource == 4 && a.Skip }}, + {PinDropTool, map[string]any{"key": "main.7", "why": "no pipeline"}, func(a TakeInAsk) bool { return a.PinDrop == "main.7" }}, + } { + asked = nil + if _, err := tools[c.tool].Run(c.args); err != nil || len(asked) != 1 || !c.want(asked[0]) { + t.Errorf("%s asked %+v (%v)", c.tool, asked, err) + } + } + if len(recorded) != 2 { + t.Errorf("recorded %v, want the skip and the pin drop", recorded) + } + held := newMem() + ix := &Indexer{Held: held} + ix.hold(t.Context(), meshdelivery.SourceMovedEvent{Branch: "main", Source: 4, Call: "0123456789abcdef"}, "the catalog store is full") + got, err := HeldTools(held, ask, time.Now, record)[0].Run(map[string]any{}) + data, _ := json.Marshal(got) + var answer struct { + Held []struct { + Branch string `json:"branch"` + Source int64 `json:"source"` + Why string `json:"why"` + Since string `json:"since"` + } `json:"held"` + } + if err != nil || json.Unmarshal(data, &answer) != nil || len(answer.Held) != 1 || answer.Held[0].Branch != "main" || answer.Held[0].Source != 4 || answer.Held[0].Why == "" { + t.Fatalf("indexer_pending answered %s (%v), want {held: [main 4 and why]}", data, err) + } + if _, err := time.Parse(time.RFC3339, answer.Held[0].Since); err != nil { + t.Errorf("since %q is not RFC 3339: %v", answer.Held[0].Since, err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 151cfbae..765f162b 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -30,9 +30,9 @@ const ( PinsCapMiB = 4 // PinMissingKind is the finding of a pinned revision the indexer no longer holds. PinMissingKind = "pin-missing" - // PinsTool reads the pins, UnpinTool removes one record's pin by hand. - PinsTool = "indexer_pins" - UnpinTool = "indexer_unpin" + // PinsTool reads the pins, PinDropTool drops one pin by its key by hand (design 47 H §765, H-A-57). + PinsTool = "indexer_pins" + PinDropTool = "indexer_pin_drop" ) // PinValue is one revision's key: the records pinning it, and whether the revision was not held when a record pinned it. @@ -302,24 +302,27 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) { return removed, nil } -// Unpin removes one record's pin from a revision by hand (indexer_unpin), for a freed that was lost, and the removal rule then applies. -func (ix *Indexer) Unpin(ctx context.Context, revision, record, by, why string) TakenIn { - r := TakenIn{Module: "pins " + revision} +// PinDrop drops one pin of the state pins by hand, by its key . (indexer_pin_drop, design 47 H §765, H-A-57): +// every record's, for a pin that names no running pipeline; the finding pin-missing of the key is cleared, and the removal +// rule then applies. +func (ix *Indexer) PinDrop(ctx context.Context, key, by, why string) TakenIn { + r := TakenIn{Module: "pins " + key} if ix.Pins == nil { - r.Answer = "the indexer keeps no state " + PinsState + ": nothing unpinned" + r.Answer = "the indexer keeps no state " + PinsState + ": nothing dropped" return r } - if _, _, ok := ParsePinKey(revision); !ok { - r.Answer = revision + " is no key of the state " + PinsState + " as indexer_pins names them: nothing unpinned" + if _, _, ok := ParsePinKey(key); !ok { + r.Answer = key + " is no key of the state " + PinsState + " as indexer_pins names them: nothing dropped" return r } - if err := ix.unpin(ctx, revision, record); err != nil { - r.Answer = "the pin of " + record + " on " + revision + " could not be removed: " + err.Error() + if err := write(ctx, ix.Pins, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil { + r.Answer = "the pin " + key + " could not be dropped: " + err.Error() ix.logf("%s", r.Answer) return r } + ix.clearPinMissing(ctx, key) r.Stored = true - r.Answer = fmt.Sprintf("the pin of %s on %s is removed by %s (%s)", record, revision, by, why) + r.Answer = fmt.Sprintf("the pin %s is dropped by %s (%s)", key, by, why) ix.logf("%s", r.Answer) ix.JudgePins(ctx) return r @@ -397,33 +400,32 @@ func PinsTools(pins Store, o Objects, ask func(key string, value []byte) (TakeIn Description: "Every catalog graph revision a delivery record pins in the indexer's state pins (design 47 G §579): the revision's " + "key `.`, the records pinning it, whether it was missing when pinned (finding pin-missing), whether it is " + "kept and whether it is current and why; the kept revisions no record pins; and the state's fill against its cap. " + - "indexer_consumer reads the consumer the events come through; indexer_unpin removes one record's pin.", + "indexer_pin_drop drops one pin by its key.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { return ReadPins(context.Background(), pins, o) }}, - {Name: UnpinTool, - Description: "Remove one record's pin from one revision of the indexer's state pins, for a freed that was lost; the removal rule " + - "then applies (a revision no record pins and not current goes). A hand act, recorded through mesh-hand-acts before it is asked. " + - "Clears the finding pin-missing of a missing revision left with no record. indexer_pins names the revisions and records.", - Input: map[string]any{"type": "object", "required": []string{"revision", "record", "why"}, "properties": map[string]any{ - "revision": map[string]any{"type": "string", "description": "the revision's key `.`, as indexer_pins names it"}, - "record": map[string]any{"type": "string", "description": "the delivery record's id"}, - "why": map[string]any{"type": "string", "description": "why it is unpinned, in plain words"}}}, + {Name: PinDropTool, + Description: "Drop one pin of the indexer's state pins by its key `.`, every record's, for a pin that names no " + + "running pipeline (design 47 H §765); the removal rule then applies (a revision no record pins and not current goes). A hand " + + "act, recorded through mesh-hand-acts before it is asked. Clears the finding pin-missing of the key. indexer_pins names the keys.", + Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ + "key": map[string]any{"type": "string", "description": "the pin's key `.`, as indexer_pins names it"}, + "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}}, Run: func(args map[string]any) (any, error) { str := func(k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } - revision, rec, why := str("revision"), str("record"), str("why") - if _, _, ok := ParsePinKey(revision); !ok || rec == "" || why == "" { - return nil, errors.New("name the revision as indexer_pins names it (revision) and the record (record) and say why (why): an unpin is a hand act, recorded through mesh-hand-acts") + key, why := str("key"), str("why") + if _, _, ok := ParsePinKey(key); !ok || why == "" { + return nil, errors.New("name the pin's key as indexer_pins names it (key) and say why (why): a drop is a hand act, recorded through mesh-hand-acts") } - data, err := json.Marshal(TakeInAsk{V: 1, UnpinRevision: revision, UnpinRecord: rec, Why: why, By: stdio.CallerOf(args), At: now().UTC()}) + data, err := json.Marshal(TakeInAsk{V: 1, PinDrop: key, Why: why, By: stdio.CallerOf(args), At: now().UTC()}) if err != nil { return nil, err } - if err := record("unpinned the record "+rec+" from the catalog graph revision "+revision+" by hand", why); err != nil { - return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was unpinned: %w", err) + if err := record("dropped the pin "+key+" of the catalog graph revisions by hand", why); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err) } - a, err := ask("unpin."+keys.EscapeName(revision)+"."+keys.EscapeName(rec), data) + a, err := ask("pin-drop."+keys.EscapeName(key), data) if err != nil { return nil, err } diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index d0ae03f6..1c48eef8 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -76,16 +76,20 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { t.Errorf("the state holds %v, want no key of the removed revision", keys) } - // A record pins a revision no longer held: its key says missing and the finding names it; unpin clears both. + // A record pins a revision no longer held: its key says missing and the finding names it; indexer_pin_drop of its key + // (design 47 H §765, H-A-57) drops the pin and clears both. started("main.7", 1) if f := ix.Findings.Held(); len(f) != 1 || f[0].Kind != PinMissingKind { t.Fatalf("findings %v, want one %s", f, PinMissingKind) } - if r := ix.Unpin(ctx, "main.1", "pipeline.main.7", "operator", "freed was lost"); !r.Stored { - t.Fatalf("unpin answered %q", r.Answer) + if r := ix.PinDrop(ctx, "main.1", "operator", "names no running pipeline"); !r.Stored { + t.Fatalf("the pin drop answered %q", r.Answer) } if f := ix.Findings.Held(); len(f) != 0 { - t.Errorf("findings %v after unpin, want none", f) + t.Errorf("findings %v after the pin drop, want none", f) + } + if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { + t.Errorf("the state holds %v after the pin drop, want no main.1", keys) } } diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index ea77fd46..3d35b93b 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -62,14 +62,13 @@ type TakeInAsk struct { // NamedFingerprint and NamedMember, with Drop, ask one member to be removed from that fingerprint's set of the state named // (indexer_named_drop, named.go, design 47 G §675). NamedFingerprint string `json:"named-fingerprint,omitempty"` - // UnpinRevision and UnpinRecord ask one record's pin on a revision of the state pins to be removed (indexer_unpin). - UnpinRevision string `json:"unpin-revision,omitempty"` + // PinDrop asks one pin of the state pins, by its key, to be dropped (indexer_pin_drop, design 47 H §765). + PinDrop string `json:"pin-drop,omitempty"` // Held names the watched branch whose held source-moved is taken in again now (indexer_take_in_pending) or, with Skip and // HeldSource, ended unwritten (indexer_take_in_skip; design 47 H §744). Held string `json:"held,omitempty"` HeldSource int64 `json:"held-source,omitempty"` Skip bool `json:"skip,omitempty"` - UnpinRecord string `json:"unpin-record,omitempty"` NamedMember string `json:"named-member,omitempty"` // RunDrop asks one run on request, by its group run-, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194). RunDrop string `json:"run-drop,omitempty"` @@ -94,7 +93,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.UnpinRevision == "" && a.RunDrop == "" && a.Registry == nil { + if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.PinDrop == "" && a.Held == "" && a.RunDrop == "" && a.Registry == nil { return a, errors.New("a takeins value naming no module, repository, pending wait or merge") } return a, nil diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index c90e17d5..e66c9f87 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -204,7 +204,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 4d6fbbe2..d482e2da 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -64,7 +64,7 @@ "indexer_seed_again", "indexer_inputs", "indexer_pins", - "indexer_unpin", + "indexer_pin_drop", "indexer_pending", "indexer_take_in_pending", "indexer_take_in_skip",