diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index cde7d759..4a42bcdd 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -286,6 +286,14 @@ func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn { return r } +// TakeInAgain takes a repository in again at its kept merge commit (indexer_take_in_again, H-A-208, H-A-210), one take-in +// again at a time beside the held ones. +func (a actor) TakeInAgain(ctx context.Context, repository string) index.TakenIn { + a.s.again.Lock() + defer a.s.again.Unlock() + return a.Indexer.TakeInAgain(ctx, repository) +} + // HeldSkip ends a branch's held source revision unwritten (indexer_take_in_skip): take-in.taken says it skipped. func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn { r := index.TakenIn{Module: branch} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 14625a36..31c7f4a8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -24,6 +24,8 @@ type takerIn interface { // HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744). HeldAgain(ctx context.Context, branch string) index.TakenIn HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn + // TakeInAgain takes a repository in again at its kept merge commit (H-A-208, H-A-210). + TakeInAgain(ctx context.Context, repository string) index.TakenIn } // takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and @@ -53,6 +55,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.Held != "" { asked, r = "take-in again of a held take-in", ix.HeldAgain(ctx, a.Held) a.Module = a.Held + } else if a.Again != "" { + asked, r = "take-in again of a repository", ix.TakeInAgain(ctx, a.Again) + a.Module = a.Again } else if a.Registry != nil { asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a) } else if a.Recompute { diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index c947f68e..5e8774ad 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -260,6 +260,32 @@ func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error) } return a.Result, nil }}, + {Name: TakeInAgainTool, + Description: "Take a repository in again now at the merge commit the indexer keeps of it (design 47 H-A-208, H-A-210): its " + + "module graph is read again (each module's requirements, transitively) and the modules state recomputed as a " + + "package's publish re-pins it, so a changed take-in rule reaches a repository with no merge since; pins and " + + "fingerprints move, no catalog revision is kept and the gate is not asked. An unknown repository is refused.", + Input: map[string]any{"type": "object", "required": []string{"repository"}, "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "the repository as owner/name, such as novox/mesh-host"}, + "why": map[string]any{"type": "string", "description": "why it is taken in again now"}}}, + Run: func(args map[string]any) (any, error) { + repository := str(args, "repository") + if repository == "" { + return nil, errors.New("name the repository (repository) as owner/name") + } + data, err := json.Marshal(TakeInAsk{V: 1, Again: repository, Why: str(args, "why"), By: stdio.CallerOf(args), At: now().UTC()}) + if err != nil { + return nil, err + } + a, err := ask("again."+keys.EscapeName(repository), data) + if err != nil { + return nil, err + } + if a.Result != nil && !a.Result.Stored { + return a.Result, fmt.Errorf("%s is not taken in again: %s", repository, a.Result.Answer) + } + return a.Result, nil + }}, {Name: TakeInSkipTool, Description: "End a watched branch's held source-moved unwritten (design 47 H §744): it is acknowledged, take-in.taken is said " + "with no entries and skipped, and the hand act is recorded through mesh-hand-acts before it is asked. indexer_pending " + diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again.go b/modules/mesh-delivery-indexer/internal/index/take_in_again.go new file mode 100644 index 00000000..ed37a307 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again.go @@ -0,0 +1,69 @@ +package index + +import ( + "context" + "errors" + "fmt" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// TakeInAgainTool takes a repository in again at its kept merge commit (design 47 H-A-208, H-A-210): a rule change of the +// take-in (such as the walk of the module graph, H-A-201) reaches a repository with no merge since. +const TakeInAgainTool = "indexer_take_in_again" + +// TakeInAgain re-reads the repository's module graph at the merge commit its taken key keeps (wholeAt, the reading of every +// take-in: the kept requirements of each of its modules, H-A-201, H-A-157) and applies it as a take-in does, whose recompute +// of the modules state is the one a publish re-pin runs (recomputeLocked): pins and fingerprints move as they would there. +// It keeps no catalog revision and asks no gate (H-A-210). A repository with no taken key is refused, said in the answer. +func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn { + repository = strings.TrimSpace(repository) + r := TakenIn{Module: repository, Repository: repository} + refuse := func(format string, a ...any) TakenIn { + r.Answer = "refused: " + fmt.Sprintf(format, a...) + ix.logf("take-in again of %s: %s", repository, r.Answer) + return r + } + if ix.Taken == nil || ix.Modules == nil { + return refuse("the indexer keeps no taken merges or no modules, so nothing is taken in again") + } + key, err := trunk.RepositoryKey(repository) + if err != nil { + return refuse("%q names no repository as owner/name", repository) + } + value, _, found, err := ix.Taken.Get(ctx, key) + if errors.Is(err, process.ErrNotYet) { + return refuse("the state taken is not on the bus yet") + } + if err != nil { + return refuse("taken.%s cannot be read: %v", key, err) + } + if !found { + return refuse("the indexer keeps no merge of %s (no key taken.%s): an unknown repository, or one never taken in", repository, key) + } + t, err := takein.DecodeTaken(value) + if err != nil || t.Merge == "" { + return refuse("taken.%s names no merge commit", key) + } + ix.pendingMu.Lock() // one take-in step at a time (pending.go) + defer ix.pendingMu.Unlock() + f, err := ix.wholeAt(ctx, repository, t.Merge) + if err != nil { + return refuse("%v", err) + } + f.Branch = TrunkBranch + applied, err := ix.ApplyMessage(ctx, f) + if err != nil { + return refuse("taking %s at %.12s in again: %v", repository, t.Merge, err) + } + r.Stored, r.Commit = true, t.Merge + r.Answer = fmt.Sprintf("taken in again at %.12s: %d modules written", t.Merge, len(applied.Written)) + if len(applied.Written) > 0 { + r.Answer += " (" + strings.Join(applied.Written, ", ") + ")" + } + ix.logf("take-in again of %s: %s", repository, r.Answer) + return r +} diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go new file mode 100644 index 00000000..0ac11588 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go @@ -0,0 +1,72 @@ +package index + +import ( + "context" + "maps" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Design 47 H-A-208, H-A-210: a repository taken in before the walk of the module graph (H-A-201) keeps its modules' direct +// requirements only; indexer_take_in_again re-reads it at its kept merge commit and recomputes the modules state, so the +// module's kept requirements and pins gain the transitive package. An unknown repository is refused. +func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) { + const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} + ix, mods, _ := ptpIndexer(t, objects) + ix.Taken = newMem() + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + direct := map[string]string{"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n"} + walked := map[string]string{"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n"} + goMods := direct // before the rule: the client's go.mod reads as requiring nothing + ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) { + if c, ok := goMods[dir]; ok { + return []byte(c), nil + } + if dir == "modules/c/client" { + return []byte("module git.novox.be/novox/mesh-delivery/modules/c/client\n"), nil + } + return []byte(ptpGoMods[dir]), nil + } + f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, + map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}}) + var readAt []string + ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) { + readAt = append(readAt, repository+"@"+sha) + return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil + } + ctx := context.Background() + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "g"); m.Inputs2 == nil || slices.Contains(m.Inputs2.Packages, conditions) || strings.Contains(m.None2, conditions) { + t.Fatalf("before: packages = %+v (%s), want no transitive requirement yet", m.Inputs2, m.None2) + } + if r := ix.TakeInAgain(ctx, "novox/unknown"); r.Stored || !strings.HasPrefix(r.Answer, "refused:") { + t.Errorf("unknown repository: %+v, want refused", r) + } + goMods = maps.Clone(direct) + maps.Copy(goMods, walked) + r := ix.TakeInAgain(ctx, "novox/mesh-delivery") + if !r.Stored || r.Commit != merge { + t.Fatalf("take-in again = %+v, want stored at the kept merge %s", r, merge) + } + if len(readAt) == 0 || readAt[0] != "novox/mesh-delivery@"+merge { + t.Errorf("read at %v, want the kept merge commit", readAt) + } + m, _ := moduleOf(t, mods, "g") + if m.Inputs2 == nil || !slices.Contains(m.Inputs2.Packages, conditions) { + t.Errorf("after: kept requirements = %+v, want %s", m.Inputs2, conditions) + } + // The recompute pins it: nothing is published here, so the fingerprint waits on it by name (H-A-161), and the pins the + // entry names from the kept requirements hold it with no version until it is (H-A-204). + if !strings.Contains(m.None2, conditions) { + t.Errorf("after: the recompute's reason %q, want it waiting on %s", m.None2, conditions) + } + if v, named := releasedPins(nil, nil, catalogue.Manifest{}, "novox/mesh-delivery", m.Inputs2.Packages)[conditions]; !named || v != "" { + t.Errorf("after: pins name %s %v (%q), want it named, waiting", conditions, named, v) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 934e19c9..3cba48d0 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -46,6 +46,8 @@ type TakeInAsk struct { HeldSource int64 `json:"held-source,omitempty"` Skip bool `json:"skip,omitempty"` NamedMember string `json:"named-member,omitempty"` + // Again names the repository taken in again at its kept merge commit (indexer_take_in_again, take_in_again.go; H-A-208, H-A-210). + Again string `json:"again,omitempty"` // RunDrop asks one run on request, by its group run-, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194). RunDrop string `json:"run-drop,omitempty"` // Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations, diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 6a4fc4c4..81ef37ba 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -206,8 +206,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 43 || !slices.Contains(m.Tools, "modules") { + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) + if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 155327c1..bc75c22c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -67,6 +67,7 @@ "indexer_pin_drop", "indexer_pending", "indexer_take_in_pending", + "indexer_take_in_again", "indexer_take_in_skip", "indexer_runs", "indexer_run_drop"