Registry milestone: mesh-delivery (issue 1966) #151
@@ -20,22 +20,26 @@ import (
|
||||
// seatEvent is one registry seat's event the builder hears: the seat and the event's name.
|
||||
type seatEvent struct{ seat, event string }
|
||||
|
||||
// seatEvents are the four registry seats' published, refused and removed, by subject (design 51 *How a publish ends*,
|
||||
// seatEvents are the four registry seats' published, refused, removed and kept, by subject (design 51 *How a publish ends*,
|
||||
// DLV-B4): each a seat event `<seat>.<event>` on the shared events stream, heard on the builder's durable consumer of
|
||||
// it through the manifest's consumes. The subjects are the seats' contracts' (mesh-contracts).
|
||||
var seatEvents = map[string]seatEvent{
|
||||
containerimageregistry.SubjectPublished: {containerimageregistry.Seat, containerimageregistry.EventPublished},
|
||||
containerimageregistry.SubjectRefused: {containerimageregistry.Seat, containerimageregistry.EventRefused},
|
||||
containerimageregistry.SubjectRemoved: {containerimageregistry.Seat, containerimageregistry.EventRemoved},
|
||||
containerimageregistry.SubjectKept: {containerimageregistry.Seat, containerimageregistry.EventKept},
|
||||
ociartifactregistry.SubjectPublished: {ociartifactregistry.Seat, ociartifactregistry.EventPublished},
|
||||
ociartifactregistry.SubjectRefused: {ociartifactregistry.Seat, ociartifactregistry.EventRefused},
|
||||
ociartifactregistry.SubjectRemoved: {ociartifactregistry.Seat, ociartifactregistry.EventRemoved},
|
||||
ociartifactregistry.SubjectKept: {ociartifactregistry.Seat, ociartifactregistry.EventKept},
|
||||
gopackageregistry.SubjectPublished: {gopackageregistry.Seat, gopackageregistry.EventPublished},
|
||||
gopackageregistry.SubjectRefused: {gopackageregistry.Seat, gopackageregistry.EventRefused},
|
||||
gopackageregistry.SubjectRemoved: {gopackageregistry.Seat, gopackageregistry.EventRemoved},
|
||||
gopackageregistry.SubjectKept: {gopackageregistry.Seat, gopackageregistry.EventKept},
|
||||
npmpackageregistry.SubjectPublished: {npmpackageregistry.Seat, npmpackageregistry.EventPublished},
|
||||
npmpackageregistry.SubjectRefused: {npmpackageregistry.Seat, npmpackageregistry.EventRefused},
|
||||
npmpackageregistry.SubjectRemoved: {npmpackageregistry.Seat, npmpackageregistry.EventRemoved},
|
||||
npmpackageregistry.SubjectKept: {npmpackageregistry.Seat, npmpackageregistry.EventKept},
|
||||
}
|
||||
|
||||
// seatHearer takes the registry seats' outcomes in: each moves a numbered version (versions.Numberer.Heard), and a
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -310,11 +311,19 @@ func doneVia(b *build.Builder) func(ctx context.Context, module, fp string) (boo
|
||||
}
|
||||
|
||||
// removeVia calls a registry seat's remove for one tagged version, as the builder's own account (design 51 *Tagged
|
||||
// versions*): the reason tag-gone; the seat says removed on its events.
|
||||
func removeVia(call toolCaller) func(ctx context.Context, seat string, e versions.Entry) error {
|
||||
return func(_ context.Context, seat string, e versions.Entry) error {
|
||||
_, err := call("seat:"+seat+".remove", map[string]any{"name": e.Name, "version": e.Version, "tag": e.Tag, "reason": "tag-gone"})
|
||||
return err
|
||||
// versions*): the reason tag-gone; the seat says removed on its events, or answers kept when another tag names the
|
||||
// digest (the registry part's RemoveAnswer: kept, reason, tags), which ends the remove.
|
||||
func removeVia(call toolCaller) func(ctx context.Context, seat string, e versions.Entry) (versions.RemoveOutcome, error) {
|
||||
return func(_ context.Context, seat string, e versions.Entry) (versions.RemoveOutcome, error) {
|
||||
raw, err := call("seat:"+seat+".remove", map[string]any{"name": e.Name, "version": e.Version, "tag": e.Tag, "reason": "tag-gone"})
|
||||
if err != nil {
|
||||
return versions.RemoveOutcome{}, err
|
||||
}
|
||||
var a ociartifactregistry.RemoveAnswer // the four registry seats share the part's answer
|
||||
if len(raw) > 0 && json.Unmarshal(raw, &a) != nil {
|
||||
return versions.RemoveOutcome{}, fmt.Errorf("%s's remove answered what is no answer of it: %s", seat, raw)
|
||||
}
|
||||
return versions.RemoveOutcome{Kept: a.Kept, Reason: a.Reason, Tags: a.Tags}, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -175,7 +175,7 @@ func Untagged(entries []Entry, repository, tag string) []Entry {
|
||||
tag = CleanTag(tag)
|
||||
var out []Entry
|
||||
for _, e := range entries {
|
||||
if e.Tag != "" && e.Repository == repository && CleanTag(e.Tag) == tag && e.State != Removed {
|
||||
if e.Tag != "" && e.Repository == repository && CleanTag(e.Tag) == tag && e.State != Removed && e.State != Kept {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
@@ -213,8 +213,8 @@ type Acts struct {
|
||||
BuildAgain func(ctx context.Context, module, fp, why, by string) (answer string, refused bool)
|
||||
// Reconsider looks at every module again, so a claim asks a name's floor.
|
||||
Reconsider func(ctx context.Context)
|
||||
// Remove calls a seat's verb remove for one tagged version, reason tag-gone.
|
||||
Remove func(ctx context.Context, seat string, e Entry) error
|
||||
// Remove calls a seat's verb remove for one tagged version, reason tag-gone: what it answered, kept or asked.
|
||||
Remove func(ctx context.Context, seat string, e Entry) (RemoveOutcome, error)
|
||||
// DropOldForm drops a retired node from the old form's count by hand (builder_old_form): its answer, or why not.
|
||||
DropOldForm func(ctx context.Context, node, by, why string, at time.Time) (string, error)
|
||||
}
|
||||
@@ -240,8 +240,10 @@ func (a Acts) TagAsked(ctx context.Context, t TagAsk, now time.Time) TagAsk {
|
||||
t.Versions[e.Name] = e.Version
|
||||
at := e.Name + "@" + e.Version
|
||||
t.Left[at] = e.Seat
|
||||
if err := a.Remove(ctx, e.Seat, e); err != nil {
|
||||
t.Left[at] = e.Seat + ": " + err.Error()
|
||||
if left, ok := a.removeOne(ctx, e); ok {
|
||||
delete(t.Left, at)
|
||||
} else if left != "" {
|
||||
t.Left[at] = left
|
||||
}
|
||||
}
|
||||
if len(t.Left) == 0 {
|
||||
@@ -253,6 +255,22 @@ func (a Acts) TagAsked(ctx context.Context, t TagAsk, now time.Time) TagAsk {
|
||||
return t
|
||||
}
|
||||
|
||||
// removeOne asks e's seat to remove it: done when the seat kept it whole (recorded Kept, no retry); otherwise left, with
|
||||
// the error of a call that failed.
|
||||
func (a Acts) removeOne(ctx context.Context, e Entry) (left string, done bool) {
|
||||
got, err := a.Remove(ctx, e.Seat, e)
|
||||
if err != nil {
|
||||
return e.Seat + ": " + err.Error(), false
|
||||
}
|
||||
if !got.Kept {
|
||||
return "", false
|
||||
}
|
||||
if err := a.Numberer.keep(ctx, e, got.Reason, got.Tags); err != nil {
|
||||
return e.Seat + ": kept, and not recorded so: " + err.Error(), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// PublishAgain is builder_publish_again of name's fingerprint: what it did, the version, and its answer; an error refuses.
|
||||
func (a Acts) PublishAgain(ctx context.Context, name, fp, why, by string, at time.Time) (Again, string, error) {
|
||||
module, _, _ := strings.Cut(name, "/")
|
||||
|
||||
@@ -67,8 +67,11 @@ func (a Acts) BranchEnded(ctx context.Context, repository, branch string) (Branc
|
||||
if a.Remove == nil {
|
||||
continue
|
||||
}
|
||||
if err := a.Remove(ctx, e.Seat, e); err != nil {
|
||||
left[at] = e.Seat + ": " + err.Error()
|
||||
// A seat that kept it whole (another tag names its digest) ends it here: recorded kept, never asked again.
|
||||
if l, done := a.removeOne(ctx, e); done {
|
||||
delete(left, at)
|
||||
} else if l != "" {
|
||||
left[at] = l
|
||||
}
|
||||
}
|
||||
return BranchEnd{Left: left, Done: len(left) == 0}, nil
|
||||
|
||||
@@ -11,6 +11,7 @@ const (
|
||||
HeardPublished = "published"
|
||||
HeardRefused = "refused"
|
||||
HeardRemoved = "removed"
|
||||
HeardKept = "kept"
|
||||
)
|
||||
|
||||
// Outcome is the body of a registry seat's published, refused and removed (mesh-contracts parts/registry Outcome), the
|
||||
@@ -22,10 +23,32 @@ type Outcome struct {
|
||||
Kind string `json:"kind"`
|
||||
Holder string `json:"holder"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
// Tags are, on kept, the other tags still naming its digest.
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
// RemoveOutcome is what a seat's remove answered: kept whole (another tag names its digest), with why and those tags; or
|
||||
// asked, the seat saying removed on its events.
|
||||
type RemoveOutcome struct {
|
||||
Kept bool
|
||||
Reason string
|
||||
Tags []string
|
||||
}
|
||||
|
||||
// keep marks a tagged version kept by its seat, so it is asked no more.
|
||||
func (n *Numberer) keep(ctx context.Context, e Entry, reason string, tags []string) error {
|
||||
if reason == "" {
|
||||
reason = "shared-digest"
|
||||
}
|
||||
_, err := n.Versions.Mark(ctx, e.Name, e.Version, func(x *Entry) {
|
||||
x.State, x.Reason, x.KeptWith = Kept, reason, append([]string(nil), tags...)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// Heard takes one seat event in: a numbered version's entry turns published (with its digest when it held none),
|
||||
// refused with the holder's reason, or removed with its reason and the event's id; a by-digest publish (no version)
|
||||
// refused with the holder's reason, removed with its reason and the event's id, or kept (another tag names its digest)
|
||||
// with the other tags; a by-digest publish (no version)
|
||||
// marks nothing. It answers the names waiting on their floor that the event wakes: a `published` says the seat
|
||||
// answers (design 51 *The floor*).
|
||||
func (n *Numberer) Heard(ctx context.Context, seat, event, id string, o Outcome) ([]string, error) {
|
||||
@@ -53,6 +76,15 @@ func (n *Numberer) Heard(ctx context.Context, seat, event, id string, o Outcome)
|
||||
}
|
||||
case HeardRemoved:
|
||||
change = func(e *Entry) { e.State, e.Reason, e.RemovedBy = Removed, o.Reason, id }
|
||||
case HeardKept:
|
||||
change = func(e *Entry) {
|
||||
if e.State != Removed {
|
||||
e.State, e.Reason, e.KeptWith = Kept, o.Reason, append([]string(nil), o.Tags...)
|
||||
if e.Reason == "" {
|
||||
e.Reason = "shared-digest"
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
return wake, fmt.Errorf("%s says %q, which the builder does not hear", seat, event)
|
||||
}
|
||||
|
||||
@@ -106,6 +106,6 @@ func AnswerVersions(ctx context.Context, r Reader, name, version, digest string)
|
||||
// VersionsDescription is builder_versions' description.
|
||||
const VersionsDescription = "A name's versions as the builder numbered them (its state versions, design 51): every trunk and tagged " +
|
||||
"version of a package, image, bundle or archive in order, each with its kind, series, fingerprint, commit, source revision, digest, " +
|
||||
"seat, state (numbered, published, refused with its reason, removed with its time, reason and the event that removed it) and when. " +
|
||||
"seat, state (numbered, published, refused with its reason, removed with its time, reason and the event that removed it, or kept: its seat kept it whole on remove, its digest shared, with the reason and the other tags naming it, kept_with) and when. " +
|
||||
"With version, and digest, it also answers recorded: whether the builder numbered that version with that digest, and why not: the " +
|
||||
"question a registry seat's holder asks before it stores a publish (ADR 0605 §4). Read-only."
|
||||
|
||||
@@ -36,6 +36,10 @@ const (
|
||||
Published = "published"
|
||||
Refused = "refused"
|
||||
Removed = "removed"
|
||||
// Kept is a tagged version its seat kept whole on remove: another tag names its digest, so the store cannot remove the
|
||||
// tag alone (review round 2; the registry seats' `kept`). Its Reason is the seat's (shared-digest), KeptWith the
|
||||
// other tags; it is asked no more.
|
||||
Kept = "kept"
|
||||
)
|
||||
|
||||
// The kinds the builder numbers: a package per source revision, the rest per fingerprint (design 51).
|
||||
@@ -69,6 +73,8 @@ type Entry struct {
|
||||
Reason string `json:"reason,omitempty"`
|
||||
// Repository is a tagged version's repository: its branch's end removes it (Untagged).
|
||||
Repository string `json:"repository,omitempty"`
|
||||
// KeptWith are the other tags naming its digest, when its seat kept it (Kept).
|
||||
KeptWith []string `json:"kept_with,omitempty"`
|
||||
// RemovedBy is the id of the event that removed it.
|
||||
RemovedBy string `json:"removed_by,omitempty"`
|
||||
// RefusedDigest is the digest the holder refused: one other than Digest means the seat was offered other bytes under
|
||||
@@ -87,7 +93,7 @@ func (e Entry) Valid() error {
|
||||
return fmt.Errorf("%s: a version with no version", e.Name)
|
||||
}
|
||||
switch e.State {
|
||||
case Numbered, Published, Refused, Removed:
|
||||
case Numbered, Published, Refused, Removed, Kept:
|
||||
default:
|
||||
return fmt.Errorf("%s@%s: unknown state %q", e.Name, e.Version, e.State)
|
||||
}
|
||||
|
||||
@@ -727,12 +727,16 @@ func TestTheBuildersVersions(t *testing.T) {
|
||||
_, _ = st.Create(context.Background(), Key(e.Name, e.Version), raw)
|
||||
}
|
||||
var removed []string
|
||||
acts := Acts{Numberer: &Numberer{Versions: Versions{Store: st}}, Remove: func(_ context.Context, seat string, e Entry) error {
|
||||
var kept map[string]bool // review round 2: the names whose seat keeps them whole (a shared digest)
|
||||
acts := Acts{Numberer: &Numberer{Versions: Versions{Store: st}}, Remove: func(_ context.Context, seat string, e Entry) (RemoveOutcome, error) {
|
||||
removed = append(removed, e.Name+"@"+e.Version)
|
||||
if seat == SeatNPM {
|
||||
return errors.New("down")
|
||||
if kept[e.Name] {
|
||||
return RemoveOutcome{Kept: true, Reason: "shared-digest", Tags: []string{"0.1.2"}}, nil
|
||||
}
|
||||
return nil
|
||||
if seat == SeatNPM {
|
||||
return RemoveOutcome{}, errors.New("down")
|
||||
}
|
||||
return RemoveOutcome{}, nil
|
||||
}}
|
||||
u, _ := DecodeTagAsk(uraw)
|
||||
got := acts.TagAsked(context.Background(), u, at)
|
||||
@@ -756,6 +760,26 @@ func TestTheBuildersVersions(t *testing.T) {
|
||||
if end, _ := acts.BranchEnded(context.Background(), "o/r", "feat/x"); !end.Done {
|
||||
t.Errorf("every seat said removed, and the branch's end is not done: %+v", end)
|
||||
}
|
||||
// Review round 2: a remove its seat answers kept (another tag names the digest) ends there, with no retry: the
|
||||
// version is recorded kept on the store with the reason and the other tags, which builder_versions shows.
|
||||
raw, _ = Encode(Entry{Name: "n4", Kind: KindPackage, Series: "0.1", Version: "0.1.2-feat-x.1", Tag: "feat-x", Repository: "o/r", Seat: SeatNPM, State: Numbered})
|
||||
_, _ = st.Create(context.Background(), Key("n4", "0.1.2-feat-x.1"), raw)
|
||||
kept, removed = map[string]bool{"n4": true}, nil
|
||||
if end, err := acts.BranchEnded(context.Background(), "o/r", "feat/x"); err != nil || !end.Done || len(removed) != 1 {
|
||||
t.Fatalf("a kept remove: %+v %v, removed %v; want done at once", end, err, removed)
|
||||
}
|
||||
if es, _ := acts.Numberer.Versions.Of(context.Background(), "n4"); len(es) != 1 || es[0].State != Kept || es[0].Reason != "shared-digest" || !reflect.DeepEqual(es[0].KeptWith, []string{"0.1.2"}) {
|
||||
t.Errorf("the kept version = %+v, want kept, shared-digest, with 0.1.2", es)
|
||||
}
|
||||
if removed = nil; func() bool { end, _ := acts.BranchEnded(context.Background(), "o/r", "feat/x"); return !end.Done || len(removed) != 0 }() {
|
||||
t.Errorf("a kept version is asked again: removed %v", removed)
|
||||
}
|
||||
if _, err := acts.Numberer.Heard(context.Background(), SeatNPM, HeardKept, "k", Outcome{Name: "n", Version: "0.1.2-feat-x.1", Tags: []string{"0.1.2"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if es, _ := acts.Numberer.Versions.Of(context.Background(), "n"); slices.ContainsFunc(es, func(e Entry) bool { return e.Version == "0.1.2-feat-x.1" && e.State == Kept }) {
|
||||
t.Error("a seat's kept turned a removed version back to kept")
|
||||
}
|
||||
// B6: a check's build of the ask's head marks it numbered with its versions; another head, or an untag, is left.
|
||||
if !a.Unanswered() {
|
||||
t.Errorf("a kept ask with no version is answered: %+v", a)
|
||||
|
||||
@@ -130,14 +130,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
// Design 51 *The floor*: the indexer's manifest-accepted and the condition keeper's condition-cleared wake a silent floor.
|
||||
// DLV-B4: the four registry seats' outcomes, each a seat event `<seat>.<event>` on the shared events stream.
|
||||
// DLV-M1: the indexer's and the assigner's views families of the states it follows.
|
||||
if want := []string{"container-image-registry.published", "container-image-registry.refused", "container-image-registry.removed",
|
||||
"go-package-registry.published", "go-package-registry.refused", "go-package-registry.removed",
|
||||
if want := []string{"container-image-registry.kept", "container-image-registry.published", "container-image-registry.refused", "container-image-registry.removed",
|
||||
"go-package-registry.kept", "go-package-registry.published", "go-package-registry.refused", "go-package-registry.removed",
|
||||
"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported",
|
||||
"mesh-conditions.condition-cleared", "mesh-delivery-assigner.views.placement", "mesh-delivery-checker.tag-wanted", "mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-delivery-indexer.manifest-accepted", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules",
|
||||
"mesh-delivery-indexer.views.packages", "node-engine.manifest-reads-changed",
|
||||
"npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed",
|
||||
"oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
"npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed",
|
||||
"oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
|
||||
@@ -28,9 +28,11 @@
|
||||
"version.old-form-closed"
|
||||
],
|
||||
"consumes": [
|
||||
"container-image-registry.kept",
|
||||
"container-image-registry.published",
|
||||
"container-image-registry.refused",
|
||||
"container-image-registry.removed",
|
||||
"go-package-registry.kept",
|
||||
"go-package-registry.published",
|
||||
"go-package-registry.refused",
|
||||
"go-package-registry.removed",
|
||||
@@ -42,9 +44,11 @@
|
||||
"mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-delivery-indexer.manifest-accepted",
|
||||
"node-engine.manifest-reads-changed",
|
||||
"npm-package-registry.kept",
|
||||
"npm-package-registry.published",
|
||||
"npm-package-registry.refused",
|
||||
"npm-package-registry.removed",
|
||||
"oci-artifact-registry.kept",
|
||||
"oci-artifact-registry.published",
|
||||
"oci-artifact-registry.refused",
|
||||
"oci-artifact-registry.removed",
|
||||
|
||||
Reference in New Issue
Block a user