Judge each move against what the node runs, again when its have or the module's policy changes; carry the operator's keys in the node-engine's composition (issue 1945, B11, B12)
The assigner holds a move of a module of policy record from the judgement beside its digest, so it is judged by ADR 0377 §3's tests against the composition the node's have names, a held one never taking its place, and written again when the have changes or a person sets the policy (ADR 0539 clauses 2 and 14). The node-engine's own file operator-keys, from its setting operator-signers, carries the operator's public keys (clause 3).
This commit is contained in:
@@ -192,12 +192,20 @@ func resourcesByID(c sdkdelivery.Composed) map[string]*judgedResource {
|
||||
return out
|
||||
}
|
||||
|
||||
// compactComposed is a resource's bytes canonical: its keys sorted, its spacing dropped, its numbers as written, so
|
||||
// two compositions that say the same compare equal however each was written.
|
||||
func compactComposed(raw []byte) []byte {
|
||||
var b bytes.Buffer
|
||||
if json.Compact(&b, raw) != nil {
|
||||
var v any
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
d.UseNumber()
|
||||
if err := d.Decode(&v); err != nil {
|
||||
return raw
|
||||
}
|
||||
return b.Bytes()
|
||||
out, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changedOf are the ids of the resources of the kinds counted that are changed, added or removed, sorted.
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// mailOn is a composition of mesh-mail on ace at fingerprint c, from the given resources, each saying its module.
|
||||
func mailOn(t *testing.T, c byte, data bool, resources ...string) sdkdelivery.Composed {
|
||||
t.Helper()
|
||||
out := sdkdelivery.Composed{Node: "ace", Module: "mesh-mail", FP: sdkdelivery.SourcePrefix2 + strings.Repeat(string(c), 64), Data: data}
|
||||
for _, r := range resources {
|
||||
out.Resources = append(out.Resources, json.RawMessage(r))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const (
|
||||
mailContainer = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:1111111111111111111111111111111111111111111111111111111111111111","volumes":["/var/lib/mesh-mail/spool:/spool"]}`
|
||||
mailContainer2 = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:2222222222222222222222222222222222222222222222222222222222222222","volumes":["/var/lib/mesh-mail/spool:/spool"]}`
|
||||
mailSpool = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/var/lib/mesh-mail/spool"}`
|
||||
mailSpoolMoved = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/srv/mesh-mail/spool"}`
|
||||
mailTools = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"a"}`
|
||||
mailTools2 = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"b"}`
|
||||
mailConfig = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"a"}`
|
||||
mailConfig2 = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"b"}`
|
||||
)
|
||||
|
||||
// 1945/AC12: captured pairs of compositions are judged by ADR 0377 §3's tests: a changed container recreates; with no
|
||||
// container changed, a directory moved, a file within the data a container mounts changed, or the data declaration
|
||||
// changed moves data; a tools file nothing mounts changed is neither.
|
||||
func TestAMoveIsJudgedByWhatItRecreatesOrMoves(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
from, to sdkdelivery.Composed
|
||||
want sdkdelivery.Judgement
|
||||
}{
|
||||
{"a new image", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools), sdkdelivery.JudgedRecreates},
|
||||
{"a container gone", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailSpool), sdkdelivery.JudgedRecreates},
|
||||
{"its spool moved", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpoolMoved), sdkdelivery.JudgedMovesData},
|
||||
{"a file in its spool", mailOn(t, 'a', true, mailContainer, mailSpool, mailConfig), mailOn(t, 'b', true, mailContainer, mailSpool, mailConfig2), sdkdelivery.JudgedMovesData},
|
||||
{"its data declaration", mailOn(t, 'a', false, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpool), sdkdelivery.JudgedMovesData},
|
||||
{"its tools only", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools2), sdkdelivery.JudgedNeither},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got, words := judged(c.from, c.to); got != c.want || words == "" {
|
||||
t.Errorf("%s: judged %s (%q), want %s", c.name, got, words, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC12: a move is judged against the composition the node's have names: one the node never ran is neither; one
|
||||
// whose composition is gone cannot be judged and says why; and with A running, B held and C composed, C is judged
|
||||
// against A, and judged again when the have changes.
|
||||
func TestAMoveIsJudgedAgainstWhatTheNodeRuns(t *testing.T) {
|
||||
at, _, _, haves := aPlace(t, "ace")
|
||||
ctx := context.Background()
|
||||
a := mailOn(t, 'a', true, mailContainer, mailSpool, mailTools)
|
||||
b := mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools)
|
||||
c := mailOn(t, 'c', true, mailContainer, mailSpool, mailTools2)
|
||||
write := func(comp sdkdelivery.Composed) (string, string) {
|
||||
key, _ := sdkdelivery.ComposedKey("ace", comp.Module, comp.FP)
|
||||
if _, err := writeComposed(ctx, at, key, comp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, _ := sdkdelivery.ComposedDigestOf(comp)
|
||||
return key, d
|
||||
}
|
||||
judgedNow := func(key string) sdkdelivery.ComposedDigest {
|
||||
e, err := at.digests.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := sdkdelivery.DecodeComposedDigest(e.Value())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
keyA, digestA := write(a)
|
||||
if v := judgedNow(keyA); v.Judgement != sdkdelivery.JudgedNeither || v.Against != "" {
|
||||
t.Errorf("a module the node never ran: %+v", v)
|
||||
}
|
||||
if v := judgedNow(keyA); v.Policy != "roll" || v.PolicyFrom != "default" {
|
||||
t.Errorf("the policy beside the judgement: %+v", v)
|
||||
}
|
||||
haves["mesh-mail"] = sdkdelivery.Have{FP: a.FP, Composed: digestA, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||
keyB, _ := write(b)
|
||||
if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedRecreates || v.Against != digestA {
|
||||
t.Errorf("B against A: %+v", v)
|
||||
}
|
||||
// B is held (the assigner writes no want for it); the node still runs A.
|
||||
keyC, digestC := write(c)
|
||||
if v := judgedNow(keyC); v.Judgement != sdkdelivery.JudgedNeither || v.Against != digestA {
|
||||
t.Errorf("C against A, B held: %+v", v)
|
||||
}
|
||||
// The have changes to C: every judgement of the module is written again, against what runs now (the have watch).
|
||||
haves["mesh-mail"] = sdkdelivery.Have{FP: c.FP, Composed: digestC, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||
if n, err := rejudge(ctx, at, "mesh-mail"); err != nil || n == 0 {
|
||||
t.Fatalf("rejudged %d (%v)", n, err)
|
||||
}
|
||||
if v := judgedNow(keyC); v.Against != digestC || v.Judgement != sdkdelivery.JudgedNeither {
|
||||
t.Errorf("C once the node runs it: %+v", v)
|
||||
}
|
||||
if v := judgedNow(keyB); v.Against != digestC || v.Judgement != sdkdelivery.JudgedRecreates {
|
||||
t.Errorf("B once the node runs C: %+v", v)
|
||||
}
|
||||
// A person sets the module's policy: written beside every composition of it.
|
||||
at.policy = func(string) (string, string, string) { return "record", "person", "its spool" }
|
||||
if _, err := rejudge(ctx, at, "mesh-mail"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := judgedNow(keyB); v.Policy != "record" || v.PolicyFrom != "person" || v.PolicyWhy != "its spool" {
|
||||
t.Errorf("the person's policy beside B: %+v", v)
|
||||
}
|
||||
// A have naming a composition the buckets no longer hold.
|
||||
haves["mesh-mail"] = sdkdelivery.Have{FP: sdkdelivery.SourcePrefix2 + strings.Repeat("9", 64), Composed: "sha256:" + strings.Repeat("9", 64),
|
||||
Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||
write(b)
|
||||
if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedCannot || !strings.Contains(v.Words, "gone") {
|
||||
t.Errorf("against a composition that is gone: %+v", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1945/AC13: the node-engine's composition on a node carries the operator's public keys as its file operator-keys,
|
||||
// from its setting operator-signers, beside the controller's key it already trusts (ADR 0539 clause 3, decided with the
|
||||
// lead 2026-10-24: the switch's last declaration carries it because today's controller composes the same file from
|
||||
// the same manifest). The manifest is mesh-host's module.json at 20d0998c (testdata/mesh-host-module.json), never
|
||||
// typed by hand.
|
||||
func TestTheNodeEnginesCompositionCarriesTheOperatorsKeys(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/mesh-host-module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its archive names a build artifact, which only its registration turns into a reference: left out here, the rest of
|
||||
// the manifest as captured.
|
||||
var kept []map[string]any
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "archive" {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
m.Resources = kept
|
||||
const key = "MCowBQYDK2VwAyEAb3BlcmF0b3Itc2lnbmluZy1rZXktZm9yLXRlc3RzLW9ubHk="
|
||||
in := compose.Input{Catalogue: map[string]catalogue.Manifest{m.Module: m},
|
||||
Nodes: []compose.Node{{Name: "ace", Assigned: []string{m.Module}, Stated: true,
|
||||
Settings: map[string][]catalogue.Layer{m.Module: {{From: "mesh", Values: map[string]any{"operator-signers": key}}}}}}}
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("6", 64)
|
||||
all, _, err := compose.Compositions(in, "ace", map[string]string{m.Module: fp}, compose.Reading)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, _ := sdkdelivery.ComposedKey("ace", m.Module, fp)
|
||||
var file struct{ Path, Content string }
|
||||
for _, r := range all[k].Resources {
|
||||
var head struct{ ID, Path, Content string }
|
||||
_ = json.Unmarshal(r, &head)
|
||||
if strings.HasSuffix(head.ID, "operator-keys") {
|
||||
file.Path, file.Content = head.Path, head.Content
|
||||
}
|
||||
}
|
||||
if file.Path != "/var/lib/mesh-host/operator-keys" || file.Content != key+"\n" {
|
||||
t.Errorf("the node-engine's composition carries operator-keys at %q with %q, want the operator's key", file.Path, file.Content)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The judgement beside each composition is rewritten when what it is judged against or with changes (novox/hq ADR
|
||||
// 0539 clauses 2 and 14): when the node's have of the module changes (it runs another composition, or holds), and when
|
||||
// a person sets the module's policy. Heard as the have bucket's writes and the `upgrade` verb, never on a timer.
|
||||
|
||||
// rejudge writes again the judgement and policy beside every composition of module on the node at, where they differ.
|
||||
func rejudge(ctx context.Context, at composedPlace, module string) (int, error) {
|
||||
lister, err := at.bucket.ListKeys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
wrote := 0
|
||||
var errs []error
|
||||
for key := range lister.Keys() {
|
||||
node, m, _, isNode, err := sdkdelivery.ParseComposedKey(key)
|
||||
if err != nil || isNode || node != at.node || m != module {
|
||||
continue
|
||||
}
|
||||
c, revision, err := readComposed(ctx, at.bucket, key)
|
||||
if err != nil || revision == 0 {
|
||||
continue
|
||||
}
|
||||
c.Signature = nil
|
||||
digest, err := sdkdelivery.ComposedDigestOf(c)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
judged, err := judgeMove(ctx, at, key, c, digest)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
w, err := keepDigest(ctx, at.digests, key, judged)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if w {
|
||||
wrote++
|
||||
}
|
||||
}
|
||||
return wrote, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// rejudgeOn opens the node's buckets and rejudges module there, under the lease.
|
||||
func rejudgeOn(ctx context.Context, open *stores, nodes []string, module string) error {
|
||||
if _, err := theLease.epoch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
upgrades, err := open.inventory.Upgrades(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for _, node := range nodes {
|
||||
at, err := openComposedPlace(ctx, js, bus.Conn(), node)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
at.policy = func(m string) (string, string, string) {
|
||||
u, ok := upgrades[m]
|
||||
if !ok {
|
||||
return "", "", ""
|
||||
}
|
||||
return u.Policy(), u.From, u.Why
|
||||
}
|
||||
if _, err := rejudge(ctx, at, module); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// rejudgeModule rejudges a module on every node it is assigned to: what a person's policy changes.
|
||||
func rejudgeModule(ctx context.Context, open *stores, module string) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err)
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == module && len(e.On) > 0 {
|
||||
if err := rejudgeOn(ctx, open, e.On, module); err != nil {
|
||||
fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// watchHavesForJudgement rejudges a node's module each time its have changes, until ctx ends: never on the watch's own
|
||||
// goroutine.
|
||||
func watchHavesForJudgement(ctx context.Context, open *stores) {
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
bus.Close()
|
||||
return
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket)
|
||||
if err != nil {
|
||||
bus.Close()
|
||||
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
bus.Close()
|
||||
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer bus.Close()
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
node, module, isNode, err := sdkdelivery.ParseHaveKey(e.Key())
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
inBackground(func() {
|
||||
if err := rejudgeOn(ctx, open, []string{node}, module); err != nil {
|
||||
fmt.Printf("compositions: %s on %s is not judged again: %v\n", module, node, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -208,6 +208,7 @@ func watchNewModules(ctx context.Context, open *stores) {
|
||||
// composed, the replaced controller composed, and from now on each new fingerprint registered, composed and granted.
|
||||
inBackground(func() { composeAtStart(ctx, open) })
|
||||
watchDoneKeys(ctx, open)
|
||||
watchHavesForJudgement(ctx, open)
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -13,6 +13,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -42,6 +43,30 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249):
|
||||
// an announcement handled by a send whose memberships could not be issued is held and redelivered,
|
||||
// rather than taken as handled with the machines left on the old version.
|
||||
func askAgainOnGrants(err error) error {
|
||||
if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// readableList names machines the way a sentence does, because this is read by a person deciding
|
||||
// whether an upgrade went where they expected.
|
||||
func readableList(names []string) string {
|
||||
switch len(names) {
|
||||
case 0:
|
||||
return "nothing"
|
||||
case 1:
|
||||
return names[0]
|
||||
case 2:
|
||||
return names[0] + " and " + names[1]
|
||||
}
|
||||
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
||||
}
|
||||
|
||||
func shortCommit(commit string) string {
|
||||
if len(commit) > 8 {
|
||||
return commit[:8]
|
||||
@@ -49,6 +74,13 @@ func shortCommit(commit string) string {
|
||||
return commit
|
||||
}
|
||||
|
||||
func isAre(n int) string {
|
||||
if n == 1 {
|
||||
return "is"
|
||||
}
|
||||
return "are"
|
||||
}
|
||||
|
||||
// Announceable is every build this mesh recorded, in the shape the builder announces one.
|
||||
//
|
||||
// **The catalogue asks for this when it starts, and the answer is the graph's foundation**
|
||||
@@ -93,6 +125,25 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so
|
||||
// the announcement is held rather than lost; anything else is returned as it was.
|
||||
func notNow(err error) error {
|
||||
if inventory.Unreachable(err) {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// unsaved marks the failure to save the plan a merge was added to, whatever it was: the merge is carried until that save
|
||||
// writes it (inventory.AddMerge), so the announcement is held and handed over again, never taken (novox/hq ADR 0407
|
||||
// decision 3, ADR 0408, issue 1092). The bus acknowledges it only after the save succeeded.
|
||||
func unsaved(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%w: the merge is not kept until its plan is saved: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
|
||||
// SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which
|
||||
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
@@ -102,6 +153,82 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
return keepForgeMerge(ctx, m)
|
||||
}
|
||||
|
||||
// movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and
|
||||
// branch is marked as moved to the merge commit; a module the merge deleted is forgotten or said; a new module
|
||||
// is asked for and sent nowhere. The names returned are what the walk builds, bases first along the graph
|
||||
// (novox/hq 04-ISSUES/131). Nothing is built or pushed here: the walk asks its tiers. Called when a batch is
|
||||
// cut, once per repository, with the latest merge of the repository's branch and every file the batch's
|
||||
// merges of it changed.
|
||||
func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) ([]string, error) {
|
||||
from, packaging, already := mergeCandidates(m, entries, read)
|
||||
if len(from) == 0 && len(packaging) == 0 {
|
||||
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
||||
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
|
||||
if already > 0 {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+
|
||||
"from it\n", m.Owner, m.Repo, m.Base, m.Commit, already)
|
||||
return nil, nil
|
||||
}
|
||||
fmt.Print(nothingReadsIt(m.Owner+"/"+m.Repo, m))
|
||||
return nil, nil
|
||||
}
|
||||
// Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows
|
||||
// another branch is not part of this merge, and whoever is waiting for its change should read why.
|
||||
for _, e := range entries {
|
||||
if sameRepository(e.Source.Repository, m) && !sourceIs(e.Source, m) {
|
||||
fmt.Printf(" %s is built from %s/%s and follows %s, not %s; this merge leaves it out\n",
|
||||
e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base)
|
||||
}
|
||||
}
|
||||
touched, added, _ := touchedBy(from, entries, m, read)
|
||||
// **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build
|
||||
// seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with
|
||||
// every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise.
|
||||
touched, deleted := splitDeleted(touched, m)
|
||||
for _, e := range deleted {
|
||||
retireDeleted(ctx, inv, e, m)
|
||||
}
|
||||
for _, e := range touched {
|
||||
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
|
||||
return nil, notNow(err)
|
||||
}
|
||||
}
|
||||
// **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan
|
||||
// says so, and assigning it is a person's act, which needs it registered first.
|
||||
// Nothing asks its first build (novox/hq ADR 0501 §6): the builder builds every fingerprint the indexer writes, and
|
||||
// the controller registers the module from its first done key (new_modules.go).
|
||||
var built []string
|
||||
if len(from) > 0 {
|
||||
built = added
|
||||
}
|
||||
moved := append(append([]inventory.Entry{}, touched...), packaging...)
|
||||
if len(moved) == 0 {
|
||||
if len(built) > 0 {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+
|
||||
"registered from the builder's done key of each, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit,
|
||||
strings.Join(built, ", "))
|
||||
return nil, nil
|
||||
}
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
|
||||
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
|
||||
return nil, nil
|
||||
}
|
||||
// Why each is in it (issue 363): the files of its build source the merge changed, or why it is read whole.
|
||||
for _, e := range moved {
|
||||
fmt.Printf(" %s: %s\n", e.Manifest.Module, whyMoved(e, read[e.Manifest.Module], m))
|
||||
}
|
||||
for _, e := range packaging {
|
||||
fmt.Printf(" %s reads %s/%s through its build context: its own source record is left where it is\n",
|
||||
e.Manifest.Module, m.Owner, m.Repo)
|
||||
}
|
||||
var names []string
|
||||
for _, e := range moved {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
||||
var actingOnMerges sync.Mutex
|
||||
|
||||
@@ -165,6 +292,63 @@ func lookedAtCommit(read []inventory.ReadRepository, commit string) bool {
|
||||
// historyMargin is how far a packaging module's last look is taken back before a merge is history for it.
|
||||
const historyMargin = time.Minute
|
||||
|
||||
// whyMoved is why a merge moves a module, as a plan says it (novox/hq ADR 0267, issue 363): the changed
|
||||
// files in its build source, or why it is read whole. For a module built from the merged repository or one
|
||||
// whose build context is that repository; a dependent is in a plan for what it stands on.
|
||||
func whyMoved(e inventory.Entry, read []inventory.ReadRepository, m link.SourceMoved) string {
|
||||
if len(m.Paths) == 0 || m.PathsTruncated {
|
||||
return "read whole: the merge's changed files were not all said"
|
||||
}
|
||||
whole := "no build source recorded"
|
||||
for _, r := range read {
|
||||
if r.Own && r.Whole != "" {
|
||||
whole = r.Whole
|
||||
}
|
||||
}
|
||||
held := func(paths []string) []string {
|
||||
var in []string
|
||||
for _, p := range m.Paths {
|
||||
if gosource.SourceHolds(paths, p) {
|
||||
in = append(in, p)
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
changed := func(where string, in []string) string {
|
||||
return fmt.Sprintf("its build source%s changed: %d changed file(s) in it, e.g. %s", where, len(in), in[0])
|
||||
}
|
||||
if sameRepository(e.Source.Repository, m) {
|
||||
if own := ownSource(read); own != nil {
|
||||
if in := held(own); len(in) > 0 {
|
||||
return changed("", in)
|
||||
}
|
||||
}
|
||||
dir := strings.Trim(e.Source.Path, "/")
|
||||
if dir == "" {
|
||||
return "read whole: " + whole + ", so every file of its repository is its build source"
|
||||
}
|
||||
for _, p := range m.Paths {
|
||||
if inside(p, dir) {
|
||||
return "read whole: " + whole + ", so its directory is its build source; e.g. " + p
|
||||
}
|
||||
}
|
||||
return "read whole: " + whole
|
||||
}
|
||||
for _, r := range read {
|
||||
if r.Own || !sameRepository(r.Repository, m) || !followsBase(r.Ref, m.Base) {
|
||||
continue
|
||||
}
|
||||
if len(r.Paths) > 0 {
|
||||
if in := held(r.Paths); len(in) > 0 {
|
||||
return changed(" in "+m.Owner+"/"+m.Repo, in)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return "read whole: " + whole + ", so every file of " + m.Owner + "/" + m.Repo + ", which its build context is, is its build source"
|
||||
}
|
||||
return "read whole: " + whole
|
||||
}
|
||||
|
||||
// lookedOf is when a module packaging another repository was last looked at, as readForPlanning says.
|
||||
func lookedOf(read []inventory.ReadRepository) time.Time {
|
||||
var at time.Time
|
||||
@@ -176,6 +360,21 @@ func lookedOf(read []inventory.ReadRepository) time.Time {
|
||||
return at
|
||||
}
|
||||
|
||||
// wouldMove is the modules acting on this merge would move and rebuild — SourceMoved's judgement, made
|
||||
// without acting (novox/hq issue 266). Empty for a merge already acted on: acting marks each module built
|
||||
// from the repository as looked at, so the merge then reads as history for it.
|
||||
//
|
||||
// **The ones packaging source from it too** (novox/hq ADR 0267): with a module moved only by the files of
|
||||
// its build source, a merge can move a packaging module and nothing built from the repository, and a missed
|
||||
// one of those was never acted on. A packaging module's look is its newest build or plan (lookedAt), so a
|
||||
// merge acted on for it reads as history once its plan is made.
|
||||
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
||||
from, packaging, _ := mergeCandidates(m, entries, read)
|
||||
touched, _ := splitDeleted(whatTheMergeTouched(from, entries, m, read), m)
|
||||
return append(touched, packaging...)
|
||||
}
|
||||
|
||||
// splitDeleted parts the modules a merge touched into those it changed and those whose manifest it
|
||||
// removed — deleted at their source (ADR 0236).
|
||||
func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted []inventory.Entry) {
|
||||
@@ -198,6 +397,39 @@ func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted
|
||||
return kept, deleted
|
||||
}
|
||||
|
||||
// retireDeleted is what the mesh does with a module deleted at its source: its record says the merge
|
||||
// was looked at, so it is not acted on again; it is forgotten where nothing holds it; where a machine
|
||||
// runs it or the mesh holds something for it, that is said, and nothing is built.
|
||||
func retireDeleted(ctx context.Context, inv *inventory.Inventory, e inventory.Entry, m link.SourceMoved) {
|
||||
name := e.Manifest.Module
|
||||
if err := inv.SourceMoved(ctx, name, m.Commit); err != nil {
|
||||
fmt.Printf(" %s was deleted from %s/%s at %.8s, and that could not be recorded: %v\n", name, m.Owner, m.Repo, m.Commit, err)
|
||||
}
|
||||
err := inv.ForgetModule(ctx, name)
|
||||
switch {
|
||||
case err == nil:
|
||||
fmt.Printf(" %s was deleted from %s/%s at %.8s: forgotten, nothing built\n", name, m.Owner, m.Repo, m.Commit)
|
||||
case errors.Is(err, inventory.ErrStillAssigned) || errors.Is(err, inventory.ErrStillHolds):
|
||||
fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; the mesh still runs or holds it, so it is "+
|
||||
"kept until a person unassigns it and `module forget %s`: %v\n", name, m.Owner, m.Repo, m.Commit, name,
|
||||
firstLine(err.Error()))
|
||||
default:
|
||||
fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; forgetting it failed: %v\n", name, m.Owner,
|
||||
m.Repo, m.Commit, err)
|
||||
}
|
||||
}
|
||||
|
||||
// nothingReadsIt is what is said of a merge no module the mesh holds reads. **A merge into a branch other than the
|
||||
// trunk is said to start no delivery** (novox/hq ADR 0401 clause 3, issue 889): every delivery is cut from a
|
||||
// repository's trunk, and whoever merged into a feature branch should not look for a broken trigger.
|
||||
func nothingReadsIt(repository string, m link.SourceMoved) string {
|
||||
if m.Base != "" && m.Base != "main" {
|
||||
return fmt.Sprintf("%s merged into %s (%.8s), not the trunk of %s; a merge into any other branch starts no "+
|
||||
"delivery\n", repository, m.Base, m.Commit, repository)
|
||||
}
|
||||
return fmt.Sprintf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit)
|
||||
}
|
||||
|
||||
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
|
||||
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
|
||||
// An empty recorded ref is the repository's default branch, which is what a merge into the base
|
||||
@@ -299,12 +531,79 @@ func readsDirectory(e inventory.Entry, p string) bool {
|
||||
return strings.Trim(e.Source.Path, "/") == "" || inside(p, e.Source.Path)
|
||||
}
|
||||
|
||||
// unchangedOfMerge are the modules a merge changed files for under a reading without recorded build sources, but
|
||||
// whose recorded build source holds none of them, each with why (novox/hq issue 611): not built, because the build
|
||||
// the mesh holds is theirs (ADR 0267 §5). That reading — a delivery plan made while another walk built the module
|
||||
// (issue 606) — would have moved them; the walk says they matched what runs rather than leaving them unnamed.
|
||||
// A module built from the repository whose directory, or whole repository when built from its root, holds a
|
||||
// changed file, and a module packaging the repository's branch, are each read so. A merge whose changed files
|
||||
// were not all said moves every one of them: none is said here. Without acting.
|
||||
func unchangedOfMerge(m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) map[string]string {
|
||||
if len(m.Paths) == 0 || m.PathsTruncated {
|
||||
return nil
|
||||
}
|
||||
from, _, _ := mergeCandidates(m, entries, read)
|
||||
touched, _, _ := touchedBy(from, entries, m, read)
|
||||
out := map[string]string{}
|
||||
for _, e := range from {
|
||||
name := e.Manifest.Module
|
||||
if ownSource(read[name]) == nil || slices.ContainsFunc(touched, func(t inventory.Entry) bool {
|
||||
return t.Manifest.Module == name
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
var in []string
|
||||
for _, p := range m.Paths {
|
||||
if readsDirectory(e, p) {
|
||||
in = append(in, p)
|
||||
}
|
||||
}
|
||||
if len(in) > 0 {
|
||||
out[name] = fmt.Sprintf("its build source holds none of the %d changed file(s) in its directory (e.g. %s): "+
|
||||
"the build the mesh holds is its build", len(in), in[0])
|
||||
}
|
||||
}
|
||||
for _, e := range entries {
|
||||
name := e.Manifest.Module
|
||||
if sourceIs(e.Source, m) || !packagesSaid(read[name], m) || readsFrom(read[name], m) {
|
||||
continue
|
||||
}
|
||||
out[name] = fmt.Sprintf("its build source in %s/%s holds none of the %d changed file(s) (e.g. %s): "+
|
||||
"the build the mesh holds is its build", m.Owner, m.Repo, len(m.Paths), m.Paths[0])
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// packagesSaid is whether a module's newest trunk build said the build source it read in a merge's repository and
|
||||
// branch, as a context beside its own.
|
||||
func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool {
|
||||
for _, r := range read {
|
||||
if !r.Own && sameRepository(r.Repository, m) && followsBase(r.Ref, m.Base) && len(r.Paths) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's
|
||||
// check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17).
|
||||
func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) {
|
||||
return inv.ReadRepositories(ctx)
|
||||
}
|
||||
|
||||
// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: **a
|
||||
// changed file touches exactly the modules whose build reads it** (novox/hq issue 280, ADR 0238). It is
|
||||
// touchedBy's first answer; touchedBy is the one place the mesh maps a changed file onto its modules.
|
||||
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved,
|
||||
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
||||
touched, _, _ := touchedBy(candidates, known, m, read)
|
||||
return touched
|
||||
}
|
||||
|
||||
// touchedBy maps a merge's changed files onto the mesh's modules — **the one place it is done**, for the
|
||||
// merge handler, the release planner's what-if, the merge gate and a pull request's check alike (novox/hq
|
||||
// ADR 0238), so planning and gating cannot disagree about what a change touches.
|
||||
@@ -458,6 +757,49 @@ func inside(path, dir string) bool {
|
||||
return path == dir || strings.HasPrefix(path, dir+"/")
|
||||
}
|
||||
|
||||
// anyInside is whether any of these changed files is in a directory.
|
||||
func anyInside(paths []string, dir string) bool {
|
||||
for _, p := range paths {
|
||||
if inside(p, dir) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// standsOnModule is whether an entry's build stood on the named module: by what its newest build
|
||||
// recorded it was handed (`artifact-store://<module>/<artifact>@…`, the module's own artifact), or
|
||||
// — for a module registered from a manifest and not yet built — by the base its manifest names.
|
||||
func standsOnModule(e inventory.Entry, module string, against map[string][]string) bool {
|
||||
if e.Manifest.Module == module {
|
||||
return false
|
||||
}
|
||||
if e.Manifest.Build != nil {
|
||||
for _, on := range e.Manifest.Build.On {
|
||||
if on.Module == module {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
prefix := catalogue.ArtifactStoreScheme + module + "/"
|
||||
for _, ref := range against[e.Manifest.Module] {
|
||||
if strings.HasPrefix(ref, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// standsOn is whether anything in the set is built on the named module's artifacts.
|
||||
func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool {
|
||||
for _, e := range entries {
|
||||
if standsOnModule(e, module, against) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge
|
||||
// with no time on it is taken as news: refusing it would silence a forge that says less.
|
||||
func isHistory(mergedAt string, seen time.Time) bool {
|
||||
@@ -471,10 +813,52 @@ func isHistory(mergedAt string, seen time.Time) bool {
|
||||
return at.Before(seen)
|
||||
}
|
||||
|
||||
// dependentsOf is every catalogued module that stands on one of the moved modules, directly or
|
||||
// through another dependent, and is not itself among them — in the catalogue's order, so the
|
||||
// answer is the same each time. A module standing on nothing that moved is left alone: a merge
|
||||
// rebuilds what it changed and what is built on top of that, not the catalogue.
|
||||
func dependentsOf(moved, entries []inventory.Entry, against map[string][]string) []inventory.Entry {
|
||||
bases := map[string]bool{}
|
||||
for _, e := range moved {
|
||||
bases[e.Manifest.Module] = true
|
||||
}
|
||||
var out []inventory.Entry
|
||||
taken := map[string]bool{}
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for _, e := range entries {
|
||||
name := e.Manifest.Module
|
||||
if bases[name] || taken[name] {
|
||||
continue
|
||||
}
|
||||
for base := range bases {
|
||||
if standsOnModule(e, base, against) {
|
||||
taken[name] = true
|
||||
out = append(out, e)
|
||||
grew = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, e := range out {
|
||||
bases[e.Manifest.Module] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moduleManifestFile is the file that says what a module is, in its directory (the build seat's
|
||||
// ManifestName).
|
||||
const moduleManifestFile = "module.json"
|
||||
|
||||
// deletedAtSource is whether a build failed because its module's manifest is not at its source any
|
||||
// more — the build seat's own words (internal/builder) — which a merge that deleted the module causes
|
||||
// when its announcer did not say which files went (ADR 0236). Such a module is not a failure of the plan.
|
||||
func deletedAtSource(failed string) bool {
|
||||
return strings.Contains(failed, "has no "+moduleManifestFile+" at ") &&
|
||||
strings.Contains(failed, "so there is nothing saying what it is")
|
||||
}
|
||||
|
||||
// reachOfMerge is what a merge of these changed files into this branch would move, without acting: a pull request's
|
||||
// reach, as the check reckons it.
|
||||
func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
@@ -607,6 +991,8 @@ func upgradeCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
// Beside each of its compositions, for the assigner's hold (novox/hq ADR 0539 clause 14).
|
||||
rejudgeModule(ctx, open, module)
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("upgrade <module> roll-out|record|default — not %q", positionals[1])
|
||||
@@ -619,6 +1005,8 @@ func upgradeCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
// Beside each of its compositions, for the assigner's hold (novox/hq ADR 0539 clause 14).
|
||||
rejudgeModule(ctx, open, module)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -642,11 +1030,3 @@ func sayUpgrade(module string, u inventory.Upgrade) string {
|
||||
return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+
|
||||
"the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from)
|
||||
}
|
||||
|
||||
// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: touchedBy's first answer
|
||||
// (novox/hq issue 280, ADR 0238).
|
||||
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved,
|
||||
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
||||
touched, _, _ := touchedBy(candidates, known, m, read)
|
||||
return touched
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user