Gate a release plan's first machine and roll a failed build back there (hq ADR 0236)
A build that reported applied was sent everywhere; one that then did nothing, served no tools or broke its machine's word reached every machine. Now the first machine is judged by the component's health (the core's definitions, as doctor probes H-*, or a module's own) three times over two minutes within ten; a failing gate puts the previous build back there once, marks the build, and says it as a condition and an event. Upgrades roll out by default; the bus is a planned step; a module deleted at its source is not built (the public-acme plan failure).
This commit is contained in:
jochen
@@ -19,6 +19,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// A Kind is what a principal is, which decides the shape of its authority rather than its
|
||||
@@ -115,6 +117,12 @@ type Principal struct {
|
||||
// else — not its declarations, which the node's tool runtime serves for it.
|
||||
SnapshotsTheBus bool
|
||||
|
||||
// WitnessesController is a machine principal whose node-engine witnesses the controller's upgrades:
|
||||
// the machine runs the controller (novox/hq ADR 0236, lease/witness.go). It may read the lease's one
|
||||
// key, and nothing else of the bucket, so it can judge a new controller build and put the previous
|
||||
// one back.
|
||||
WitnessesController bool
|
||||
|
||||
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
|
||||
// and never appears here: this file is written to a node's disk and read by a server, and a
|
||||
// secret that can be read from a configuration file is a secret with a wider blast radius
|
||||
@@ -432,6 +440,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// `report` verb healer H1 asks): its own machine's, on core NATS and off any stream. It
|
||||
// answers through its report, the one thing it already says — no reply to anybody's inbox.
|
||||
AskReportSubject(p.Node)}
|
||||
// The node-engine witnesses the core builds it places (novox/hq to-be 45 §8, ADR 0236; the
|
||||
// contract is lease/witness.go): it asks its own machine's node tools PING, and, where the
|
||||
// machine runs the controller, reads the lease's one key — read, never written.
|
||||
pub = append(pub, WitnessSubjects(p)...)
|
||||
|
||||
case KindModule:
|
||||
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
|
||||
@@ -979,3 +991,15 @@ func announcing(names ...string) []string {
|
||||
func discovering() []string {
|
||||
return []string{"$SRV.PING", "$SRV.PING.>", "$SRV.INFO", "$SRV.INFO.>"}
|
||||
}
|
||||
|
||||
// WitnessSubjects are the subjects a machine's node-engine publishes to witness the core builds it
|
||||
// places (novox/hq ADR 0236, lease/witness.go): its own node tools' PING, and the lease's key where it
|
||||
// runs the controller. Reads only: a write to the bucket is `$KV.<bucket>.>`, the controller's alone
|
||||
// (the writers table). The answers come to the machine's own inbox.
|
||||
func WitnessSubjects(p Principal) []string {
|
||||
out := []string{lease.PingSubject(p.Node)}
|
||||
if p.WitnessesController {
|
||||
out = append(out, lease.LeaseReadSubject(LeaseBucket))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -28,6 +28,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
||||
states = append(states, link.KeySecretReplaced)
|
||||
// And every act a healer takes (novox/hq to-be 45 §7).
|
||||
states = append(states, link.KeyHealerActed)
|
||||
// And a build put back after its gate failed (novox/hq ADR 0235).
|
||||
states = append(states, link.KeyRolledBack)
|
||||
for _, event := range states {
|
||||
if !slices.Contains(broker.ControllerStates, event) {
|
||||
t.Errorf("the mesh states %q and its account may not publish it", event)
|
||||
|
||||
@@ -213,7 +213,9 @@ var ControllerStates = []string{"applied", "refused", "built-before",
|
||||
"secret-replaced",
|
||||
// And every act a healer takes on a condition (novox/hq to-be 45 §7, Phase 3): a repair the mesh
|
||||
// made by itself is said like one a person made, never quietly.
|
||||
"healer-acted"}
|
||||
"healer-acted",
|
||||
// And a build put back after its gate failed on its first machine (novox/hq ADR 0235, to-be 45 §8).
|
||||
"rolled-back"}
|
||||
|
||||
// BusAdvisories are what the bus server says about the mesh's own account that the controller
|
||||
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
|
||||
|
||||
+2
-2
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
@@ -33,7 +33,7 @@ accounts {
|
||||
subscribe: { allow: ["_INBOX.enrol.one.>"] }
|
||||
} }
|
||||
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "mesh.control.one.>"] }
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.report", "mesh.node.one.declare"] }
|
||||
} }
|
||||
{ user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: {
|
||||
|
||||
@@ -72,7 +72,13 @@ func Users(r Records) ([]Principal, error) {
|
||||
out := []Principal{{Kind: KindController}}
|
||||
|
||||
for _, node := range sortedCopy(r.Nodes) {
|
||||
out = append(out, Principal{Kind: KindNode, Node: node})
|
||||
witness := false
|
||||
for _, d := range r.Assigned[node] {
|
||||
if d.Module == controllerModule {
|
||||
witness = true
|
||||
}
|
||||
}
|
||||
out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness})
|
||||
// **Where the runtime is assigned, the machine gets one runtime principal in place of the
|
||||
// runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the
|
||||
// node: its serving grants are the union of theirs. Every other module keeps its own
|
||||
@@ -171,3 +177,6 @@ func sortedNames(in map[string][]string) []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// controllerModule is the controller's module: the machine assigned it witnesses its upgrades.
|
||||
const controllerModule = "mesh-controller"
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Every machine's node-engine may ask its own node tools PING; the one running the controller may read
|
||||
// the lease's key, and nothing else of the bucket (novox/hq ADR 0236).
|
||||
func TestTheWitnessIsGrantedWhatItReadsAndNoMore(t *testing.T) {
|
||||
users, err := Users(Records{Nodes: []string{"control", "edge"},
|
||||
Assigned: map[string][]Declared{"control": {{Module: "mesh-controller"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease := "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder"
|
||||
for _, u := range users {
|
||||
if u.Kind != KindNode {
|
||||
continue
|
||||
}
|
||||
perms, err := PermissionsFor(u)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(perms.Publish, "$SRV.PING.node-tools."+u.Node) {
|
||||
t.Errorf("%s may not ask its node tools: %v", u.Node, perms.Publish)
|
||||
}
|
||||
if got := slices.Contains(perms.Publish, lease); got != (u.Node == "control") {
|
||||
t.Errorf("%s may read the lease: %v", u.Node, got)
|
||||
}
|
||||
for _, p := range perms.Publish {
|
||||
if p == "$KV.mesh-controller_lease.>" || p == "$KV.mesh-controller_lease.holder" {
|
||||
t.Errorf("%s may write the lease", u.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -453,6 +453,11 @@ type Manifest struct {
|
||||
// unassignment retires and what the self-check measures are all derived from it.
|
||||
Data *Data `json:"data,omitempty"`
|
||||
|
||||
// Upgrade is how this module's new builds reach its machines (novox/hq ADR 0235): rolled out one
|
||||
// machine first and gated when unsaid; `together`, or `record` — wait for a person's push — with
|
||||
// why. A person's choice through the `upgrade` verb stands over it; the bus records whatever it says.
|
||||
Upgrade *UpgradePolicy `json:"upgrade,omitempty"`
|
||||
|
||||
// Reads are other modules' state this module reads and watches, each `<module>.<name>`
|
||||
// (novox/hq ADR 0201). Read-only: only the owner's instances write.
|
||||
Reads []string `json:"reads,omitempty"`
|
||||
@@ -2018,6 +2023,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
problems = append(problems, m.contributionPlaceholderProblems()...)
|
||||
problems = append(problems, m.seatContributionProblems()...)
|
||||
problems = append(problems, m.dataProblems()...)
|
||||
problems = append(problems, m.upgradeProblems()...)
|
||||
|
||||
for i, r := range m.Resources {
|
||||
id, _ := r["id"].(string)
|
||||
|
||||
@@ -89,7 +89,9 @@ var defaultSeats = append([]Seat{
|
||||
// A value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
"secret-replaced",
|
||||
// Every act a healer takes (novox/hq to-be 45 §7).
|
||||
"healer-acted"},
|
||||
"healer-acted",
|
||||
// A build put back after its gate failed (novox/hq ADR 0235, to-be 45 §8).
|
||||
"rolled-back"},
|
||||
Serves: ControllerVerbs},
|
||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||
// served by whichever module holds the seat with tools of these names.
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What the mesh does when a module's build moves (novox/hq ADR 0235, extending ADR 0162 §3 and ADR
|
||||
// 0218 §2).
|
||||
//
|
||||
// **Rolled out by default, one machine first and gated.** With the gate on the first machine and the
|
||||
// rollback after it (to-be 45 §8), a build that moves is sent to one machine, judged there by its own
|
||||
// health, and only then to the rest — or put back there, said, and sent nowhere else. Recording an
|
||||
// upgrade and waiting for a person to push it is kept where a module says why, and where the mesh knows
|
||||
// a rollback cannot undo what a new build does.
|
||||
|
||||
// The policies a module may declare.
|
||||
const (
|
||||
// PolicyRoll sends one machine first, judges it at the gate, then the rest.
|
||||
PolicyRoll = "roll"
|
||||
// PolicyTogether sends every machine running the module at once — for a module that must change
|
||||
// everywhere in the same minute. Still judged, on every machine, after.
|
||||
PolicyTogether = "together"
|
||||
// PolicyRecord builds and sends nothing: the machines running it are behind until a person pushes.
|
||||
PolicyRecord = "record"
|
||||
)
|
||||
|
||||
// UpgradePolicy is what a module says about how its new builds reach its machines: `upgrade` in its
|
||||
// manifest.
|
||||
type UpgradePolicy struct {
|
||||
Policy string `json:"policy"`
|
||||
// Why is required for anything but roll: a person reading the catalogue sees why this module waits
|
||||
// for them, or why it changes everywhere at once.
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
func (m Manifest) upgradeProblems() []string {
|
||||
if m.Upgrade == nil {
|
||||
return nil
|
||||
}
|
||||
switch m.Upgrade.Policy {
|
||||
case PolicyRoll:
|
||||
case PolicyTogether, PolicyRecord:
|
||||
if strings.TrimSpace(m.Upgrade.Why) == "" {
|
||||
return []string{fmt.Sprintf("upgrade %q says why: a module that does not roll out one machine first "+
|
||||
"names the reason a person reads", m.Upgrade.Policy)}
|
||||
}
|
||||
default:
|
||||
return []string{fmt.Sprintf("upgrade is %q, %q or %q, not %q", PolicyRoll, PolicyTogether, PolicyRecord,
|
||||
m.Upgrade.Policy)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Where a policy came from, as `upgrade` says it.
|
||||
const (
|
||||
FromPerson = "person"
|
||||
FromModule = "module"
|
||||
FromBus = "the bus"
|
||||
FromData = "irreplaceable data"
|
||||
FromDefault = "default"
|
||||
)
|
||||
|
||||
// DerivedUpgrade is the policy a module's manifest gives it when no person has chosen one, with where
|
||||
// it came from and why (ADR 0235):
|
||||
//
|
||||
// - **the bus is never rolled**: a module that provides the mesh's bus records, whatever it says — its
|
||||
// upgrade is a planned step a person starts (to-be 45 §8);
|
||||
// - a module that says its policy has it;
|
||||
// - a module that keeps irreplaceable data records — sending the previous build cannot undo what a new
|
||||
// one did to data that cannot be had again, so a person takes it, after a backup;
|
||||
// - everything else rolls out, one machine first.
|
||||
func DerivedUpgrade(m Manifest) (policy, from, why string) {
|
||||
if ProvidesBus(m) {
|
||||
return PolicyRecord, FromBus, "the bus is replaced only as a planned step a person starts (`bus upgrade`): " +
|
||||
"its streams are snapshotted first and checked after"
|
||||
}
|
||||
if m.Upgrade != nil && m.Upgrade.Policy != "" {
|
||||
return m.Upgrade.Policy, FromModule, m.Upgrade.Why
|
||||
}
|
||||
if item := m.irreplaceable(); item != "" {
|
||||
return PolicyRecord, FromData, "it keeps irreplaceable data (" + item + "): a rollback cannot undo what a new " +
|
||||
"build does to it, so a person takes each build, after a backup"
|
||||
}
|
||||
return PolicyRoll, FromDefault, ""
|
||||
}
|
||||
|
||||
// ProvidesBus is whether a manifest provides the mesh's bus.
|
||||
func ProvidesBus(m Manifest) bool {
|
||||
for _, o := range m.Provides {
|
||||
if o.Name == "mesh-bus" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// irreplaceable names the first irreplaceable data the module keeps, its own or its consumers', or
|
||||
// nothing.
|
||||
func (m Manifest) irreplaceable() string {
|
||||
if m.Data == nil {
|
||||
return ""
|
||||
}
|
||||
for _, it := range m.Data.Own {
|
||||
if it.Class == ClassIrreplaceable {
|
||||
return it.ID
|
||||
}
|
||||
}
|
||||
for provision, c := range m.Data.Consumers {
|
||||
if c.Class == ClassIrreplaceable {
|
||||
return "its consumers' " + provision
|
||||
}
|
||||
}
|
||||
for provision, k := range m.Data.KeptBy {
|
||||
if k.Class == ClassIrreplaceable {
|
||||
return "what it keeps with " + provision
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module rolls out by default; it records where it says so with why, where it keeps irreplaceable
|
||||
// data, and always where it is the bus (novox/hq ADR 0236).
|
||||
func TestWhereAModulesUpgradePolicyComesFrom(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
m Manifest
|
||||
policy, from string
|
||||
}{
|
||||
{"default", Manifest{Module: "app"}, PolicyRoll, FromDefault},
|
||||
{"says record", Manifest{Module: "db", Upgrade: &UpgradePolicy{Policy: PolicyRecord, Why: "a restart costs"}},
|
||||
PolicyRecord, FromModule},
|
||||
{"says together", Manifest{Module: "dns", Upgrade: &UpgradePolicy{Policy: PolicyTogether, Why: "one zone"}},
|
||||
PolicyTogether, FromModule},
|
||||
{"irreplaceable data", Manifest{Module: "media", Data: &Data{Own: []DataItem{{ID: "library", Class: ClassIrreplaceable}}}},
|
||||
PolicyRecord, FromData},
|
||||
{"valuable data rolls", Manifest{Module: "notes", Data: &Data{Own: []DataItem{{ID: "db", Class: ClassValuable}}}},
|
||||
PolicyRoll, FromDefault},
|
||||
{"irreplaceable but says roll", Manifest{Module: "photos", Upgrade: &UpgradePolicy{Policy: PolicyRoll},
|
||||
Data: &Data{Own: []DataItem{{ID: "originals", Class: ClassIrreplaceable}}}}, PolicyRoll, FromModule},
|
||||
{"the bus, whatever it says", Manifest{Module: "nats", Provides: []Offer{{Name: "mesh-bus"}},
|
||||
Upgrade: &UpgradePolicy{Policy: PolicyRoll}}, PolicyRecord, FromBus},
|
||||
}
|
||||
for _, c := range cases {
|
||||
policy, from, _ := DerivedUpgrade(c.m)
|
||||
if policy != c.policy || from != c.from {
|
||||
t.Errorf("%s: %s from %s, want %s from %s", c.name, policy, from, c.policy, c.from)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A policy other than roll says why, and an unknown one is refused.
|
||||
func TestAnUpgradePolicySaysWhy(t *testing.T) {
|
||||
for _, u := range []UpgradePolicy{{Policy: PolicyRecord}, {Policy: PolicyTogether}, {Policy: "sometimes", Why: "x"}} {
|
||||
if problems := (Manifest{Module: "m", Upgrade: &u}).upgradeProblems(); len(problems) == 0 {
|
||||
t.Errorf("%+v was accepted", u)
|
||||
}
|
||||
}
|
||||
if problems := (Manifest{Module: "m", Upgrade: &UpgradePolicy{Policy: PolicyRecord, Why: "a restart costs"}}).upgradeProblems(); len(problems) != 0 {
|
||||
t.Errorf("a record with why was refused: %v", problems)
|
||||
}
|
||||
if _, err := ParseManifest([]byte(`{"module":"m","upgrade":{"policy":"record"}}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "says why") {
|
||||
t.Errorf("a manifest recording without why parsed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -267,6 +267,31 @@ var ControllerVerbs = []Verb{
|
||||
"probes": "\"true\": the registry — what each probe asserts, and the condition it raises",
|
||||
"signals": "\"true\": the signals table, each row with the age of its newest signal",
|
||||
}, nil, "run", "probes", "signals")},
|
||||
// How a module's new builds reach its machines, and the bus's planned step (novox/hq ADR 0235).
|
||||
{Name: "upgrade", Description: "How each module's new builds reach its machines (novox/hq ADR 0235): rolled " +
|
||||
"out one machine first and judged there at the gate, then the rest — or recorded, waiting for a person's " +
|
||||
"push — with where that comes from (a person, the module, the bus, its irreplaceable data, the default) and " +
|
||||
"why. With module, that one; with policy, a person's choice for it — roll-out, record (with why) or default " +
|
||||
"to take the choice back. The bus is never rolled out.",
|
||||
Input: schema(map[string]string{
|
||||
"module": "one module",
|
||||
"policy": "with module: roll-out, record or default",
|
||||
"together": "\"true\": with roll-out, every machine at once instead of one machine first",
|
||||
"why": "with policy: why — required for record, kept and said with the policy",
|
||||
}, nil, "together")},
|
||||
{Name: "bus", Description: "The bus as a planned step (novox/hq to-be 45 §8, ADR 0235): what a bus upgrade " +
|
||||
"would do — the bus's build on each machine against the one the mesh holds — and how the last step went. " +
|
||||
"With upgrade, start one: a person's act with why, after the streams are snapshotted (snapshot-taken says " +
|
||||
"where, while the mesh takes none itself), saying first whether it can be reverted; bus-maintenance is open " +
|
||||
"while it runs and every stream, consumer and a round trip are checked after.",
|
||||
Input: schema(map[string]string{
|
||||
"upgrade": "\"true\": start the bus's upgrade",
|
||||
"why": "with upgrade: why — required, recorded in the hand-act log",
|
||||
"cause": "with upgrade: the cause in a word (default bus-upgrade)",
|
||||
"reversible": "\"true\": with upgrade, the new version can be undone by putting the old one back",
|
||||
"irreversible": "\"true\": with upgrade, it cannot — your explicit word that it runs anyway",
|
||||
"snapshot-taken": "with upgrade: where the streams' snapshot you took is",
|
||||
}, nil, "upgrade", "reversible", "irreversible")},
|
||||
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
|
||||
// (novox/hq ADR 0230).
|
||||
{Name: "retire", Description: "A consumer the mesh stops asking for is retired by its provider — access " +
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// BusStep is one planned bus upgrade (novox/hq to-be 45 §8, ADR 0235).
|
||||
type BusStep struct {
|
||||
ID int64
|
||||
Module string
|
||||
Machines []string
|
||||
From, To string
|
||||
Snapshot string
|
||||
Reversible bool
|
||||
By, Why string
|
||||
Started time.Time
|
||||
Ended *time.Time
|
||||
Outcome string
|
||||
Found string
|
||||
}
|
||||
|
||||
// StartBusStep records a bus upgrade starting. Refused while another runs.
|
||||
func (i *Inventory) StartBusStep(ctx context.Context, s BusStep) (BusStep, error) {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return s, err
|
||||
}
|
||||
if open, found, err := i.LatestBusStep(ctx); err != nil {
|
||||
return s, err
|
||||
} else if found && open.Ended == nil {
|
||||
return s, ErrBusStepRunning
|
||||
}
|
||||
if s.Machines == nil {
|
||||
s.Machines = []string{}
|
||||
}
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`insert into bus_step (module, machines, from_build, to_build, snapshot, reversible, by_whom, why)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8) returning id, started`,
|
||||
s.Module, s.Machines, s.From, s.To, s.Snapshot, s.Reversible, s.By, s.Why).Scan(&s.ID, &s.Started)
|
||||
return s, err
|
||||
}
|
||||
|
||||
// ErrBusStepRunning is a bus upgrade asked while one runs.
|
||||
var ErrBusStepRunning = errors.New("a bus upgrade is already running")
|
||||
|
||||
// EndBusStep records how a bus upgrade ended: done or failed, with what was found.
|
||||
func (i *Inventory) EndBusStep(ctx context.Context, id int64, outcome, found string) error {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update bus_step set ended = now(), outcome = $2, found = $3 where id = $1 and ended is null`, id, outcome, found)
|
||||
return err
|
||||
}
|
||||
|
||||
// LatestBusStep is the newest bus upgrade, and whether there is any.
|
||||
func (i *Inventory) LatestBusStep(ctx context.Context) (BusStep, bool, error) {
|
||||
var s BusStep
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, module, machines, from_build, to_build, snapshot, reversible, by_whom, why, started, ended,
|
||||
outcome, found
|
||||
from bus_step order by id desc limit 1`).
|
||||
Scan(&s.ID, &s.Module, &s.Machines, &s.From, &s.To, &s.Snapshot, &s.Reversible, &s.By, &s.Why, &s.Started,
|
||||
&s.Ended, &s.Outcome, &s.Found)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return s, false, nil
|
||||
}
|
||||
return s, err == nil, err
|
||||
}
|
||||
+120
-19
@@ -1194,15 +1194,62 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
|
||||
// providedBy is what the source column says for a module the control plane ships.
|
||||
const providedBy = "the control plane"
|
||||
|
||||
// Upgrade is what the mesh decided to do when a module's current version moves.
|
||||
// Upgrade is what the mesh does when a module's current version moves (ADR 0162 §3, ADR 0235).
|
||||
type Upgrade struct {
|
||||
// RollOut is true when the machines running it should be sent the new version. False means
|
||||
// record it and stop — which needs no record of its own, because a machine not running what
|
||||
// the mesh would send it is already something the mesh reports.
|
||||
// RollOut is true when the machines running it are sent the new version: one machine first, judged
|
||||
// at the gate, then the rest (ADR 0218, ADR 0235). False means record it and stop — the machines
|
||||
// running it are behind until a person pushes, which the mesh already reports.
|
||||
RollOut bool
|
||||
// Together is true when every machine running it is sent the new version at once, rather than
|
||||
// one after another. Only meaningful when RollOut is.
|
||||
// Together is true when every machine running it is sent the new version at once. Only meaningful
|
||||
// when RollOut is.
|
||||
Together bool
|
||||
// From is where the policy came from: a person, the module, the bus, its irreplaceable data, or the
|
||||
// default (catalogue.From*); Why is the reason said with it.
|
||||
From string
|
||||
Why string
|
||||
// By is the person who chose it, when one did.
|
||||
By string
|
||||
}
|
||||
|
||||
// Policy is the policy as a word: roll, together or record.
|
||||
func (u Upgrade) Policy() string {
|
||||
switch {
|
||||
case !u.RollOut:
|
||||
return catalogue.PolicyRecord
|
||||
case u.Together:
|
||||
return catalogue.PolicyTogether
|
||||
}
|
||||
return catalogue.PolicyRoll
|
||||
}
|
||||
|
||||
// upgradeFrom is a module's policy from what the store holds of it: a person's choice, over the module's
|
||||
// own word, over the default — except that the bus is never rolled out, whoever says so (ADR 0235).
|
||||
func upgradeFrom(chosen *string, together bool, why, by string, manifest []byte) Upgrade {
|
||||
var m catalogue.Manifest
|
||||
// Leniently: a policy is read from what was registered, and a manifest registered before a field it
|
||||
// carries was known is still a manifest whose bus and data can be read.
|
||||
_ = json.Unmarshal(manifest, &m)
|
||||
policy, from, said := catalogue.DerivedUpgrade(m)
|
||||
if from != catalogue.FromBus && chosen != nil {
|
||||
policy, from, said = catalogue.PolicyRecord, catalogue.FromPerson, why
|
||||
if *chosen == "roll-out" {
|
||||
policy = catalogue.PolicyRoll
|
||||
if together {
|
||||
policy = catalogue.PolicyTogether
|
||||
}
|
||||
}
|
||||
}
|
||||
u := Upgrade{From: from, Why: said}
|
||||
if from == catalogue.FromPerson {
|
||||
u.By = by
|
||||
}
|
||||
switch policy {
|
||||
case catalogue.PolicyRoll:
|
||||
u.RollOut = true
|
||||
case catalogue.PolicyTogether:
|
||||
u.RollOut, u.Together = true, true
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// UpgradeOf is what to do when this module moves.
|
||||
@@ -1211,30 +1258,80 @@ type Upgrade struct {
|
||||
// mesh has never registered, and being told one of them moved is information, not a fault. The
|
||||
// answer is the safe one — record it — because there is nothing to roll out to.
|
||||
func (i *Inventory) UpgradeOf(ctx context.Context, module string) (Upgrade, error) {
|
||||
var u Upgrade
|
||||
var policy string
|
||||
var chosen *string
|
||||
var together bool
|
||||
var why, by string
|
||||
var manifest []byte
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select upgrade, upgrade_together from module where name = $1`, module).
|
||||
Scan(&policy, &u.Together)
|
||||
`select upgrade, upgrade_together, upgrade_why, upgrade_by, manifest from module where name = $1`, module).
|
||||
Scan(&chosen, &together, &why, &by, &manifest)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Upgrade{}, nil
|
||||
return Upgrade{From: catalogue.FromDefault, Why: "the mesh holds no such module"}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Upgrade{}, err
|
||||
}
|
||||
u.RollOut = policy == "roll-out"
|
||||
return u, nil
|
||||
return upgradeFrom(chosen, together, why, by, manifest), nil
|
||||
}
|
||||
|
||||
// SetUpgradeOf records what to do when this module moves.
|
||||
// Upgrades is every module's policy, by name: what `upgrade` lists.
|
||||
func (i *Inventory) Upgrades(ctx context.Context) (map[string]Upgrade, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select name, upgrade, upgrade_together, upgrade_why, upgrade_by, manifest from module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]Upgrade{}
|
||||
for rows.Next() {
|
||||
var name, why, by string
|
||||
var chosen *string
|
||||
var together bool
|
||||
var manifest []byte
|
||||
if err := rows.Scan(&name, &chosen, &together, &why, &by, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = upgradeFrom(chosen, together, why, by, manifest)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ErrBusIsPlanned is a person asking the bus to be rolled out: its upgrade is a planned step (ADR 0235).
|
||||
var ErrBusIsPlanned = errors.New("the bus is never rolled out: its upgrade is a planned step a person starts " +
|
||||
"with `bus upgrade`, which snapshots its streams first and checks them after")
|
||||
|
||||
// SetUpgradeOf records a person's choice of what to do when this module moves, with why and who. A
|
||||
// record says why; the bus is refused a roll-out.
|
||||
func (i *Inventory) SetUpgradeOf(ctx context.Context, module string, u Upgrade) error {
|
||||
policy := "record"
|
||||
if u.RollOut {
|
||||
policy = "roll-out"
|
||||
var manifest []byte
|
||||
err := i.store.Pool().QueryRow(ctx, `select manifest from module where name = $1`, module).Scan(&manifest)
|
||||
if err == nil {
|
||||
var m catalogue.Manifest
|
||||
if json.Unmarshal(manifest, &m) == nil && catalogue.ProvidesBus(m) {
|
||||
return fmt.Errorf("%s provides the mesh's bus: %w", module, ErrBusIsPlanned)
|
||||
}
|
||||
}
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update module set upgrade = $2, upgrade_together = $3 where name = $1`,
|
||||
module, policy, u.Together)
|
||||
`update module set upgrade = $2, upgrade_together = $3, upgrade_why = $4, upgrade_by = $5 where name = $1`,
|
||||
module, policy, u.Together, u.Why, u.By)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("this mesh holds no module called %s", module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearUpgradeOf takes a person's choice back: the module's own word, or the default, decides again.
|
||||
func (i *Inventory) ClearUpgradeOf(ctx context.Context, module string) error {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update module set upgrade = null, upgrade_together = false, upgrade_why = '', upgrade_by = '' where name = $1`,
|
||||
module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1258,18 +1355,22 @@ type CurrentBuild struct {
|
||||
// it carried, and what a push compares a machine's last send against.
|
||||
func (i *Inventory) CurrentBuilds(ctx context.Context) (map[string]CurrentBuild, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select name, coalesce(built_from, ''), upgrade = 'roll-out' from module`)
|
||||
`select name, coalesce(built_from, ''), upgrade, upgrade_together, upgrade_why, upgrade_by, manifest from module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]CurrentBuild{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var name, why, by string
|
||||
var chosen *string
|
||||
var together bool
|
||||
var manifest []byte
|
||||
var b CurrentBuild
|
||||
if err := rows.Scan(&name, &b.Commit, &b.RollOut); err != nil {
|
||||
if err := rows.Scan(&name, &b.Commit, &chosen, &together, &why, &by, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.RollOut = upgradeFrom(chosen, together, why, by, manifest).RollOut
|
||||
out[name] = b
|
||||
}
|
||||
return out, rows.Err()
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The gate's verdicts (novox/hq ADR 0235, to-be 45 §8): what a build did on its first machine, and,
|
||||
// for one that failed there, how it was put back. One row per build, written by the plan that rolled it
|
||||
// out, under the lease.
|
||||
|
||||
// The gate's verdicts and a failed build's rollback.
|
||||
const (
|
||||
GatePassed = "passed"
|
||||
GateFailed = "failed"
|
||||
|
||||
RollingBack = "rolling-back"
|
||||
RolledBack = "rolled-back"
|
||||
NotRolledBack = "not-rolled-back"
|
||||
)
|
||||
|
||||
// GateVerdict is one build's verdict at its gate.
|
||||
type GateVerdict struct {
|
||||
Build string
|
||||
Module string
|
||||
Commit string
|
||||
Previous string
|
||||
Plan string
|
||||
Machines []string
|
||||
Verdict string
|
||||
Rollback string
|
||||
Why string
|
||||
Component string
|
||||
// JudgingFrom is when the first machine reported the build applied and the judging began.
|
||||
JudgingFrom *time.Time
|
||||
JudgedAt time.Time
|
||||
Epoch uint64
|
||||
}
|
||||
|
||||
// RecordGate writes a build's verdict. **A failed build's row is written once**: a second failure for
|
||||
// the same build is refused with ErrGateKept, which is what keeps a rollback to one per build — the row
|
||||
// is written before the rollback's send, and a controller replaced in between finds it.
|
||||
func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error {
|
||||
epoch, err := i.actingEpoch(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the gate's verdict on %s is not written: %w", v.Build, err)
|
||||
}
|
||||
if v.Machines == nil {
|
||||
v.Machines = []string{}
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`insert into build_gate (build, module, commit_hash, previous, plan, machines, verdict, rollback, why,
|
||||
component, judging_from, judged_at, epoch)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12)
|
||||
on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback,
|
||||
why = excluded.why, previous = excluded.previous, machines = excluded.machines,
|
||||
judged_at = now(), epoch = excluded.epoch
|
||||
where build_gate.verdict = 'passed' and excluded.verdict = 'passed'`,
|
||||
v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component,
|
||||
v.JudgingFrom, epoch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: %s", ErrGateKept, v.Build)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrGateKept is a verdict already kept for the build, which is not written over.
|
||||
var ErrGateKept = errors.New("this build's verdict at its gate is already kept")
|
||||
|
||||
// SetRollback records how a failed build's rollback went.
|
||||
func (i *Inventory) SetRollback(ctx context.Context, build, rollback, why string) error {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update build_gate set rollback = $2, why = $3, judged_at = now() where build = $1 and verdict = 'failed'`,
|
||||
build, rollback, why)
|
||||
return err
|
||||
}
|
||||
|
||||
// GateOf is a build's verdict, and whether it has one.
|
||||
func (i *Inventory) GateOf(ctx context.Context, build string) (GateVerdict, bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, gateSelect+` where build = $1`, build)
|
||||
if err != nil {
|
||||
return GateVerdict{}, false, err
|
||||
}
|
||||
list, err := scanGates(rows)
|
||||
if err != nil || len(list) == 0 {
|
||||
return GateVerdict{}, false, err
|
||||
}
|
||||
return list[0], true, nil
|
||||
}
|
||||
|
||||
// GateFailed is whether a build failed its gate: one the mesh never registers or sends again on its own.
|
||||
func (i *Inventory) GateFailed(ctx context.Context, build string) (bool, error) {
|
||||
v, found, err := i.GateOf(ctx, build)
|
||||
return found && v.Verdict == GateFailed, err
|
||||
}
|
||||
|
||||
// LatestGates is the newest verdict of every module that has one: what the gate probe (DG) reads.
|
||||
func (i *Inventory) LatestGates(ctx context.Context) ([]GateVerdict, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select distinct on (module) build, module, commit_hash, previous, plan,
|
||||
machines, verdict, rollback, why, component, judging_from, judged_at, coalesce(epoch, 0)
|
||||
from build_gate order by module, judged_at desc`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanGates(rows)
|
||||
}
|
||||
|
||||
// Gates is the newest verdicts, newest first: what `plans gates` lists.
|
||||
func (i *Inventory) Gates(ctx context.Context, limit int) ([]GateVerdict, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, gateSelect+` order by judged_at desc limit $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanGates(rows)
|
||||
}
|
||||
|
||||
const gateSelect = `select build, module, commit_hash, previous, plan, machines, verdict, rollback, why, component,
|
||||
judging_from, judged_at, coalesce(epoch, 0) from build_gate`
|
||||
|
||||
func scanGates(rows pgx.Rows) ([]GateVerdict, error) {
|
||||
defer rows.Close()
|
||||
var out []GateVerdict
|
||||
for rows.Next() {
|
||||
var v GateVerdict
|
||||
var epoch int64
|
||||
if err := rows.Scan(&v.Build, &v.Module, &v.Commit, &v.Previous, &v.Plan, &v.Machines, &v.Verdict,
|
||||
&v.Rollback, &v.Why, &v.Component, &v.JudgingFrom, &v.JudgedAt, &epoch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v.Epoch = uint64(epoch)
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// PreviousBuild is the build a module goes back to when a build of it fails its gate: the newest build
|
||||
// that worked, made from the commit the first machine ran before, asked before the failed one, and not
|
||||
// itself failed at a gate. Among the builds whose artifacts the mesh keeps (KeptBuilds): an older one
|
||||
// may already be gone from the artifact store. False when there is none to go back to.
|
||||
func (i *Inventory) PreviousBuild(ctx context.Context, module, commit string, failed Build) (Build, bool, error) {
|
||||
builds, err := i.Builds(ctx, module, 50)
|
||||
if err != nil {
|
||||
return Build{}, false, err
|
||||
}
|
||||
kept := 0
|
||||
for _, b := range builds {
|
||||
if !b.Worked() {
|
||||
continue
|
||||
}
|
||||
kept++
|
||||
if kept > KeptBuilds {
|
||||
break
|
||||
}
|
||||
if b.ID == failed.ID || (commit != "" && b.Commit != commit) {
|
||||
continue
|
||||
}
|
||||
if !failed.AskedOrAt().IsZero() && !b.AskedOrAt().Before(failed.AskedOrAt()) {
|
||||
continue
|
||||
}
|
||||
if bad, err := i.GateFailed(ctx, b.ID); err != nil {
|
||||
return Build{}, false, err
|
||||
} else if bad {
|
||||
continue
|
||||
}
|
||||
var manifest []byte
|
||||
if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, b.ID).Scan(&manifest); err != nil {
|
||||
return Build{}, false, err
|
||||
}
|
||||
if len(manifest) == 0 || string(manifest) == "null" {
|
||||
continue
|
||||
}
|
||||
b.Manifest = manifest
|
||||
return b, true, nil
|
||||
}
|
||||
return Build{}, false, nil
|
||||
}
|
||||
|
||||
// RestoreModule puts a module's registered build back to an earlier one: its manifest, the commit it
|
||||
// was built from, and when it was asked — as now, so the build that failed its gate, asked before, can
|
||||
// never register over it again (issue 219's order). The source's head is left where the merge moved it:
|
||||
// the module IS behind its source, and `status` says so.
|
||||
func (i *Inventory) RestoreModule(ctx context.Context, b Build) error {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return fmt.Errorf("%s is not put back: %w", b.Module, err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(b.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's build %s is not a manifest the mesh can register again: %w", b.Module, b.ID, err)
|
||||
}
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update module set manifest = $2, version = nullif($3, ''), built_from = nullif($4, ''),
|
||||
built_asked = now(), registered = now()
|
||||
where name = $1`, b.Module, raw, m.Version, b.Commit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: %s", ErrNoSuchModule, b.Module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
-- A module's build rolls out one machine first, judged at a gate, and rolls back there when the gate
|
||||
-- fails (novox/hq ADR 0235, to-be 45 §8, Phase 4).
|
||||
--
|
||||
-- 1. The upgrade policy becomes a person's choice over the module's own word. Until now every module
|
||||
-- held a policy here, 'record' unless a person had said 'roll-out', and nothing distinguished a
|
||||
-- 'record' somebody chose from the default it always was. From here a null policy is no choice: the
|
||||
-- module's manifest decides (its `upgrade`, its data, whether it is the bus), and its default is to
|
||||
-- roll out. A 'roll-out' a person chose is kept as their choice. A 'record' is the old default and
|
||||
-- becomes no choice — ADR 0235 decides it, and lists every module's resulting policy; a person who
|
||||
-- wants one held again says so with `upgrade <module> record --why`, which is kept with its why.
|
||||
alter table module alter column upgrade drop not null;
|
||||
alter table module alter column upgrade drop default;
|
||||
alter table module drop constraint if exists module_upgrade_check;
|
||||
alter table module add constraint module_upgrade_chosen check (upgrade is null or upgrade in ('record', 'roll-out'));
|
||||
update module set upgrade = null where upgrade = 'record';
|
||||
-- Why the person chose it, and who: said back by `upgrade`, so a held module says why it is held.
|
||||
alter table module add column upgrade_why text not null default '';
|
||||
alter table module add column upgrade_by text not null default '';
|
||||
|
||||
-- 2. The gate's verdict on each build a plan rolled out, one row per build: passed on its first machine,
|
||||
-- or failed there and rolled back. **A build that failed its gate is marked, and is never sent again
|
||||
-- automatically**: registration refuses it, and the rollback is attempted once per build — the row is
|
||||
-- written before the rollback's send, so a controller replaced in between does not send it twice.
|
||||
create table build_gate (
|
||||
build text primary key,
|
||||
module text not null,
|
||||
-- The commit the build was made from, and the one the module was put back to.
|
||||
commit_hash text not null default '',
|
||||
previous text not null default '',
|
||||
plan text not null default '',
|
||||
-- The machines it was judged on: the first machine, and the bus holder when it went with it.
|
||||
machines text[] not null default '{}',
|
||||
-- 'passed', or 'failed'; and for a failed one how the rollback went: 'rolling-back', 'rolled-back',
|
||||
-- or 'not-rolled-back' (no previous build to put back, or the send refused), said in `why`.
|
||||
verdict text not null check (verdict in ('passed', 'failed')),
|
||||
rollback text not null default '' check (rollback in ('', 'rolling-back', 'rolled-back', 'not-rolled-back')),
|
||||
why text not null default '',
|
||||
-- The core component it is, when it is one: mesh-controller, mesh-host, node-tools.
|
||||
component text not null default '',
|
||||
judging_from timestamptz,
|
||||
judged_at timestamptz not null default now(),
|
||||
epoch bigint
|
||||
);
|
||||
create index build_gate_module on build_gate (module, judged_at desc);
|
||||
|
||||
-- 3. The bus's planned step (to-be 45 §8): a bus upgrade is never rolled out; a person starts it, with
|
||||
-- why, after its streams are snapshotted, and it is checked after. One row per step; the open one is
|
||||
-- the step running, which the self-check says as `bus-maintenance` until the bus is healthy again or
|
||||
-- the step's bound passes and it is said failed, with its snapshot as the way back.
|
||||
create table bus_step (
|
||||
id bigserial primary key,
|
||||
module text not null,
|
||||
machines text[] not null default '{}',
|
||||
from_build text not null default '',
|
||||
to_build text not null default '',
|
||||
-- Where the streams' snapshot is: taken by the mesh, or one a person says they took.
|
||||
snapshot text not null,
|
||||
-- Whether the new version can be reverted by putting the old one back, as the person said it.
|
||||
reversible boolean not null,
|
||||
by_whom text not null default '',
|
||||
why text not null,
|
||||
started timestamptz not null default now(),
|
||||
ended timestamptz,
|
||||
-- '', then 'done' or 'failed', with what was found.
|
||||
outcome text not null default '' check (outcome in ('', 'done', 'failed')),
|
||||
found text not null default ''
|
||||
);
|
||||
@@ -67,6 +67,42 @@ type PlanModule struct {
|
||||
// its module's name included. Empty in a plan from before it was kept, which is matched by
|
||||
// module, or by repository and path, as before.
|
||||
Build string `json:"build,omitempty"`
|
||||
// Previous is the build the first machine ran of this module before the plan sent it the new one —
|
||||
// the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back
|
||||
// (novox/hq ADR 0235). Empty when the machine had never been sent the module, or what it was sent
|
||||
// is not known.
|
||||
Previous string `json:"previous,omitempty"`
|
||||
// Gate is the new build's judging on its first machine (novox/hq ADR 0235, to-be 45 §8), kept so a
|
||||
// controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record.
|
||||
Gate *PlanGate `json:"gate,omitempty"`
|
||||
}
|
||||
|
||||
// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,
|
||||
// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back.
|
||||
type PlanGate struct {
|
||||
// Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty
|
||||
// for any other module, judged by its own health.
|
||||
Component string `json:"component,omitempty"`
|
||||
Machines []string `json:"machines"`
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to,omitempty"`
|
||||
// Since is when the judging began: the first machine reported the new build applied.
|
||||
Since *time.Time `json:"since,omitempty"`
|
||||
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
|
||||
// does not resets them.
|
||||
Passes int `json:"passes,omitempty"`
|
||||
LastPass *time.Time `json:"last_pass,omitempty"`
|
||||
// Last is what the newest judging found wanting, while it still may pass.
|
||||
Last string `json:"last,omitempty"`
|
||||
// Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.
|
||||
Verdict string `json:"verdict,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
JudgedAt *time.Time `json:"judged_at,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why.
|
||||
Rollback string `json:"rollback,omitempty"`
|
||||
// Kept says a passing verdict was written to the gate's records.
|
||||
Kept bool `json:"kept,omitempty"`
|
||||
}
|
||||
|
||||
// The states a plan passes through.
|
||||
|
||||
@@ -72,7 +72,8 @@ func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1"}) {
|
||||
// Rolled out by default, one machine first and gated (novox/hq ADR 0235).
|
||||
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true}) {
|
||||
t.Errorf("resolver is at %+v", got)
|
||||
}
|
||||
if got := current["by-hand"]; got != (CurrentBuild{RollOut: true}) {
|
||||
|
||||
@@ -46,6 +46,9 @@ type Holder struct {
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
Taken time.Time `json:"taken"`
|
||||
Renewed time.Time `json:"renewed"`
|
||||
// Health is the holder's word about itself, written with every renewal (witness.go): nil from a
|
||||
// process that says none, which the controller's gate reads as not ready. The host ignores it.
|
||||
Health *Health `json:"health,omitempty"`
|
||||
}
|
||||
|
||||
// Defaults, as to-be 45 §6 sets them. The age is the bucket's, read from it (Open), so the bucket
|
||||
@@ -85,6 +88,9 @@ type Options struct {
|
||||
Moved func(was, floor uint64)
|
||||
// Now is the clock; nil is time.Now.
|
||||
Now func() time.Time
|
||||
// Health is asked at every take and renewal for the holder's word about itself (witness.go); nil
|
||||
// writes none.
|
||||
Health func() *Health
|
||||
}
|
||||
|
||||
// Lease is one instance's hold, or its wait for one.
|
||||
@@ -241,6 +247,9 @@ func (l *Lease) TryTake(ctx context.Context) (uint64, error) {
|
||||
now := l.o.Now()
|
||||
h := l.o.Holder
|
||||
h.Taken, h.Renewed, h.Epoch = now, now, 0
|
||||
if l.o.Health != nil {
|
||||
h.Health = l.o.Health()
|
||||
}
|
||||
value, err := json.Marshal(h)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -390,6 +399,9 @@ func (l *Lease) Renew(ctx context.Context) error {
|
||||
h.Epoch, h.Taken = epoch, taken
|
||||
anchor := l.o.Now()
|
||||
h.Renewed = anchor
|
||||
if l.o.Health != nil {
|
||||
h.Health = l.o.Health()
|
||||
}
|
||||
value, err := json.Marshal(h)
|
||||
if err != nil {
|
||||
return l.lose(err)
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
package lease
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The controller's rollback witness: the contract between the controller and the node-engine that
|
||||
// placed it (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236). Its other half is mesh-host's
|
||||
// internal/witness/contract.go and the Report's `rollbacks` and `witness` (mesh-host internal/link);
|
||||
// the two are held field for field, and a change on either side is a change to both.
|
||||
//
|
||||
// **The component being replaced is never the only witness of its successor.** A new controller that
|
||||
// starts and does nothing, crashes, or cannot reach the bus cannot say so, and cannot put back the build
|
||||
// before it. The node-engine on the machine running the controller can: it placed the new bundle, it
|
||||
// keeps the previous one beside it, and it reads one key on the bus to judge the new one by.
|
||||
//
|
||||
// **What the host reads.** A direct get of the lease bucket's one key — LeaseReadSubject — answered with
|
||||
// the Holder below as JSON (unknown fields ignored, `build` not read). The node principal of every
|
||||
// machine assigned the controller is granted that one subject, and every machine's node principal the
|
||||
// runtime's PING on its own machine (broker.WitnessSubjects); replies come to its own inbox.
|
||||
//
|
||||
// **When the host calls a new controller healthy** (HeldBySince, as the host has it): the holder's host
|
||||
// is this machine, it took the key at or after the moment the host started the new build (less Skew),
|
||||
// and it renewed it within the key's age. Asked every WitnessEvery, within ControllerWithin of the
|
||||
// start; not met by then, the host stops the new build, starts the one it kept, and says so in its
|
||||
// reports (Rollback). A key absent, deleted or purged is held by nobody.
|
||||
//
|
||||
// **What the controller adds, and the host does not read.** Health: whether the holder says it is
|
||||
// ready — its self-check ran and `status` answered in bound. The host's bound is the lease alone, sixty
|
||||
// seconds; the controller's own gate (gate.go) also asks for ready within ten minutes and, failing that,
|
||||
// sends the previous controller build itself, which the host applies as any declaration.
|
||||
//
|
||||
// **What the controller does with what the host says.** Each Rollback a report carries is a condition
|
||||
// `core.<component>.<node>.<outcome>` — urgent for rolled-back, not-reversible, restore-failed and halted;
|
||||
// a warning for nothing-to-restore and unwitnessed — kept while reports carry it and cleared by the first
|
||||
// report from that machine without it. A rolled-back controller or node tools build is marked failed at
|
||||
// its gate and the module's registered build is put back to the one running, so nothing sends it again.
|
||||
|
||||
// Witness bounds, as the host has them (mesh-host internal/witness).
|
||||
const (
|
||||
// ControllerWithin: the new controller holds the lease within this of starting.
|
||||
ControllerWithin = 60 * time.Second
|
||||
// NodeToolsWithin: the node tools answer PING within this of starting, each PING within PingWithin.
|
||||
NodeToolsWithin = 60 * time.Second
|
||||
PingWithin = 5 * time.Second
|
||||
// WitnessEvery is how often the host asks.
|
||||
WitnessEvery = 5 * time.Second
|
||||
// Skew is how far the controller's clock and the host's may disagree about when it took the lease:
|
||||
// one machine, one clock — a margin, not a tolerance.
|
||||
Skew = 2 * time.Second
|
||||
// FreshWithin is the key's age: a holder not renewed within it is not holding it.
|
||||
FreshWithin = 15 * time.Second
|
||||
// ReadyWithin is the controller's own bound for saying it is ready (Health), which its gate judges.
|
||||
ReadyWithin = 10 * time.Minute
|
||||
)
|
||||
|
||||
// LeaseReadSubject is the one subject the witness of the controller publishes to read the lease: a
|
||||
// direct get of the key `holder`.
|
||||
func LeaseReadSubject(bucket string) string {
|
||||
return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + Key
|
||||
}
|
||||
|
||||
// PingSubject is the subject a machine's witness asks its own node tools on: the services protocol's
|
||||
// PING to the runtime, whose instance is the machine's name.
|
||||
func PingSubject(node string) string { return "$SRV.PING." + NodeToolsService + "." + node }
|
||||
|
||||
// NodeToolsService is the runtime's name on the services protocol.
|
||||
const NodeToolsService = "node-tools"
|
||||
|
||||
// Health is what the holding controller says of itself in every write of the lease's key. The host
|
||||
// does not read it; the controller's gate does. A controller that says nothing is not ready.
|
||||
type Health struct {
|
||||
// Ready is the controller's health definition met (to-be 45 §8): it holds the lease, its self-check
|
||||
// has run once, and in that run `status` answered in full within ten seconds (D9).
|
||||
Ready bool `json:"ready"`
|
||||
// ReadyAt is when it first became ready; zero while it is not.
|
||||
ReadyAt time.Time `json:"ready_at,omitempty"`
|
||||
// Started is when this process started.
|
||||
Started time.Time `json:"started"`
|
||||
// DoctorRan is when its self-check last finished a run; zero before the first.
|
||||
DoctorRan time.Time `json:"doctor_ran,omitempty"`
|
||||
// Why says what is missing while it is not ready, in the mesh's words.
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// The witness contract version a host keeps (Report.Witness): its presence says the host reads a
|
||||
// process's `witness` and `not-reversible`, which an older, strict host refuses — so the controller
|
||||
// sends them only to a machine whose report carries it.
|
||||
const WitnessContract = 1
|
||||
|
||||
// The core components a witness judges, as a Rollback names them.
|
||||
const (
|
||||
ComponentEngine = "node-engine"
|
||||
ComponentController = "controller"
|
||||
ComponentNodeTools = "node-tools"
|
||||
)
|
||||
|
||||
// What a witness concluded, as a Rollback says it.
|
||||
const (
|
||||
OutcomeRolledBack = "rolled-back"
|
||||
OutcomeNotReversible = "not-reversible"
|
||||
OutcomeNothingToRestore = "nothing-to-restore"
|
||||
OutcomeRestoreFailed = "restore-failed"
|
||||
OutcomeUnwitnessed = "unwitnessed"
|
||||
OutcomeHalted = "halted"
|
||||
)
|
||||
|
||||
// Urgent says whether a witness's outcome needs the operator now.
|
||||
func Urgent(outcome string) bool {
|
||||
switch outcome {
|
||||
case OutcomeRolledBack, OutcomeNotReversible, OutcomeRestoreFailed, OutcomeHalted:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Rollback is one witness's verdict on one core build, as the node-engine says it in its report
|
||||
// (`rollbacks`) — on every report while it stands, until a newer build of that component is declared to
|
||||
// it and proves itself.
|
||||
type Rollback struct {
|
||||
// Component is node-engine, controller or node-tools.
|
||||
Component string `json:"component"`
|
||||
// From is the build judged: a host version for the node-engine, a bundle's digest for a process.
|
||||
From string `json:"from"`
|
||||
// To is the build restored; empty when none was.
|
||||
To string `json:"to,omitempty"`
|
||||
// Outcome is one of the Outcome words.
|
||||
Outcome string `json:"outcome"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// ModuleOf is the module a core component is delivered as.
|
||||
func ModuleOf(component string) string {
|
||||
switch component {
|
||||
case ComponentController:
|
||||
return "mesh-controller"
|
||||
case ComponentEngine:
|
||||
return "mesh-host"
|
||||
case ComponentNodeTools:
|
||||
return NodeToolsService
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// HeldBySince is the host's judgement of a new controller (mesh-host witness.ControllerLease.HeldBySince,
|
||||
// kept here so both sides test one rule): the lease is held by a controller on machine `host` that took
|
||||
// it at or after `since`, less Skew, and renewed it within the key's age.
|
||||
func (h Holder) HeldBySince(host string, since, now time.Time) (bool, string) {
|
||||
last := h.Taken
|
||||
if h.Renewed.After(last) {
|
||||
last = h.Renewed
|
||||
}
|
||||
switch {
|
||||
case h.Instance == "":
|
||||
return false, "the lease names no holder"
|
||||
case host != "" && !sameMachine(h.Host, host):
|
||||
return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", h.Instance, h.Host)
|
||||
case h.Taken.Before(since.Add(-Skew)):
|
||||
return false, fmt.Sprintf("the lease is held by %s, taken before the new build started", h.Instance)
|
||||
case now.Sub(last) > FreshWithin:
|
||||
return false, fmt.Sprintf("the lease names %s and was last renewed %s ago", h.Instance,
|
||||
now.Sub(last).Round(time.Second))
|
||||
}
|
||||
return true, fmt.Sprintf("%s holds the lease, epoch %d", h.Instance, h.Epoch)
|
||||
}
|
||||
|
||||
// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree.
|
||||
func sameMachine(a, b string) bool {
|
||||
short := func(s string) string {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
if i := strings.IndexByte(s, '.'); i > 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
return short(a) == short(b)
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package lease
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The host's judgement of a new controller, held here as the host holds it (mesh-host internal/witness):
|
||||
// the lease held by a controller on this machine, taken since the new build started, and fresh.
|
||||
func TestTheWitnessJudgesANewControllerByTheLease(t *testing.T) {
|
||||
started := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||
now := started.Add(30 * time.Second)
|
||||
fresh := Holder{Instance: "controller@control pid 2", Host: "control.example", Taken: started.Add(5 * time.Second),
|
||||
Renewed: now.Add(-3 * time.Second), Epoch: 9}
|
||||
if ok, why := fresh.HeldBySince("control", started, now); !ok {
|
||||
t.Fatalf("a new controller holding the lease was not healthy: %s", why)
|
||||
}
|
||||
old := fresh
|
||||
old.Taken = started.Add(-time.Minute)
|
||||
if ok, _ := old.HeldBySince("control", started, now); ok {
|
||||
t.Error("the controller from before the build counted as the new one")
|
||||
}
|
||||
elsewhere := fresh
|
||||
elsewhere.Host = "other"
|
||||
if ok, _ := elsewhere.HeldBySince("control", started, now); ok {
|
||||
t.Error("a controller on another machine counted")
|
||||
}
|
||||
stale := fresh
|
||||
stale.Renewed = now.Add(-20 * time.Second)
|
||||
stale.Taken = stale.Renewed
|
||||
if ok, _ := stale.HeldBySince("control", started.Add(-time.Minute), now); ok {
|
||||
t.Error("a lease not renewed within its age counted as held")
|
||||
}
|
||||
if ok, _ := (Holder{}).HeldBySince("control", started, now); ok {
|
||||
t.Error("nobody counted as the holder")
|
||||
}
|
||||
}
|
||||
|
||||
// The outcomes the operator is woken for, and the subjects the host reads.
|
||||
func TestTheWitnessContractsWords(t *testing.T) {
|
||||
for outcome, urgent := range map[string]bool{OutcomeRolledBack: true, OutcomeNotReversible: true,
|
||||
OutcomeRestoreFailed: true, OutcomeHalted: true, OutcomeNothingToRestore: false, OutcomeUnwitnessed: false} {
|
||||
if Urgent(outcome) != urgent {
|
||||
t.Errorf("%s urgent: %v", outcome, Urgent(outcome))
|
||||
}
|
||||
}
|
||||
if got := LeaseReadSubject("mesh-controller_lease"); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" {
|
||||
t.Errorf("the lease is read on %s", got)
|
||||
}
|
||||
if got := PingSubject("ace"); got != "$SRV.PING.node-tools.ace" {
|
||||
t.Errorf("the node tools are asked on %s", got)
|
||||
}
|
||||
for c, m := range map[string]string{ComponentController: "mesh-controller", ComponentEngine: "mesh-host",
|
||||
ComponentNodeTools: "node-tools"} {
|
||||
if ModuleOf(c) != m {
|
||||
t.Errorf("%s is delivered as %s", c, ModuleOf(c))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,9 @@ const (
|
||||
// is spent and the condition is the operator's (novox/hq to-be 45 §7). Never a person's act: those
|
||||
// are the hand-act log's.
|
||||
KeyHealerActed = "healer-acted"
|
||||
// KeyRolledBack: a build failed its gate on its first machine and was put back there, or could not
|
||||
// be (novox/hq ADR 0235, to-be 45 §8); or a witness on a machine put a core component back.
|
||||
KeyRolledBack = "rolled-back"
|
||||
)
|
||||
|
||||
// Applied is what a machine now runs, as the mesh states it.
|
||||
@@ -180,6 +183,11 @@ type SourceMoved struct {
|
||||
// rather than inferred from a round number, because "this is all of it" and "this is as much as
|
||||
// I asked for" are the difference between rebuilding a module and leaving it stale.
|
||||
PathsTruncated bool `json:"paths_truncated,omitempty"`
|
||||
|
||||
// Removed are the files among Paths the merge deleted. A module whose manifest is among them was
|
||||
// deleted at its source: it is forgotten, or said, and never built (novox/hq ADR 0235). Empty from an
|
||||
// announcer that does not say which files went, and then a build that finds no manifest says it.
|
||||
Removed []string `json:"removed,omitempty"`
|
||||
}
|
||||
|
||||
type Upgraded struct {
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// Exchange is where nodes publish everything they have to say.
|
||||
@@ -251,6 +253,15 @@ type Report struct {
|
||||
// and the mesh's up in its place, and where the found configuration's original was kept.
|
||||
Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
|
||||
|
||||
// Rollbacks is what this machine's witnesses decided about a core build that was not healthy in
|
||||
// bound (novox/hq to-be 45 §8, ADR 0236; the contract is lease/witness.go and mesh-host's
|
||||
// internal/witness): the node-engine's launcher about the engine, the engine about the controller and
|
||||
// the node tools. Said on every report while it stands; absent from a host that witnesses nothing.
|
||||
Rollbacks []lease.Rollback `json:"rollbacks,omitempty"`
|
||||
// Witness is the witness contract version the host keeps (lease.WitnessContract): a process's
|
||||
// `witness` and `not-reversible` are sent only to a machine whose report carries it.
|
||||
Witness int `json:"witness,omitempty"`
|
||||
|
||||
// Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq
|
||||
// ADR 0105). A report carrying one is not an account of the machine: it moves the node's
|
||||
// overlay key and tunnel and nothing else.
|
||||
|
||||
Reference in New Issue
Block a user