Merge commit 'b4eb78c2' into feat/47-G-fix11-b5g6
This commit is contained in:
@@ -21,6 +21,7 @@ const (
|
||||
VerbDrop = "drop"
|
||||
VerbKeptArtifact = "kept-artifact"
|
||||
VerbSteps = "steps"
|
||||
VerbOutput = "output"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -41,7 +42,7 @@ const (
|
||||
|
||||
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
|
||||
func Verbs() []string {
|
||||
return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps}
|
||||
return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps, VerbOutput}
|
||||
}
|
||||
|
||||
// Events is every event published on the seat's subject: a new copy each call.
|
||||
@@ -804,9 +805,9 @@ type StepEnded struct {
|
||||
Node string `json:"node"`
|
||||
// never passed for a step that ran nothing
|
||||
Outcome string `json:"outcome"`
|
||||
// the step's output when it fits the message
|
||||
// the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254)
|
||||
Output string `json:"output,omitempty"`
|
||||
// the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16)
|
||||
// the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output
|
||||
OutputObject string `json:"output-object,omitempty"`
|
||||
Pipeline string `json:"pipeline,omitempty"`
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
@@ -1064,6 +1065,22 @@ type StepsAnswer struct {
|
||||
Steps []DetectedStep `json:"steps,omitempty"`
|
||||
}
|
||||
|
||||
// OutputInput is what output takes.
|
||||
type OutputInput struct {
|
||||
// the ask's id, the `output-object` of its step-ended naming it
|
||||
Ask string `json:"ask"`
|
||||
// 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements
|
||||
Call string `json:"call"`
|
||||
}
|
||||
|
||||
// OutputAnswer is what output answers.
|
||||
type OutputAnswer struct {
|
||||
// the ask's id
|
||||
Ask string `json:"ask"`
|
||||
// how many chunks are sent; 0 when this node keeps no answer of that ask
|
||||
Chunks int64 `json:"chunks"`
|
||||
}
|
||||
|
||||
// DeliveryAccept is the body of delivery.
|
||||
type DeliveryAccept = StepAsk
|
||||
|
||||
|
||||
@@ -121,6 +121,10 @@ func TestFixtures(t *testing.T) {
|
||||
decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}",
|
||||
decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "output asks a run's answer by its ask", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}",
|
||||
decode: func(b []byte) (any, error) { var v OutputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "output answers how many chunks carry the answer", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}",
|
||||
decode: func(b []byte) (any, error) { var v OutputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
}
|
||||
for _, c := range cases {
|
||||
v, err := c.decode([]byte(c.given))
|
||||
|
||||
@@ -21,6 +21,7 @@ const (
|
||||
VerbDrop = "drop"
|
||||
VerbKeptArtifact = "kept-artifact"
|
||||
VerbSteps = "steps"
|
||||
VerbOutput = "output"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -40,7 +41,7 @@ const (
|
||||
|
||||
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
|
||||
func Verbs() []string {
|
||||
return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps}
|
||||
return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps, VerbOutput}
|
||||
}
|
||||
|
||||
// Events is every event published on the seat's subject: a new copy each call.
|
||||
@@ -802,9 +803,9 @@ type StepEnded struct {
|
||||
Node string `json:"node"`
|
||||
// never passed for a step that ran nothing
|
||||
Outcome string `json:"outcome"`
|
||||
// the step's output when it fits the message
|
||||
// the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254)
|
||||
Output string `json:"output,omitempty"`
|
||||
// the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16)
|
||||
// the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output
|
||||
OutputObject string `json:"output-object,omitempty"`
|
||||
Pipeline string `json:"pipeline,omitempty"`
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
@@ -1062,6 +1063,22 @@ type StepsAnswer struct {
|
||||
Steps []DetectedStep `json:"steps,omitempty"`
|
||||
}
|
||||
|
||||
// OutputInput is what output takes.
|
||||
type OutputInput struct {
|
||||
// the ask's id, the `output-object` of its step-ended naming it
|
||||
Ask string `json:"ask"`
|
||||
// 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements
|
||||
Call string `json:"call"`
|
||||
}
|
||||
|
||||
// OutputAnswer is what output answers.
|
||||
type OutputAnswer struct {
|
||||
// the ask's id
|
||||
Ask string `json:"ask"`
|
||||
// how many chunks are sent; 0 when this node keeps no answer of that ask
|
||||
Chunks int64 `json:"chunks"`
|
||||
}
|
||||
|
||||
// DeliveryAccept is the body of delivery.
|
||||
type DeliveryAccept = StepAsk
|
||||
|
||||
|
||||
@@ -103,6 +103,10 @@ func TestFixtures(t *testing.T) {
|
||||
decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}",
|
||||
decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "output asks a run's answer by its ask", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}",
|
||||
decode: func(b []byte) (any, error) { var v OutputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "output answers how many chunks carry the answer", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}",
|
||||
decode: func(b []byte) (any, error) { var v OutputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
}
|
||||
for _, c := range cases {
|
||||
v, err := c.decode([]byte(c.given))
|
||||
|
||||
@@ -21,6 +21,7 @@ const (
|
||||
VerbDrop = "drop"
|
||||
VerbKeptArtifact = "kept-artifact"
|
||||
VerbSteps = "steps"
|
||||
VerbOutput = "output"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -40,7 +41,7 @@ const (
|
||||
|
||||
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
|
||||
func Verbs() []string {
|
||||
return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps}
|
||||
return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps, VerbOutput}
|
||||
}
|
||||
|
||||
// Events is every event published on the seat's subject: a new copy each call.
|
||||
@@ -802,9 +803,9 @@ type StepEnded struct {
|
||||
Node string `json:"node"`
|
||||
// never passed for a step that ran nothing
|
||||
Outcome string `json:"outcome"`
|
||||
// the step's output when it fits the message
|
||||
// the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254)
|
||||
Output string `json:"output,omitempty"`
|
||||
// the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16)
|
||||
// the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output
|
||||
OutputObject string `json:"output-object,omitempty"`
|
||||
Pipeline string `json:"pipeline,omitempty"`
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
@@ -1062,6 +1063,22 @@ type StepsAnswer struct {
|
||||
Steps []DetectedStep `json:"steps,omitempty"`
|
||||
}
|
||||
|
||||
// OutputInput is what output takes.
|
||||
type OutputInput struct {
|
||||
// the ask's id, the `output-object` of its step-ended naming it
|
||||
Ask string `json:"ask"`
|
||||
// 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements
|
||||
Call string `json:"call"`
|
||||
}
|
||||
|
||||
// OutputAnswer is what output answers.
|
||||
type OutputAnswer struct {
|
||||
// the ask's id
|
||||
Ask string `json:"ask"`
|
||||
// how many chunks are sent; 0 when this node keeps no answer of that ask
|
||||
Chunks int64 `json:"chunks"`
|
||||
}
|
||||
|
||||
// DeliveryAccept is the body of delivery.
|
||||
type DeliveryAccept = StepAsk
|
||||
|
||||
|
||||
@@ -107,6 +107,10 @@ func TestFixtures(t *testing.T) {
|
||||
decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}",
|
||||
decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "output asks a run's answer by its ask", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}",
|
||||
decode: func(b []byte) (any, error) { var v OutputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "output answers how many chunks carry the answer", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}",
|
||||
decode: func(b []byte) (any, error) { var v OutputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
}
|
||||
for _, c := range cases {
|
||||
v, err := c.decode([]byte(c.given))
|
||||
|
||||
@@ -233,6 +233,55 @@
|
||||
"required": []
|
||||
},
|
||||
"class": "act"
|
||||
},
|
||||
{
|
||||
"name": "output",
|
||||
"class": "read",
|
||||
"callers": [
|
||||
"seat:mesh-catalog",
|
||||
"seat:mesh-wants",
|
||||
"seat:mesh-delivery-checker",
|
||||
"seat:mesh-pipeline-runner"
|
||||
],
|
||||
"description": "A run's answer by its ask, when the step-ended of the run names it as `output-object` because the answer is over what the event carries (design 47 G A-254): the answer's bytes in chunks of at most 512 KiB on the chunk subject of the call, each acknowledged by the caller on the ack subject within 30 seconds; the answer is of any size. An answer inline in the event is never asked here.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ask": {
|
||||
"type": "string",
|
||||
"description": "the ask's id, the `output-object` of its step-ended naming it",
|
||||
"minLength": 1
|
||||
},
|
||||
"call": {
|
||||
"type": "string",
|
||||
"description": "128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"ask",
|
||||
"call"
|
||||
]
|
||||
},
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ask": {
|
||||
"type": "string",
|
||||
"description": "the ask's id"
|
||||
},
|
||||
"chunks": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "how many chunks are sent; 0 when this node keeps no answer of that ask"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"ask",
|
||||
"chunks"
|
||||
]
|
||||
},
|
||||
"chunks": true
|
||||
}
|
||||
],
|
||||
"defs": {
|
||||
@@ -1595,11 +1644,11 @@
|
||||
},
|
||||
"output": {
|
||||
"type": "string",
|
||||
"description": "the step's output when it fits the message"
|
||||
"description": "the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254)"
|
||||
},
|
||||
"output-object": {
|
||||
"type": "string",
|
||||
"description": "the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16)"
|
||||
"description": "the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output"
|
||||
},
|
||||
"rerun": {
|
||||
"type": "boolean"
|
||||
|
||||
@@ -124,5 +124,21 @@
|
||||
"given": {
|
||||
"error": "the hand-act log refused: nothing was deleted"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "output asks a run's answer by its ask",
|
||||
"of": "verb:output.input",
|
||||
"given": {
|
||||
"ask": "m2kx9a1b3c4d5f",
|
||||
"call": "0123456789abcdef0123456789abcdef"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "output answers how many chunks carry the answer",
|
||||
"of": "verb:output.answer",
|
||||
"given": {
|
||||
"ask": "m2kx9a1b3c4d5f",
|
||||
"chunks": 3
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -124,5 +124,21 @@
|
||||
"given": {
|
||||
"error": "the hand-act log refused: nothing was deleted"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "output asks a run's answer by its ask",
|
||||
"of": "verb:output.input",
|
||||
"given": {
|
||||
"ask": "m2kx9a1b3c4d5f",
|
||||
"call": "0123456789abcdef0123456789abcdef"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "output answers how many chunks carry the answer",
|
||||
"of": "verb:output.answer",
|
||||
"given": {
|
||||
"ask": "m2kx9a1b3c4d5f",
|
||||
"chunks": 3
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -124,5 +124,21 @@
|
||||
"given": {
|
||||
"error": "the hand-act log refused: nothing was deleted"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "output asks a run's answer by its ask",
|
||||
"of": "verb:output.input",
|
||||
"given": {
|
||||
"ask": "m2kx9a1b3c4d5f",
|
||||
"call": "0123456789abcdef0123456789abcdef"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "output answers how many chunks carry the answer",
|
||||
"of": "verb:output.answer",
|
||||
"given": {
|
||||
"ask": "m2kx9a1b3c4d5f",
|
||||
"chunks": 3
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -13,6 +13,7 @@ export declare const verbs: {
|
||||
readonly drop: "drop";
|
||||
readonly keptArtifact: "kept-artifact";
|
||||
readonly steps: "steps";
|
||||
readonly output: "output";
|
||||
};
|
||||
|
||||
export declare const accepts: {
|
||||
@@ -146,6 +147,8 @@ export type LevelAnswerDigestPackagesItem = buildStep.LevelAnswerDigestPackagesI
|
||||
export type LevelAnswerKeptItem = buildStep.LevelAnswerKeptItem;
|
||||
export type LevelAnswerUnmetItem = buildStep.LevelAnswerUnmetItem;
|
||||
export type ModuleTree = buildStep.ModuleTree;
|
||||
export type OutputAnswer = buildStep.OutputAnswer;
|
||||
export type OutputInput = buildStep.OutputInput;
|
||||
export type Pause = buildStep.Pause;
|
||||
export type PauseAnswer = buildStep.PauseAnswer;
|
||||
export type PauseInput = buildStep.PauseInput;
|
||||
|
||||
@@ -11,6 +11,7 @@ export const verbs = {
|
||||
drop: "drop",
|
||||
keptArtifact: "kept-artifact",
|
||||
steps: "steps",
|
||||
output: "output",
|
||||
};
|
||||
|
||||
export const accepts = {
|
||||
|
||||
+19
-2
@@ -9,6 +9,7 @@ export declare const verbs: {
|
||||
readonly drop: "drop";
|
||||
readonly keptArtifact: "kept-artifact";
|
||||
readonly steps: "steps";
|
||||
readonly output: "output";
|
||||
};
|
||||
|
||||
export declare const events: {
|
||||
@@ -432,9 +433,9 @@ export interface StepEnded {
|
||||
"node": string;
|
||||
/** never passed for a step that ran nothing */
|
||||
"outcome": "passed" | "failed" | "nothing to run" | "could not run";
|
||||
/** the step's output when it fits the message */
|
||||
/** the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) */
|
||||
"output"?: string;
|
||||
/** the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) */
|
||||
/** the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output */
|
||||
"output-object"?: string;
|
||||
"pipeline"?: string;
|
||||
"rerun"?: boolean;
|
||||
@@ -655,6 +656,22 @@ export interface StepsAnswer {
|
||||
"steps"?: DetectedStep[];
|
||||
}
|
||||
|
||||
/** is what output takes. */
|
||||
export interface OutputInput {
|
||||
/** the ask's id, the `output-object` of its step-ended naming it */
|
||||
"ask": string;
|
||||
/** 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements */
|
||||
"call": string;
|
||||
}
|
||||
|
||||
/** is what output answers. */
|
||||
export interface OutputAnswer {
|
||||
/** the ask's id */
|
||||
"ask": string;
|
||||
/** how many chunks are sent; 0 when this node keeps no answer of that ask */
|
||||
"chunks": number;
|
||||
}
|
||||
|
||||
/** is the body of the event step-ended. */
|
||||
export type StepEndedEvent = StepEnded;
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ export const verbs = {
|
||||
drop: "drop",
|
||||
keptArtifact: "kept-artifact",
|
||||
steps: "steps",
|
||||
output: "output",
|
||||
};
|
||||
|
||||
export const events = {
|
||||
|
||||
@@ -13,6 +13,7 @@ export declare const verbs: {
|
||||
readonly drop: "drop";
|
||||
readonly keptArtifact: "kept-artifact";
|
||||
readonly steps: "steps";
|
||||
readonly output: "output";
|
||||
};
|
||||
|
||||
export declare const accepts: {
|
||||
@@ -143,6 +144,8 @@ export type LevelAnswerDigestPackagesItem = buildStep.LevelAnswerDigestPackagesI
|
||||
export type LevelAnswerKeptItem = buildStep.LevelAnswerKeptItem;
|
||||
export type LevelAnswerUnmetItem = buildStep.LevelAnswerUnmetItem;
|
||||
export type ModuleTree = buildStep.ModuleTree;
|
||||
export type OutputAnswer = buildStep.OutputAnswer;
|
||||
export type OutputInput = buildStep.OutputInput;
|
||||
export type Pause = buildStep.Pause;
|
||||
export type PauseAnswer = buildStep.PauseAnswer;
|
||||
export type PauseInput = buildStep.PauseInput;
|
||||
|
||||
@@ -11,6 +11,7 @@ export const verbs = {
|
||||
drop: "drop",
|
||||
keptArtifact: "kept-artifact",
|
||||
steps: "steps",
|
||||
output: "output",
|
||||
};
|
||||
|
||||
export const accepts = {
|
||||
|
||||
+19
-2
@@ -9,6 +9,7 @@ export declare const verbs: {
|
||||
readonly drop: "drop";
|
||||
readonly keptArtifact: "kept-artifact";
|
||||
readonly steps: "steps";
|
||||
readonly output: "output";
|
||||
};
|
||||
|
||||
export declare const events: {
|
||||
@@ -432,9 +433,9 @@ export interface StepEnded {
|
||||
"node": string;
|
||||
/** never passed for a step that ran nothing */
|
||||
"outcome": "passed" | "failed" | "nothing to run" | "could not run";
|
||||
/** the step's output when it fits the message */
|
||||
/** the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) */
|
||||
"output"?: string;
|
||||
/** the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) */
|
||||
/** the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output */
|
||||
"output-object"?: string;
|
||||
"pipeline"?: string;
|
||||
"rerun"?: boolean;
|
||||
@@ -655,6 +656,22 @@ export interface StepsAnswer {
|
||||
"steps"?: DetectedStep[];
|
||||
}
|
||||
|
||||
/** is what output takes. */
|
||||
export interface OutputInput {
|
||||
/** the ask's id, the `output-object` of its step-ended naming it */
|
||||
"ask": string;
|
||||
/** 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements */
|
||||
"call": string;
|
||||
}
|
||||
|
||||
/** is what output answers. */
|
||||
export interface OutputAnswer {
|
||||
/** the ask's id */
|
||||
"ask": string;
|
||||
/** how many chunks are sent; 0 when this node keeps no answer of that ask */
|
||||
"chunks": number;
|
||||
}
|
||||
|
||||
/** is the body of the event step-ended. */
|
||||
export type StepEndedEvent = StepEnded;
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ export const verbs = {
|
||||
drop: "drop",
|
||||
keptArtifact: "kept-artifact",
|
||||
steps: "steps",
|
||||
output: "output",
|
||||
};
|
||||
|
||||
export const events = {
|
||||
|
||||
@@ -13,6 +13,7 @@ export declare const verbs: {
|
||||
readonly drop: "drop";
|
||||
readonly keptArtifact: "kept-artifact";
|
||||
readonly steps: "steps";
|
||||
readonly output: "output";
|
||||
};
|
||||
|
||||
export declare const accepts: {
|
||||
@@ -143,6 +144,8 @@ export type LevelAnswerDigestPackagesItem = buildStep.LevelAnswerDigestPackagesI
|
||||
export type LevelAnswerKeptItem = buildStep.LevelAnswerKeptItem;
|
||||
export type LevelAnswerUnmetItem = buildStep.LevelAnswerUnmetItem;
|
||||
export type ModuleTree = buildStep.ModuleTree;
|
||||
export type OutputAnswer = buildStep.OutputAnswer;
|
||||
export type OutputInput = buildStep.OutputInput;
|
||||
export type Pause = buildStep.Pause;
|
||||
export type PauseAnswer = buildStep.PauseAnswer;
|
||||
export type PauseInput = buildStep.PauseInput;
|
||||
|
||||
@@ -11,6 +11,7 @@ export const verbs = {
|
||||
drop: "drop",
|
||||
keptArtifact: "kept-artifact",
|
||||
steps: "steps",
|
||||
output: "output",
|
||||
};
|
||||
|
||||
export const accepts = {
|
||||
|
||||
+19
-2
@@ -9,6 +9,7 @@ export declare const verbs: {
|
||||
readonly drop: "drop";
|
||||
readonly keptArtifact: "kept-artifact";
|
||||
readonly steps: "steps";
|
||||
readonly output: "output";
|
||||
};
|
||||
|
||||
export declare const events: {
|
||||
@@ -432,9 +433,9 @@ export interface StepEnded {
|
||||
"node": string;
|
||||
/** never passed for a step that ran nothing */
|
||||
"outcome": "passed" | "failed" | "nothing to run" | "could not run";
|
||||
/** the step's output when it fits the message */
|
||||
/** the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) */
|
||||
"output"?: string;
|
||||
/** the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) */
|
||||
/** the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output */
|
||||
"output-object"?: string;
|
||||
"pipeline"?: string;
|
||||
"rerun"?: boolean;
|
||||
@@ -655,6 +656,22 @@ export interface StepsAnswer {
|
||||
"steps"?: DetectedStep[];
|
||||
}
|
||||
|
||||
/** is what output takes. */
|
||||
export interface OutputInput {
|
||||
/** the ask's id, the `output-object` of its step-ended naming it */
|
||||
"ask": string;
|
||||
/** 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements */
|
||||
"call": string;
|
||||
}
|
||||
|
||||
/** is what output answers. */
|
||||
export interface OutputAnswer {
|
||||
/** the ask's id */
|
||||
"ask": string;
|
||||
/** how many chunks are sent; 0 when this node keeps no answer of that ask */
|
||||
"chunks": number;
|
||||
}
|
||||
|
||||
/** is the body of the event step-ended. */
|
||||
export type StepEndedEvent = StepEnded;
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ export const verbs = {
|
||||
drop: "drop",
|
||||
keptArtifact: "kept-artifact",
|
||||
steps: "steps",
|
||||
output: "output",
|
||||
};
|
||||
|
||||
export const events = {
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -77,22 +76,3 @@ func (o *inputObjects) List() ([]stateobjects.Object, error) {
|
||||
}
|
||||
return b.List()
|
||||
}
|
||||
|
||||
// coreChunks is the core connection as a send of chunks reaches it (index.ChunkLine): a chunk is published on no stream, an
|
||||
// acknowledgement is a request the holder answers.
|
||||
type coreChunks struct{ nc *nats.Conn }
|
||||
|
||||
func (c coreChunks) Publish(subject string, data []byte) error { return c.nc.Publish(subject, data) }
|
||||
|
||||
func (c coreChunks) Hear(subject string, fn func(string)) (func(), error) {
|
||||
sub, err := c.nc.Subscribe(subject, func(m *nats.Msg) {
|
||||
fn(m.Subject)
|
||||
if m.Reply != "" {
|
||||
_ = m.Respond([]byte("{}"))
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() { _ = sub.Unsubscribe() }, nil
|
||||
}
|
||||
|
||||
@@ -450,7 +450,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
inputsStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.InputsBucket, "no input of a run ask is kept, so the verb input answers none", logger.Printf)}
|
||||
inObjects := &inputObjects{js: js, module: cred.Module}
|
||||
stopInput, err := calls.AnswerSeat(meshcatalog.Seat, map[string]process.SeatHandler{
|
||||
meshcatalog.VerbInput: index.InputVerb(inputsStore, inObjects, coreChunks{conn}, buscalls.SeatAckWithin, logger.Printf)})
|
||||
meshcatalog.VerbInput: index.InputVerb(inputsStore, inObjects, conn, buscalls.SeatAckWithin, logger.Printf)})
|
||||
if err != nil {
|
||||
return fmt.Errorf("answering the verb input: %w", err)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
@@ -42,7 +44,7 @@ func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil }
|
||||
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
|
||||
type lineMem struct {
|
||||
mu sync.Mutex
|
||||
hear func(string)
|
||||
hear nats.MsgHandler
|
||||
chunks map[string][]byte
|
||||
}
|
||||
|
||||
@@ -55,15 +57,15 @@ func (l *lineMem) Publish(subject string, data []byte) error {
|
||||
if err := json.Unmarshal(data, &c); err != nil {
|
||||
return err
|
||||
}
|
||||
go h(buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}.Ack(c.N))
|
||||
go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *lineMem) Hear(_ string, fn func(string)) (func(), error) {
|
||||
func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
|
||||
l.mu.Lock()
|
||||
l.hear = fn
|
||||
l.mu.Unlock()
|
||||
return func() {}, nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Design 47 G §712, A-237: an input is of any size: over a key's limit it is kept as an object beside the state, and the verb
|
||||
|
||||
@@ -225,20 +225,12 @@ func InputBody(ctx context.Context, st Store, objs stateobjects.Objects, digest
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// ChunkLine is the core bus as a send of chunks reaches it: a chunk is published on its subject, kept on no stream, and each is
|
||||
// acknowledged by the caller as a request on its ack subject, which the holder answers (design 47 G §681).
|
||||
type ChunkLine interface {
|
||||
Publish(subject string, data []byte) error
|
||||
// Hear calls fn with the subject of each request made on subject (a pattern) and answers it, until stop is called.
|
||||
Hear(subject string, fn func(subject string)) (stop func(), err error)
|
||||
}
|
||||
|
||||
var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`)
|
||||
|
||||
// InputVerb answers the verb input (marked chunks): the digest and how many chunks carry the bytes, then sends them in chunks of at
|
||||
// most buscalls.SeatChunkBytes on the call's chunk subjects, each acknowledged before the next (A-237, A-139). The input is of any
|
||||
// size. A caller that does not acknowledge within buscalls.SeatAckWithin ends the send, said in logf.
|
||||
func InputVerb(st Store, objs stateobjects.Objects, line ChunkLine, within time.Duration, logf func(string, ...any)) func(caller string, body json.RawMessage) (any, error) {
|
||||
func InputVerb(st Store, objs stateobjects.Objects, line buscalls.ChunkLine, within time.Duration, logf func(string, ...any)) func(caller string, body json.RawMessage) (any, error) {
|
||||
return func(_ string, raw json.RawMessage) (any, error) {
|
||||
var in meshcatalog.InputInput
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
@@ -253,31 +245,10 @@ func InputVerb(st Store, objs stateobjects.Objects, line ChunkLine, within time.
|
||||
}
|
||||
send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: in.Call}
|
||||
total := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
|
||||
acks := make(chan string, 1)
|
||||
// heard before the answer goes, since the caller acknowledges only after it
|
||||
stop, err := line.Hear(buscalls.SeatAcks(meshcatalog.Seat, meshcatalog.VerbInput), func(subject string) {
|
||||
select {
|
||||
case acks <- subject:
|
||||
default:
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the acknowledgements cannot be heard, so no chunk is sent: %w", err)
|
||||
}
|
||||
// the send subscribes to the acknowledgements itself and ends when it has sent every chunk or one is not acknowledged
|
||||
go func() {
|
||||
defer stop()
|
||||
for n := int64(1); n <= total; n++ {
|
||||
from := (n - 1) * buscalls.SeatChunkBytes
|
||||
to := min(int64(len(body)), from+buscalls.SeatChunkBytes)
|
||||
data, _ := json.Marshal(buscalls.SeatChunk{N: n, Of: total, Data: body[from:to]})
|
||||
if err := line.Publish(send.Chunk(n), data); err != nil {
|
||||
logf("the send %s of the input %s ends at chunk %d of %d: %v", in.Call, in.Digest, n, total, err)
|
||||
return
|
||||
}
|
||||
if !acknowledged(acks, send.Ack(n), within) {
|
||||
logf("the send %s of the input %s ends at chunk %d of %d: no acknowledgement within %s", in.Call, in.Digest, n, total, within)
|
||||
return
|
||||
}
|
||||
if err := send.Send(line, body, within, logf); err != nil {
|
||||
logf("the send %s of the input %s: %v", in.Call, in.Digest, err)
|
||||
}
|
||||
}()
|
||||
return meshcatalog.InputAnswer{Digest: in.Digest, Chunks: total}, nil
|
||||
@@ -339,19 +310,3 @@ func InputsReadTool(st Store, objs stateobjects.Objects) stdio.Tool {
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// acknowledged waits within for the acknowledgement on subject, skipping another call's.
|
||||
func acknowledged(acks <-chan string, subject string, within time.Duration) bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), within)
|
||||
defer cancel()
|
||||
for {
|
||||
select {
|
||||
case got := <-acks:
|
||||
if got == subject {
|
||||
return true
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -385,9 +385,7 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1238/AC9 (ADR
|
||||
for _, f := range forbidden {
|
||||
// The seat list reader's Bus interface names a method Subscribe; the indexer's own bus hands it its own events
|
||||
// directly and subscribes to nothing on the bus.
|
||||
// The acknowledgements of the chunks of the verb input are heard on the core connection while a send runs (design 47 G
|
||||
// §712, A-237, §681): cmd/mesh-delivery-indexer/inputs.go.
|
||||
if f == "Subscribe(" && (strings.HasSuffix(path, "seatlist.go") || strings.HasSuffix(path, "mesh-delivery-indexer/inputs.go")) {
|
||||
if f == "Subscribe(" && strings.HasSuffix(path, "seatlist.go") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(string(src), f) {
|
||||
|
||||
@@ -203,6 +203,16 @@ func AskWorker(language, kind string) (string, error) {
|
||||
return stream + "_" + strings.ToUpper(kind) + "_worker", nil
|
||||
}
|
||||
|
||||
// AskNamedWorker is the pull worker of a queue's named asks, `SEAT_<SEAT>_<QUEUE>_named_worker`, filtered to
|
||||
// `mesh.seat.<seat>.accept.<queue>.*`, beside main's worker AskWorker, filtered to main's subject alone (design 47 G §544).
|
||||
func AskNamedWorker(language, kind string) (string, error) {
|
||||
worker, err := AskWorker(language, kind)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSuffix(worker, "_worker") + "_named_worker", nil
|
||||
}
|
||||
|
||||
// AskStepKey is the key of the step of the ask askID on node.
|
||||
func AskStepKey(node, askID string) (string, error) {
|
||||
if err := CheckAskID(askID); err != nil {
|
||||
|
||||
@@ -39,6 +39,14 @@ var StepOutcomes = []string{OutcomePassed, OutcomeFailed, OutcomeNothingToRun, O
|
||||
// StepEndedVersion is the layout version of a step's end.
|
||||
const StepEndedVersion = 1
|
||||
|
||||
// StepEndedInlineMost is the most bytes of a run's answer a step-ended carries inline as `output`: the bus's message limit
|
||||
// less the room its headers take and 16 KiB for the end's other fields. An answer over it is kept by the holder as an object,
|
||||
// which `output-object` names and the seat's verb `output` answers (design 47 G A-254).
|
||||
const StepEndedInlineMost = (1 << 20) - (8 << 10) - (16 << 10)
|
||||
|
||||
// OutputObjectName is the name of the object a build seat's holder keeps a run's answer in, by the ask's id.
|
||||
func OutputObjectName(ask string) string { return "output." + ask }
|
||||
|
||||
// KeptResult is a package's passing result at the digest of what its tests read (ADR 0666 §8).
|
||||
type KeptResult struct {
|
||||
Package string `json:"package"`
|
||||
@@ -154,6 +162,9 @@ func (e StepEnded) check() error {
|
||||
if err := checkNode(e.Node); err != nil {
|
||||
return err
|
||||
}
|
||||
if e.Output != "" && e.OutputObject != "" {
|
||||
return fmt.Errorf("buildstep: the end of %s carries its answer inline and as an object", e.ID)
|
||||
}
|
||||
if e.Pipeline == "" {
|
||||
// G §548, §602: the end of an ask outside a pipeline names its asker, the ask and when, and no commit or
|
||||
// fingerprint.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// OutputVerb is the verb of a build seat that answers a run's answer kept as an object (design 47 G A-254).
|
||||
const OutputVerb = "output"
|
||||
|
||||
// AskVerb asks a seat verb as the runtime calls it (stdio.Ask): the key `seat:<seat>.<verb>[@<node>]` and the body in, the
|
||||
// verb's answer out. A verb marked chunks is asked by the plain call, and the runtime hands the whole file in `file` (A-249).
|
||||
type AskVerb func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
// ReadOutput is the answer a build seat's step-ended carries (design 47 G A-254): its inline `output`, else the object
|
||||
// `output-object` names, read through the verb `output` of the seat of language on the node that ran the ask. Every asker of a
|
||||
// run reads the answer this way, whichever build seat ran it; none reads the seat's step state.
|
||||
func (e StepEnded) ReadOutput(language string, ask AskVerb) ([]byte, error) {
|
||||
if e.OutputObject == "" {
|
||||
return []byte(e.Output), nil
|
||||
}
|
||||
seat, err := BuildSeat(language)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ask == nil {
|
||||
return nil, errors.New("buildstep: the answer is in the output object " + e.OutputObject + ", which this reader has no verb to read")
|
||||
}
|
||||
var call [16]byte
|
||||
if _, err := rand.Read(call[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := ask("seat:"+seat+"."+OutputVerb+"@"+e.Node, map[string]any{"ask": e.Ask, "call": hex.EncodeToString(call[:])})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("buildstep: asking %s for the answer %s of the ask %s: %w", seat, e.OutputObject, e.Ask, err)
|
||||
}
|
||||
var got struct {
|
||||
Ask string `json:"ask"`
|
||||
File string `json:"file"`
|
||||
Chunks int `json:"chunks"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &got); err != nil {
|
||||
return nil, fmt.Errorf("buildstep: the verb output of %s answered no answer for %s: %w", seat, e.Ask, err)
|
||||
}
|
||||
if got.Ask != e.Ask {
|
||||
return nil, fmt.Errorf("buildstep: the verb output of %s answered the ask %q, asked %s", seat, got.Ask, e.Ask)
|
||||
}
|
||||
if got.File == "" {
|
||||
return nil, fmt.Errorf("buildstep: %s on %s keeps no answer of the ask %s", seat, e.Node, e.Ask)
|
||||
}
|
||||
body, err := base64.StdEncoding.DecodeString(got.File)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("buildstep: the answer of %s in no base64: %w", e.Ask, err)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Design 47 G A-254: one reader of a run's answer from a step-ended, inline or through the seat's verb output on the node that ran it.
|
||||
func TestStepEndedReadOutput(t *testing.T) {
|
||||
const ask = "m2kx9a1b3c4d5f"
|
||||
var key string
|
||||
var body map[string]any
|
||||
verb := func(answer string, err error) AskVerb {
|
||||
return func(k string, b any) (json.RawMessage, error) {
|
||||
key = k
|
||||
body, _ = b.(map[string]any)
|
||||
return json.RawMessage(answer), err
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
end StepEnded
|
||||
ask AskVerb
|
||||
want string
|
||||
wantKey string
|
||||
fail string
|
||||
}{
|
||||
{name: "inline", end: StepEnded{Ask: ask, Node: "ace", Output: `{"a":1}`}, want: `{"a":1}`},
|
||||
{name: "an object is read through the verb on its node", end: StepEnded{Ask: ask, Node: "ace", OutputObject: OutputObjectName(ask)},
|
||||
ask: verb(`{"ask":"`+ask+`","file":"eyJhIjoxfQ=="}`, nil), want: `{"a":1}`, wantKey: "seat:node-build-go.output@ace"},
|
||||
{name: "no answer kept", end: StepEnded{Ask: ask, Node: "ace", OutputObject: "o"}, ask: verb(`{"ask":"`+ask+`","chunks":0}`, nil), fail: "keeps no answer"},
|
||||
{name: "another ask's answer", end: StepEnded{Ask: ask, Node: "ace", OutputObject: "o"}, ask: verb(`{"ask":"x","file":"e30="}`, nil), fail: "answered the ask"},
|
||||
{name: "the verb fails", end: StepEnded{Ask: ask, Node: "ace", OutputObject: "o"}, ask: verb(``, errors.New("no seat")), fail: "no seat"},
|
||||
{name: "no verb", end: StepEnded{Ask: ask, Node: "ace", OutputObject: "o"}, fail: "no verb"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
key, body = "", nil
|
||||
got, err := c.end.ReadOutput("go", c.ask)
|
||||
if c.fail != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.fail) {
|
||||
t.Fatalf("err %v, want %q", err, c.fail)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || string(got) != c.want || key != c.wantKey {
|
||||
t.Fatalf("got %q, %v on %q, want %q on %q", got, err, key, c.want, c.wantKey)
|
||||
}
|
||||
if c.wantKey != "" && (body["ask"] != ask || len(body["call"].(string)) != 32) {
|
||||
t.Fatalf("asked with %v, want the ask and 128 random bits as call", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A step-ended names its answer inline or as an object, never both.
|
||||
func TestStepEndedRefusesAnswerTwice(t *testing.T) {
|
||||
e := StepEnded{ID: "m2kx9a1b3c4d5f", Ask: "m2kx9a1b3c4d5f", Asker: "a", Node: "ace", Outcome: OutcomePassed, Output: "1", OutputObject: "o"}
|
||||
if _, err := EncodeStepEnded(e); err == nil || !strings.Contains(err.Error(), "inline and as an object") {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package steprun
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
buildrun "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildrun"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
)
|
||||
|
||||
// **A holder refuses a kind its asker may not ask** (design 47 G §543, A-163, A-170). An ask rides
|
||||
// `mesh.seat.<seat>.accept.<queue>.<asker>`, the asker the asking seat's own name, and its kind stays in its body. The
|
||||
// holder takes the asker from the subject and checks the kind against the callers the seat's definition lists for it (read
|
||||
// from the seat list); on main's subject, which names no asker, it takes only a build and a run.
|
||||
|
||||
// KindOf is the word the seat definition gives an ask's kind: run for a run or a compose, else the ask's own kind.
|
||||
func KindOf(ask sdkstep.StepAsk) string {
|
||||
if ask.Run != nil || ask.Compose != nil {
|
||||
return "run"
|
||||
}
|
||||
return ask.Kind
|
||||
}
|
||||
|
||||
// CallersFromDefinitions is a Puller.Callers: the seats the definition of language's build seat lists as callers of an
|
||||
// accepted kind, from the seat list's definitions (seatlist.Reader.Definitions). found is false when the definition or the
|
||||
// kind is not held yet. A caller entry is `seat:<name>`.
|
||||
func CallersFromDefinitions(language string, definitions func() map[string]seatdef.Definition) func(kind string) (callers []string, found bool) {
|
||||
return func(kind string) ([]string, bool) {
|
||||
seat, err := sdkstep.BuildSeat(language)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
def, ok := definitions()[seat]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
var out []string
|
||||
found := false
|
||||
for _, m := range def.Accepts {
|
||||
if m.Name != kind {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if m.Callers != nil {
|
||||
for _, e := range m.Callers.List {
|
||||
if name, ok := strings.CutPrefix(e, "seat:"); ok {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, found
|
||||
}
|
||||
}
|
||||
|
||||
// askerRefusal is why an ask must not be taken from its subject's asker: "" when it may be, and judged false when the seat
|
||||
// list does not say yet (the ask is given back, not refused).
|
||||
func (p *Puller) askerRefusal(msg *Pulled, ask sdkstep.StepAsk) (why string, judged bool) {
|
||||
if msg.Subject == "" {
|
||||
return "", true // a source that gives no subject (a test's)
|
||||
}
|
||||
_, _, asker, err := sdkstep.ParseAskSubjectBy(msg.Subject)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the ask came on %s, which is no accept subject of this seat", msg.Subject), true
|
||||
}
|
||||
kind := KindOf(ask)
|
||||
if asker == "" {
|
||||
if (ask.Build != nil && ask.Kind == sdkstep.AskDelivery) || kind == "run" {
|
||||
return "", true
|
||||
}
|
||||
return fmt.Sprintf("the kind %s is not taken on main's subject, which takes only a build and a run (asker: none named)", kind), true
|
||||
}
|
||||
if p.Callers == nil {
|
||||
return "", true
|
||||
}
|
||||
callers, found := p.Callers(kind)
|
||||
if !found {
|
||||
return "", false
|
||||
}
|
||||
if !slices.Contains(callers, asker) {
|
||||
return fmt.Sprintf("the kind %s is not one %s may ask: its callers are %s", kind, asker, strings.Join(callers, ", ")), true
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// refuseAsker ends an ask whose asker may not ask its kind: its step key says failed with the reason, naming the kind and the
|
||||
// asker, step-ended says failed, and the message is dropped.
|
||||
func (p *Puller) refuseAsker(msg *Pulled, ask sdkstep.StepAsk, why string) {
|
||||
key, err := sdkstep.AskStepKey(p.Node, ask.ID)
|
||||
if err == nil {
|
||||
at := now().UTC()
|
||||
step := sdkstep.Step{ID: ask.ID, Tool: ask.Kind + " ask", Node: p.Node, State: sdkstep.StepFailed, Started: at, Ended: at,
|
||||
Reason: "error: " + why, Owner: buildrun.Owner, Run: int(msg.Delivered)}
|
||||
err = writeEnd(p.Put, key, step)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the ask %s of the wrong asker cannot be answered on %s, so it is given back: %v", ask.ID, p.Node, err)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
p.logf("the ask %s is refused on %s: %s", ask.ID, p.Node, why)
|
||||
if ask.Step == nil {
|
||||
p.sayEnded(msg, ask, sdkstep.OutcomeFailed, why, nil)
|
||||
} else if end, err := sdkstep.EndOf(ask, p.Node, sdkstep.OutcomeFailed); err == nil {
|
||||
// A pipeline step's end names its pipeline, step, commit and fingerprint (its own form, §602).
|
||||
end.Said, end.At, end.Asker = why, now().UTC(), askerOf(msg, ask)
|
||||
if body, err := sdkstep.EncodeStepEnded(end); err == nil {
|
||||
p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID)
|
||||
}
|
||||
}
|
||||
_ = msg.Term()
|
||||
}
|
||||
|
||||
// giveBack hands an ask back to be judged again a little later, once the seat list says more.
|
||||
func (p *Puller) giveBack(msg *Pulled, ask sdkstep.StepAsk) {
|
||||
p.logf("the ask %s is given back on %s: the seat list holds no definition of its kind yet", ask.ID, p.Node)
|
||||
if msg.NakAfter != nil && msg.NakAfter(DefaultPullWait) == nil {
|
||||
return
|
||||
}
|
||||
_ = msg.Nak()
|
||||
}
|
||||
|
||||
// SeatListCallers is a Puller.Callers that follows the seat list: it starts the reader once the runtime serves
|
||||
// (seatlist.Start through the runtime's own way in) and answers from its definitions, found false until it has read them.
|
||||
// A reader that cannot start is said and the asks are given back until it can (design 47 G §543, A-163).
|
||||
func SeatListCallers(language string, logf func(format string, args ...any)) func(kind string) ([]string, bool) {
|
||||
var reader atomic.Pointer[seatlist.Reader]
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
r, err := seatlist.Start(seatlist.Config{Bus: seatlist.StdioBus(), Bootstrap: seatlist.Bootstrapped()})
|
||||
if err != nil && logf != nil {
|
||||
logf("following the seat list: %v", err)
|
||||
}
|
||||
reader.Store(r)
|
||||
}()
|
||||
return func(kind string) ([]string, bool) {
|
||||
r := reader.Load()
|
||||
if r == nil {
|
||||
return nil, false
|
||||
}
|
||||
return CallersFromDefinitions(language, r.Definitions)(kind)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package steprun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
func callersFor(table map[string][]string) func(string) ([]string, bool) {
|
||||
return func(kind string) ([]string, bool) { c, ok := table[kind]; return c, ok }
|
||||
}
|
||||
|
||||
// Design 47 G §543 (BG2, BG3, A-163, A-170): an ask whose kind its asker, the last token of its subject, is no caller of is
|
||||
// answered error naming the kind and the asker and ended failed; a step on main's subject is refused as well; one the
|
||||
// seat list does not judge yet is given back.
|
||||
func TestPullRefusesAKindItsAskerMayNotAsk(t *testing.T) {
|
||||
step := testAsk(sdkstep.AskStep)
|
||||
step.Check = nil
|
||||
step.Step = &sdkstep.StepRun{Pipeline: "p", Step: sdkstep.StepUnit, Module: "m", Artifact: "m:a", Commit: "0123456789abcdef0123456789abcdef01234567",
|
||||
Fingerprint: "fp", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: "0123456789abcdef0123456789abcdef01234567"}}
|
||||
build := testAsk(sdkstep.AskDelivery)
|
||||
table := map[string][]string{"step": {"mesh-pipeline-runner"}, "delivery": {"mesh-builds"}}
|
||||
for _, c := range []struct {
|
||||
name, subject string
|
||||
ask sdkstep.StepAsk
|
||||
callers func(string) ([]string, bool)
|
||||
want string // "" runs it, "refused" refuses it, "back" gives it back
|
||||
}{
|
||||
{"the indexer's step", "mesh.seat.node-build-go.accept.check.mesh-catalog", step, callersFor(table), "refused"},
|
||||
{"the runner's step", "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner", step, callersFor(table), ""},
|
||||
{"a step on main's subject", "mesh.seat.node-build-go.accept.check", step, callersFor(table), "refused"},
|
||||
{"a build on main's subject", "mesh.seat.node-build-go.accept.delivery", build, callersFor(table), ""},
|
||||
{"a kind the seat list does not hold", "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner", step, callersFor(nil), "back"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
src, st, sy := NewFakeAsks(), newSteps(), &sayer{}
|
||||
raw, _ := sdkstep.EncodeStepAsk(c.ask)
|
||||
src.PublishWithSubject(c.ask.Kind, raw, c.subject)
|
||||
ran := make(chan struct{}, 1)
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, Callers: c.callers,
|
||||
PullWait: 5 * time.Millisecond, Heartbeat: time.Hour, Say: sy.say, ProgressEvery: time.Hour,
|
||||
Run: func(context.Context, sdkstep.StepAsk) (any, error) { ran <- struct{}{}; return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
switch c.want {
|
||||
case "":
|
||||
<-ran
|
||||
case "refused":
|
||||
waitFor(t, "the ask ended", func() bool { _, termed, _ := src.Fates(); return len(termed) == 1 })
|
||||
key, _ := sdkstep.AskStepKey("shanks", c.ask.ID)
|
||||
s, _ := st.get(key)
|
||||
asker := c.subject[strings.LastIndex(c.subject, ".")+1:]
|
||||
if s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, KindOf(c.ask)) || (strings.Contains(c.subject, "mesh-catalog") && !strings.Contains(s.Reason, asker)) {
|
||||
t.Fatalf("the step = %+v, want failed naming the kind %s and the asker", s, KindOf(c.ask))
|
||||
}
|
||||
if end := sy.of("step-ended"); len(end) != 1 || end[0].body["outcome"] != sdkstep.OutcomeFailed {
|
||||
t.Fatalf("step-ended = %+v, want failed", end)
|
||||
}
|
||||
select {
|
||||
case <-ran:
|
||||
t.Fatal("a refused ask ran")
|
||||
default:
|
||||
}
|
||||
case "back":
|
||||
waitFor(t, "the ask given back", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return len(src.Delayed) > 0 })
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallersFromDefinitionsReadsTheSeatsCallers(t *testing.T) {
|
||||
def := seatdef.Definition{Seat: "node-build-go", Accepts: []seatdef.Message{
|
||||
{Name: "step", Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner"}}}}}
|
||||
callers := CallersFromDefinitions(sdkstep.AskGo, func() map[string]seatdef.Definition { return map[string]seatdef.Definition{"node-build-go": def} })
|
||||
if got, ok := callers("step"); !ok || len(got) != 1 || got[0] != "mesh-pipeline-runner" {
|
||||
t.Fatalf("callers of step = %v, %v", got, ok)
|
||||
}
|
||||
if _, ok := callers("run"); ok {
|
||||
t.Fatal("a kind the definition does not list is found")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package steprun
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// OutputsStore is the local name of the object store a build seat's holder keeps a run's answer in when it is over what
|
||||
// step-ended carries inline (design 47 G A-254), as the holder's manifest `requires` asks for it with the template objects.
|
||||
const OutputsStore = "outputs"
|
||||
|
||||
// OutputObjects is the object store of OutputsStore as a holder uses it (mesh-sdk busstate.ObjectsOf(OutputsStore)).
|
||||
type OutputObjects interface {
|
||||
Put(name string, data []byte, description string) (buscalls.ObjectInfo, error)
|
||||
Get(name string) (buscalls.ObjectData, error)
|
||||
}
|
||||
|
||||
// KeepOutputIn is a holder's Puller.KeepOutput over its store of outputs.
|
||||
func KeepOutputIn(o OutputObjects) func(object string, body []byte) error {
|
||||
return func(object string, body []byte) error {
|
||||
if o == nil {
|
||||
return fmt.Errorf("this module keeps no %s store", OutputsStore)
|
||||
}
|
||||
_, err := o.Put(object, body, "a run's answer, named by its step-ended")
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// ReadOutputFrom is a holder's read of a kept answer for OutputTool over its store of outputs.
|
||||
func ReadOutputFrom(o OutputObjects) func(object string) ([]byte, error) {
|
||||
return func(object string) ([]byte, error) {
|
||||
if o == nil {
|
||||
return nil, fmt.Errorf("this module keeps no %s store", OutputsStore)
|
||||
}
|
||||
got, err := o.Get(object)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the answer %s in the %s store: %w", object, OutputsStore, err)
|
||||
}
|
||||
return got.Data, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package steprun
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
type fakeOutputs struct{ m map[string][]byte }
|
||||
|
||||
func (f fakeOutputs) Put(name string, data []byte, _ string) (buscalls.ObjectInfo, error) {
|
||||
f.m[name] = data
|
||||
return buscalls.ObjectInfo{}, nil
|
||||
}
|
||||
func (f fakeOutputs) Get(name string) (buscalls.ObjectData, error) {
|
||||
d, ok := f.m[name]
|
||||
if !ok {
|
||||
return buscalls.ObjectData{}, errors.New("object not found")
|
||||
}
|
||||
return buscalls.ObjectData{Data: d}, nil
|
||||
}
|
||||
|
||||
// Design 47 G A-254: what a holder keeps through KeepOutputIn it reads again through ReadOutputFrom; with no store both refuse by name.
|
||||
func TestOutputsKeptAreRead(t *testing.T) {
|
||||
st := fakeOutputs{m: map[string][]byte{}}
|
||||
if err := KeepOutputIn(st)("output.a1", []byte("x")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := ReadOutputFrom(st)("output.a1"); err != nil || string(got) != "x" {
|
||||
t.Fatalf("read %q, %v", got, err)
|
||||
}
|
||||
if _, err := ReadOutputFrom(st)("output.none"); err == nil {
|
||||
t.Fatal("an absent answer was read")
|
||||
}
|
||||
if KeepOutputIn(nil)("o", nil) == nil {
|
||||
t.Fatal("no store kept an answer")
|
||||
}
|
||||
if _, err := ReadOutputFrom(nil)("o"); err == nil {
|
||||
t.Fatal("no store read an answer")
|
||||
}
|
||||
}
|
||||
@@ -65,3 +65,10 @@ func (r *phases) end() sdkstep.Step {
|
||||
r.ended = true
|
||||
return r.step
|
||||
}
|
||||
|
||||
// phase is the phase the ask is in now, "" before its first.
|
||||
func (r *phases) phase() string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.step.Phase
|
||||
}
|
||||
|
||||
@@ -27,6 +27,9 @@ type Pulled struct {
|
||||
// Kind is the worker it came from, filled by the puller.
|
||||
Kind string
|
||||
Data []byte
|
||||
// Subject is the subject the ask was published on: its last token is the asker the bus proved (design 47 G §481); empty
|
||||
// for a source that has none, and for main's form, which names none.
|
||||
Subject string
|
||||
// Delivered is how many times the bus has handed it over, this time included.
|
||||
Delivered uint64
|
||||
// Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its
|
||||
@@ -53,6 +56,8 @@ var (
|
||||
// DefaultHeartbeat is how often a running ask tells the bus it is still being worked; under the seat worker's
|
||||
// acknowledgement wait of 60 s (mesh-controller internal/broker HolderConsumerFor).
|
||||
DefaultHeartbeat = 20 * time.Second
|
||||
// DefaultProgressEvery is how often a running ask says step-progress (design 47 G §496: once a minute).
|
||||
DefaultProgressEvery = time.Minute
|
||||
)
|
||||
|
||||
// Puller pulls a build seat's asks and runs them.
|
||||
@@ -72,7 +77,19 @@ type Puller struct {
|
||||
// sdkstep.AgainAfter (a minute, then four). A test's own is short.
|
||||
Delay func(run int) time.Duration
|
||||
Heartbeat time.Duration
|
||||
Logf func(format string, args ...any)
|
||||
// Say publishes one event of the seat on this node's subject (SayOn): ask-taken when an ask starts, step-progress once
|
||||
// every ProgressEvery while it runs, step-ended when an ask outside a pipeline ends (design 47 G §496, §602). Nil says
|
||||
// none, a test's. ProgressEvery defaults to DefaultProgressEvery.
|
||||
Say func(event string, body any, id string) error
|
||||
// KeepOutput keeps a run's answer over what step-ended carries inline (sdkstep.StepEndedInlineMost) as the object named
|
||||
// object, which the seat's verb `output` answers (design 47 G A-254); a holder keeps it in the object store its manifest
|
||||
// requires. Nil keeps none: such an answer ends the ask failed, saying so.
|
||||
KeepOutput func(object string, body []byte) error
|
||||
// Callers says which seats the build seat's definition lists as callers of an accepted kind (CallersFromDefinitions);
|
||||
// found is false while the seat list holds no such kind. Nil checks no named asker (design 47 G §543, A-163).
|
||||
Callers func(kind string) (callers []string, found bool)
|
||||
ProgressEvery time.Duration
|
||||
Logf func(format string, args ...any)
|
||||
// Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull
|
||||
// reads the node's pause key (sdkstep.PauseKey) before it pulls, and Pause and Resume write it with PausePut, so a
|
||||
// pause outlives a restart.
|
||||
@@ -630,6 +647,13 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) {
|
||||
p.refuseUnread(msg, err)
|
||||
return
|
||||
}
|
||||
if why, judged := p.askerRefusal(msg, ask); !judged {
|
||||
p.giveBack(msg, ask)
|
||||
return
|
||||
} else if why != "" {
|
||||
p.refuseAsker(msg, ask, why)
|
||||
return
|
||||
}
|
||||
if d := p.droppedAsk(ask.ID); d != nil {
|
||||
p.refuseDropped(msg, ask, d)
|
||||
return
|
||||
@@ -665,6 +689,7 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) {
|
||||
p.running[ask.ID] = RunningAsk{ID: ask.ID, Kind: ask.Kind, Module: module, Started: step.Started}
|
||||
p.mu.Unlock()
|
||||
p.logf("the %s ask %s starts on %s: %s", ask.Kind, ask.ID, p.Node, step.What)
|
||||
p.say("ask-taken", askTaken{Ask: ask.ID, Asker: askerOf(msg, ask), Node: p.Node, At: now().UTC()}, "ask-taken."+ask.ID)
|
||||
go p.run(ctx, msg, ask, key, step)
|
||||
}
|
||||
|
||||
@@ -696,6 +721,25 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key
|
||||
}
|
||||
}()
|
||||
ph := &phases{put: p.Put, key: key, step: step, logf: p.Logf, prefix: fmt.Sprintf("the %s ask %s on %s", ask.Kind, ask.ID, p.Node)}
|
||||
if p.Say != nil {
|
||||
every := p.ProgressEvery
|
||||
if every <= 0 {
|
||||
every = DefaultProgressEvery
|
||||
}
|
||||
go func() {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-t.C:
|
||||
p.say("step-progress", stepProgress{Ask: ask.ID, Asker: askerOf(msg, ask), Phase: progressPhase(ask, ph.phase()), At: now().UTC()},
|
||||
fmt.Sprintf("step-progress.%s.%d", ask.ID, now().UnixNano()))
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
runCtx := p.cancellable(ctx, ask.ID)
|
||||
result, err := protect(runCtx, func(ctx context.Context) (any, error) {
|
||||
return p.Run(context.WithValue(ctx, phaseKey{}, ph), ask)
|
||||
@@ -713,6 +757,7 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key
|
||||
var dec *Declining
|
||||
if errors.As(err, &dec) && ctx.Err() == nil {
|
||||
p.declineRun(msg, ask, key, step, dec)
|
||||
p.sayEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text, nil)
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
@@ -750,6 +795,9 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key
|
||||
if len(outcome) > 300 {
|
||||
outcome = outcome[:300] + "..."
|
||||
}
|
||||
p.sayEnded(msg, ask, sdkstep.OutcomeFailed, err.Error(), nil)
|
||||
} else {
|
||||
p.sayEnded(msg, ask, sdkstep.OutcomePassed, "", result)
|
||||
}
|
||||
p.logf("the %s ask %s ended on %s after %s, in its phase %q; %s", ask.Kind, ask.ID, p.Node,
|
||||
now().Sub(step.Started).Round(time.Second), step.Phase, outcome)
|
||||
@@ -884,3 +932,103 @@ func askRead(ask sdkstep.StepAsk) sdkstep.Ask {
|
||||
out, _ := sdkstep.AskAs[sdkstep.Ask](ask)
|
||||
return out
|
||||
}
|
||||
|
||||
// askTaken and stepProgress are the bodies of the seat's events ask-taken and step-progress (design 47 G §602, §496).
|
||||
type askTaken struct {
|
||||
Ask string `json:"ask"`
|
||||
Asker string `json:"asker"`
|
||||
Node string `json:"node"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
type stepProgress struct {
|
||||
Ask string `json:"ask"`
|
||||
Asker string `json:"asker"`
|
||||
Phase string `json:"phase"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// askerOf is the asker the bus proved: the last token of the subject the ask came on (design 47 G §481), the ask's own
|
||||
// field only for a source that gives no subject and for main's form.
|
||||
func askerOf(msg *Pulled, ask sdkstep.StepAsk) string {
|
||||
if _, _, asker, err := sdkstep.ParseAskSubjectBy(msg.Subject); err == nil && asker != "" {
|
||||
return asker
|
||||
}
|
||||
return ask.Asker
|
||||
}
|
||||
|
||||
// progressPhase is the phase step-progress says (fetch, build, run or report): fetch until the ask has entered a phase of
|
||||
// its own or while it names the fetch, then the ask's kind (build for a build, run for any other).
|
||||
func progressPhase(ask sdkstep.StepAsk, phase string) string {
|
||||
switch {
|
||||
case phase == "" || strings.HasPrefix(phase, "fetch"):
|
||||
return "fetch"
|
||||
case ask.Build != nil:
|
||||
return "build"
|
||||
}
|
||||
return "run"
|
||||
}
|
||||
|
||||
// say publishes one event of the seat; a failure is logged and never fails the ask.
|
||||
func (p *Puller) say(event string, body any, id string) {
|
||||
if p.Say == nil {
|
||||
return
|
||||
}
|
||||
if err := p.Say(event, body, id); err != nil {
|
||||
p.logf("the event %s of the ask %s could not be said on %s: %v", event, id, p.Node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// sayEnded says step-ended for an ask outside a pipeline, whatever its asker (design 47 G §602): a pipeline step's end is
|
||||
// its run's own (AskSteps.SayStep). The end of a run carries the run's answer (A-254): the last line of its program's
|
||||
// standard output, inline as `output` up to sdkstep.StepEndedInlineMost, else as `output-object`, kept by KeepOutput; a
|
||||
// program that exits other than 0 ends failed with its answer carried.
|
||||
func (p *Puller) sayEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string, result any) {
|
||||
if p.Say == nil || ask.Step != nil {
|
||||
return
|
||||
}
|
||||
end := sdkstep.StepEnded{ID: ask.ID, Asker: askerOf(msg, ask), Ask: ask.ID, Node: p.Node, Outcome: outcome, At: now().UTC()}
|
||||
if ans, ok := runAnswerOf(ask, result); ok {
|
||||
if ans.Exit != 0 && outcome == sdkstep.OutcomePassed {
|
||||
outcome, said = sdkstep.OutcomeFailed, fmt.Sprintf("the program exited %d: %s", ans.Exit, ans.Said)
|
||||
}
|
||||
if len(ans.Output) <= sdkstep.StepEndedInlineMost {
|
||||
end.Output = string(ans.Output)
|
||||
} else if object := sdkstep.OutputObjectName(ask.ID); p.KeepOutput == nil {
|
||||
outcome, said = sdkstep.OutcomeFailed, fmt.Sprintf("the run's answer is %d bytes, over what the end carries, and this holder keeps no object for it", len(ans.Output))
|
||||
} else if err := p.KeepOutput(object, ans.Output); err != nil {
|
||||
outcome, said = sdkstep.OutcomeFailed, fmt.Sprintf("the run's answer of %d bytes could not be kept: %v", len(ans.Output), err)
|
||||
} else {
|
||||
end.OutputObject = object
|
||||
}
|
||||
}
|
||||
if len(said) > 300 {
|
||||
said = said[:300] + "..."
|
||||
}
|
||||
if said == "" && outcome != sdkstep.OutcomePassed {
|
||||
said = "the ask did not pass"
|
||||
}
|
||||
end.Outcome, end.Said = outcome, said
|
||||
body, err := sdkstep.EncodeStepEnded(end)
|
||||
if err != nil {
|
||||
p.logf("the end of the ask %s cannot be said on %s: %v", ask.ID, p.Node, err)
|
||||
return
|
||||
}
|
||||
p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID)
|
||||
}
|
||||
|
||||
// runAnswerOf is the run answer a run ask's step handed back, whatever its form (the value or a pointer).
|
||||
func runAnswerOf(ask sdkstep.StepAsk, result any) (sdkstep.RunAnswer, bool) {
|
||||
if ask.Run == nil {
|
||||
return sdkstep.RunAnswer{}, false
|
||||
}
|
||||
switch a := result.(type) {
|
||||
case sdkstep.RunAnswer:
|
||||
return a, true
|
||||
case *sdkstep.RunAnswer:
|
||||
if a != nil {
|
||||
return *a, true
|
||||
}
|
||||
}
|
||||
return sdkstep.RunAnswer{}, false
|
||||
}
|
||||
|
||||
@@ -16,14 +16,19 @@ type JetStreamAsks struct {
|
||||
JS jetstream.JetStream
|
||||
Language string
|
||||
|
||||
mu sync.Mutex
|
||||
cons map[string]jetstream.Consumer
|
||||
mu sync.Mutex
|
||||
cons map[string]jetstream.Consumer
|
||||
missing map[string]bool
|
||||
}
|
||||
|
||||
func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Consumer, error) {
|
||||
func (s *JetStreamAsks) consumer(ctx context.Context, kind string, named bool) (jetstream.Consumer, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if c, ok := s.cons[kind]; ok {
|
||||
cacheKey := kind
|
||||
if named {
|
||||
cacheKey = kind + "/named"
|
||||
}
|
||||
if c, ok := s.cons[cacheKey]; ok {
|
||||
return c, nil
|
||||
}
|
||||
stream, err := sdkstep.AskStream(s.Language)
|
||||
@@ -31,6 +36,9 @@ func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Co
|
||||
return nil, err
|
||||
}
|
||||
worker, err := sdkstep.AskWorker(s.Language, kind)
|
||||
if named {
|
||||
worker, err = sdkstep.AskNamedWorker(s.Language, kind)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -41,17 +49,60 @@ func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Co
|
||||
if s.cons == nil {
|
||||
s.cons = map[string]jetstream.Consumer{}
|
||||
}
|
||||
s.cons[kind] = c
|
||||
s.cons[cacheKey] = c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Next is AskSource's: a fetch of one with no wait when wait is 0, else one that waits at most wait.
|
||||
// Next is AskSource's: main's worker first, then the queue's named worker once it is bound (design 47 G §544), with no
|
||||
// wait when wait is 0, else waiting at most wait between the two. A named worker not bound is only main's alone.
|
||||
func (s *JetStreamAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
|
||||
c, err := s.consumer(ctx, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var half time.Duration
|
||||
if wait > 0 {
|
||||
half = wait / 2
|
||||
}
|
||||
for i, step := range []struct {
|
||||
named bool
|
||||
wait time.Duration
|
||||
}{{false, 0}, {true, 0}, {false, half}, {true, wait - half}} {
|
||||
if step.named && s.unbound(kind) {
|
||||
continue
|
||||
}
|
||||
if i >= 2 && wait <= 0 {
|
||||
break // the two waiting pulls are for a wait
|
||||
}
|
||||
c, err := s.consumer(ctx, kind, step.named)
|
||||
if err != nil {
|
||||
if step.named {
|
||||
s.markUnbound(kind)
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if msg, err := s.fetch(ctx, c, step.wait); err != nil || msg != nil {
|
||||
return msg, err
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *JetStreamAsks) unbound(kind string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.missing[kind]
|
||||
}
|
||||
|
||||
func (s *JetStreamAsks) markUnbound(kind string) {
|
||||
s.mu.Lock()
|
||||
if s.missing == nil {
|
||||
s.missing = map[string]bool{}
|
||||
}
|
||||
s.missing[kind] = true
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *JetStreamAsks) fetch(ctx context.Context, c jetstream.Consumer, wait time.Duration) (*Pulled, error) {
|
||||
var batch jetstream.MessageBatch
|
||||
var err error
|
||||
if wait <= 0 {
|
||||
batch, err = c.FetchNoWait(1)
|
||||
} else {
|
||||
@@ -69,7 +120,7 @@ func (s *JetStreamAsks) Next(ctx context.Context, kind string, wait time.Duratio
|
||||
if meta, err := msg.Metadata(); err == nil && meta != nil {
|
||||
delivered = meta.NumDelivered
|
||||
}
|
||||
return &Pulled{Data: append([]byte(nil), msg.Data()...), Delivered: delivered,
|
||||
return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered,
|
||||
Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term,
|
||||
NakAfter: func(delay time.Duration) error { return msg.NakWithDelay(delay) }}, nil
|
||||
case <-ctx.Done():
|
||||
@@ -93,11 +144,16 @@ func DeclareAskWorkers(ctx context.Context, js jetstream.JetStream, language str
|
||||
for _, kind := range sdkstep.AskKinds {
|
||||
worker, _ := sdkstep.AskWorker(language, kind)
|
||||
subject, _ := sdkstep.AskSubject(language, kind)
|
||||
// Both forms of the accept subject: main's and the asker's, `<main's>.<asker>` (design 47 G §600).
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject, subject + ".*"},
|
||||
// Two workers whose filters do not overlap: main's subject and the asker's, `<main's>.<asker>` (design 47 G §544).
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject},
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil {
|
||||
return fmt.Errorf("the worker %s: %w", worker, err)
|
||||
}
|
||||
named, _ := sdkstep.AskNamedWorker(language, kind)
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: named, FilterSubjects: []string{subject + ".*"},
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil {
|
||||
return fmt.Errorf("the worker %s: %w", named, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package steprun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
type said struct {
|
||||
event, id string
|
||||
body map[string]any
|
||||
}
|
||||
|
||||
type sayer struct {
|
||||
mu sync.Mutex
|
||||
got []said
|
||||
}
|
||||
|
||||
func (s *sayer) say(event string, body any, id string) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
var m map[string]any
|
||||
_ = json.Unmarshal(raw, &m)
|
||||
s.mu.Lock()
|
||||
s.got = append(s.got, said{event, id, m})
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sayer) of(event string) []said {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []said
|
||||
for _, g := range s.got {
|
||||
if g.event == event {
|
||||
out = append(out, g)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Design 47 G §496, §602 (BG4, BN4, BI4): a holder says ask-taken when an ask starts, step-progress while it runs and
|
||||
// step-ended when an ask outside a pipeline ends, whatever its asker.
|
||||
func TestPullSaysAskTakenProgressAndEnd(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
runErr error
|
||||
outcome string
|
||||
}{{"passed", nil, sdkstep.OutcomePassed}, {"failed", errors.New("the build broke"), sdkstep.OutcomeFailed}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
src, st, sy := NewFakeAsks(), newSteps(), &sayer{}
|
||||
a := testAsk(sdkstep.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
release, entered := make(chan struct{}), make(chan struct{})
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Say: sy.say, ProgressEvery: 5 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) {
|
||||
Phase(ctx, "build m")
|
||||
close(entered)
|
||||
<-release
|
||||
return nil, c.runErr
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
<-entered
|
||||
waitFor(t, "a step-progress in the build", func() bool {
|
||||
pr := sy.of("step-progress")
|
||||
return len(pr) > 0 && pr[len(pr)-1].body["phase"] == "build"
|
||||
})
|
||||
close(release)
|
||||
waitFor(t, "step-ended", func() bool { return len(sy.of("step-ended")) == 1 })
|
||||
taken := sy.of("ask-taken")
|
||||
if len(taken) != 1 || taken[0].body["ask"] != a.ID || taken[0].body["node"] != "shanks" || taken[0].id != "ask-taken."+a.ID {
|
||||
t.Fatalf("ask-taken = %+v, want one for %s on shanks", taken, a.ID)
|
||||
}
|
||||
if pr := sy.of("step-progress")[len(sy.of("step-progress"))-1].body; pr["ask"] != a.ID || pr["phase"] != "build" || pr["asker"] != a.Asker {
|
||||
t.Fatalf("step-progress = %v, want the ask, its asker and the phase build", pr)
|
||||
}
|
||||
end := sy.of("step-ended")[0].body
|
||||
if end["outcome"] != c.outcome || end["ask"] != a.ID || end["asker"] != a.Asker || end["node"] != "shanks" || end["at"] == nil {
|
||||
t.Fatalf("step-ended = %v, want outcome %s of ask %s", end, c.outcome, a.ID)
|
||||
}
|
||||
if c.runErr != nil && end["said"] != "the build broke" {
|
||||
t.Fatalf("step-ended said %v, want why", end["said"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,9 @@ import (
|
||||
type FakeAsks struct {
|
||||
mu sync.Mutex
|
||||
waiting map[string][][]byte
|
||||
added chan struct{}
|
||||
// subjects are the subjects asks were published with, by their data.
|
||||
subjects map[string]string
|
||||
added chan struct{}
|
||||
// Acked, Termed and Naked are the asks' ids or data by fate, InProgress how many heartbeats were given.
|
||||
Acked, Termed, Naked []string
|
||||
// Delayed are the asks handed back to be given again after a delay (NakAfter), and Delays that delay of each.
|
||||
@@ -40,6 +42,17 @@ func (f *FakeAsks) Publish(kind string, data []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// PublishWithSubject puts an encoded ask on kind's worker as the bus would hand it over from subject (design 47 G §481).
|
||||
func (f *FakeAsks) PublishWithSubject(kind string, data []byte, subject string) {
|
||||
f.mu.Lock()
|
||||
if f.subjects == nil {
|
||||
f.subjects = map[string]string{}
|
||||
}
|
||||
f.subjects[string(data)] = subject
|
||||
f.mu.Unlock()
|
||||
f.Publish(kind, data)
|
||||
}
|
||||
|
||||
// PublishAsk encodes a and puts it on its kind's worker.
|
||||
func (f *FakeAsks) PublishAsk(a sdkstep.StepAsk) error {
|
||||
raw, err := sdkstep.EncodeStepAsk(a)
|
||||
@@ -107,7 +120,10 @@ func (f *FakeAsks) pulled(data []byte) *Pulled {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
f.mu.Lock()
|
||||
subject := f.subjects[string(data)]
|
||||
delivered := f.Delivered
|
||||
f.mu.Unlock()
|
||||
if delivered == 0 {
|
||||
delivered = 1
|
||||
}
|
||||
@@ -117,6 +133,6 @@ func (f *FakeAsks) pulled(data []byte) *Pulled {
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
return &Pulled{Data: data, Delivered: delivered, NakAfter: nakAfter, Ack: fate(&f.Acked), Nak: fate(&f.Naked), Term: fate(&f.Termed),
|
||||
return &Pulled{Data: data, Subject: subject, Delivered: delivered, NakAfter: nakAfter, Ack: fate(&f.Acked), Nak: fate(&f.Naked), Term: fate(&f.Termed),
|
||||
InProgress: func() error { f.mu.Lock(); f.InProgress++; f.mu.Unlock(); return nil }}
|
||||
}
|
||||
|
||||
@@ -40,7 +40,11 @@ func publishAsk(t *testing.T, js jetstream.JetStream, a sdkstep.StepAsk) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A build rides main's subject; any other kind a named asker's (design 47 G §543: main takes only a build and a run).
|
||||
subject, _ := sdkstep.AskSubject(a.Language, a.Kind)
|
||||
if a.Kind != sdkstep.AskDelivery {
|
||||
subject, _ = sdkstep.AskSubjectBy(a.Language, a.Kind, "mesh-delivery-checker")
|
||||
}
|
||||
if _, err := js.Publish(context.Background(), subject, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -197,31 +201,35 @@ func TestAnAskThatMetTheNetworkIsGivenAgainAndADroppedOneIsEnded(t *testing.T) {
|
||||
|
||||
func contains(s, sub string) bool { return strings.Contains(s, sub) }
|
||||
|
||||
// Design 47 G §600: the seat's worker takes an ask put on either form of the accept subject, main's and the asker's.
|
||||
func TestTheWorkerTakesBothFormsOfTheAcceptSubject(t *testing.T) {
|
||||
// Design 47 G §544: main's worker takes an ask put on main's subject and the named worker one put on the asker's, the two
|
||||
// filters not overlapping.
|
||||
func TestTheTwoWorkersTakeTheTwoFormsOfTheAcceptSubject(t *testing.T) {
|
||||
js := askBus(t, 30*time.Second)
|
||||
stream, _ := sdkstep.AskStream(sdkstep.AskGo)
|
||||
worker, _ := sdkstep.AskWorker(sdkstep.AskGo, sdkstep.AskDelivery)
|
||||
named, _ := sdkstep.AskNamedWorker(sdkstep.AskGo, sdkstep.AskDelivery)
|
||||
main, _ := sdkstep.AskSubject(sdkstep.AskGo, sdkstep.AskDelivery)
|
||||
for _, subject := range []string{main, main + ".builder"} {
|
||||
for _, subject := range []string{main, main + ".mesh-builds"} {
|
||||
if _, err := js.Publish(context.Background(), subject, []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
c, err := js.Consumer(context.Background(), stream, worker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := c.Fetch(2, jetstream.FetchMaxWait(3*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for m := range batch.Messages() {
|
||||
got = append(got, m.Subject())
|
||||
_ = m.Ack()
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Errorf("the worker took %v, want both forms of %s", got, main)
|
||||
for name, want := range map[string]string{worker: main, named: main + ".mesh-builds"} {
|
||||
c, err := js.Consumer(context.Background(), stream, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := c.Fetch(2, jetstream.FetchMaxWait(time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for m := range batch.Messages() {
|
||||
got = append(got, m.Subject())
|
||||
_ = m.Ack()
|
||||
}
|
||||
if len(got) != 1 || got[0] != want {
|
||||
t.Errorf("%s took %v, want only %s", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package steprun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
@@ -18,10 +19,15 @@ type SeatAsks struct {
|
||||
Settle func(id, fate string) error
|
||||
// SettleAfter is stdio.SeatSettleAfter when nil (ADR 0664 §1).
|
||||
SettleAfter func(id string, delay time.Duration) error
|
||||
// Logf says `named worker not bound` once for each queue whose named worker the membership does not list yet.
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
noted map[string]bool
|
||||
}
|
||||
|
||||
// Next is AskSource's.
|
||||
func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
|
||||
func (s *SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
|
||||
pull, settle := s.Pull, s.Settle
|
||||
if pull == nil {
|
||||
pull = stdio.SeatPull
|
||||
@@ -36,7 +42,33 @@ func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*P
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := pull(worker, wait)
|
||||
named, err := sdkstep.AskNamedWorker(s.Language, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Main's worker first, then the named one (design 47 G §544): a holder of G pulls main's alone until its membership
|
||||
// names the named worker, and says so meanwhile. A wait is shared by the two.
|
||||
half := wait / 2
|
||||
p, err := pull(worker, min(wait, 0))
|
||||
if err == nil && p == nil {
|
||||
var nerr error
|
||||
if p, nerr = pull(named, 0); nerr != nil {
|
||||
s.noteUnbound(kind, nerr)
|
||||
p = nil
|
||||
} else {
|
||||
s.mu.Lock()
|
||||
delete(s.noted, kind)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
if err == nil && p == nil && wait > 0 {
|
||||
if p, err = pull(worker, half); err == nil && p == nil {
|
||||
if p, err = pull(named, wait-half); err != nil {
|
||||
s.noteUnbound(kind, err)
|
||||
p, err = nil, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil || p == nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -45,7 +77,21 @@ func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*P
|
||||
after = stdio.SeatSettleAfter
|
||||
}
|
||||
fate := func(f string) func() error { return func() error { return settle(p.ID, f) } }
|
||||
return &Pulled{Data: p.Body, Delivered: p.Delivered, Ack: fate(stdio.SettleAck), InProgress: fate(stdio.SettleProgress),
|
||||
return &Pulled{Data: p.Body, Subject: p.Subject, Delivered: p.Delivered, Ack: fate(stdio.SettleAck), InProgress: fate(stdio.SettleProgress),
|
||||
Nak: fate(stdio.SettleNak), Term: fate(stdio.SettleTerm),
|
||||
NakAfter: func(delay time.Duration) error { return after(p.ID, delay) }}, nil
|
||||
}
|
||||
|
||||
// noteUnbound says `named worker not bound` for a queue, once until it is bound (design 47 G §544).
|
||||
func (s *SeatAsks) noteUnbound(kind string, err error) {
|
||||
s.mu.Lock()
|
||||
if s.noted == nil {
|
||||
s.noted = map[string]bool{}
|
||||
}
|
||||
said := s.noted[kind]
|
||||
s.noted[kind] = true
|
||||
s.mu.Unlock()
|
||||
if !said && s.Logf != nil {
|
||||
s.Logf("named worker not bound: the %s queue's named asks wait until the membership lists %s: %v", kind, "its named worker", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
func TestSeatAsksPullsTheKindsWorkerAndSettlesByID(t *testing.T) {
|
||||
var pulled []string
|
||||
var settled []string
|
||||
src := SeatAsks{Language: sdkstep.AskGo,
|
||||
src := &SeatAsks{Language: sdkstep.AskGo,
|
||||
Pull: func(worker string, wait time.Duration) (*stdio.Pulled, error) {
|
||||
pulled = append(pulled, worker)
|
||||
if worker != "SEAT_NODE_BUILD_GO_CHECK_worker" {
|
||||
@@ -36,7 +36,7 @@ func TestSeatAsksPullsTheKindsWorkerAndSettlesByID(t *testing.T) {
|
||||
}
|
||||
_ = p.InProgress()
|
||||
_ = p.Ack()
|
||||
if len(pulled) != 2 || pulled[0] != "SEAT_NODE_BUILD_GO_DELIVERY_worker" {
|
||||
if len(pulled) != 3 || pulled[0] != "SEAT_NODE_BUILD_GO_DELIVERY_worker" || pulled[1] != "SEAT_NODE_BUILD_GO_DELIVERY_named_worker" {
|
||||
t.Errorf("pulled %v", pulled)
|
||||
}
|
||||
if len(settled) != 2 || settled[0] != "p9 progress" || settled[1] != "p9 ack" {
|
||||
@@ -146,3 +146,34 @@ func TestADroppedAskIsRefusedAtPullAndARunningOneIsEnded(t *testing.T) {
|
||||
}
|
||||
|
||||
func ptr[T any](v T) *T { return &v }
|
||||
|
||||
// Design 47 G §544 (BG3): a holder pulls main's worker, then the queue's named worker; while its membership does not list the
|
||||
// named worker it takes main's alone and says `named worker not bound` once.
|
||||
func TestSeatAsksPullsTheNamedWorkerAndSaysWhenItIsNotBound(t *testing.T) {
|
||||
var said []string
|
||||
bound := false
|
||||
src := &SeatAsks{Language: sdkstep.AskGo, Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) },
|
||||
Pull: func(worker string, wait time.Duration) (*stdio.Pulled, error) {
|
||||
if worker != "SEAT_NODE_BUILD_GO_CHECK_named_worker" {
|
||||
return nil, nil
|
||||
}
|
||||
if !bound {
|
||||
return nil, fmt.Errorf("the module pulls no work from %s", worker)
|
||||
}
|
||||
return &stdio.Pulled{ID: "n1", Worker: worker, Subject: "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner", Body: []byte(`{"v":1}`), Delivered: 1}, nil
|
||||
},
|
||||
Settle: func(string, string) error { return nil }}
|
||||
for range 2 {
|
||||
if p, err := src.Next(context.Background(), sdkstep.AskCheck, 0); err != nil || p != nil {
|
||||
t.Fatalf("unbound: %+v, %v", p, err)
|
||||
}
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "named worker not bound") {
|
||||
t.Fatalf("said %q, want `named worker not bound` once", said)
|
||||
}
|
||||
bound = true
|
||||
p, err := src.Next(context.Background(), sdkstep.AskCheck, 0)
|
||||
if err != nil || p == nil || !strings.HasSuffix(p.Subject, ".mesh-pipeline-runner") {
|
||||
t.Fatalf("bound: %+v, %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package steprun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// Design 47 G A-254: a build seat's step-ended of a run carries the run's answer, the last line of its program's standard output:
|
||||
// inline as `output` up to the event bound, else as `output-object`, the object the holder keeps, which the verb `output` answers.
|
||||
func TestStepEndedOfARunCarriesItsAnswer(t *testing.T) {
|
||||
big := `"` + strings.Repeat("x", sdkstep.StepEndedInlineMost) + `"`
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
answer sdkstep.RunAnswer
|
||||
keepErr error
|
||||
outcome string
|
||||
inline string
|
||||
objectKept bool
|
||||
said string
|
||||
}{
|
||||
{name: "a small answer is inline", answer: sdkstep.RunAnswer{Output: json.RawMessage(`{"verdict":true}`)}, outcome: sdkstep.OutcomePassed, inline: `{"verdict":true}`},
|
||||
{name: "an answer over the event bound is an object", answer: sdkstep.RunAnswer{Output: json.RawMessage(big)}, outcome: sdkstep.OutcomePassed, objectKept: true},
|
||||
{name: "a program that exits other than 0 failed, its answer kept", answer: sdkstep.RunAnswer{Exit: 3, Output: json.RawMessage(`"no"`), Said: "bad input"},
|
||||
outcome: sdkstep.OutcomeFailed, inline: `"no"`, said: "the program exited 3: bad input"},
|
||||
{name: "an answer the holder cannot keep is said", answer: sdkstep.RunAnswer{Output: json.RawMessage(big)}, keepErr: errors.New("store full"),
|
||||
outcome: sdkstep.OutcomeFailed, said: "store full"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
src, st, sy := NewFakeAsks(), newSteps(), &sayer{}
|
||||
a := testAsk(sdkstep.AskCheck)
|
||||
a.Check = nil
|
||||
in, _ := sdkstep.RunInputOf("seat:mesh-catalog.input", []byte(`{"v":1}`))
|
||||
commit := "0123456789abcdef0123456789abcdef01234567"
|
||||
a.Run = &sdkstep.RunJob{Program: sdkstep.RunProgram{Repository: "novox/mesh-delivery", Commit: commit, Dir: "modules/x", Package: "./cmd/x"}, Input: in}
|
||||
_ = src.PublishAsk(a)
|
||||
var mu sync.Mutex
|
||||
kept := map[string][]byte{}
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Say: sy.say,
|
||||
KeepOutput: func(object string, body []byte) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if c.keepErr != nil {
|
||||
return c.keepErr
|
||||
}
|
||||
kept[object] = body
|
||||
return nil
|
||||
},
|
||||
Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { return c.answer, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "step-ended", func() bool { return len(sy.of("step-ended")) == 1 })
|
||||
end := sy.of("step-ended")[0].body
|
||||
if end["outcome"] != c.outcome {
|
||||
t.Fatalf("outcome %v, want %s (%v)", end["outcome"], c.outcome, end)
|
||||
}
|
||||
if c.inline != "" && end["output"] != c.inline {
|
||||
t.Fatalf("output %v, want %s inline", end["output"], c.inline)
|
||||
}
|
||||
if c.said != "" && !strings.Contains(end["said"].(string), c.said) {
|
||||
t.Fatalf("said %v, want it to hold %q", end["said"], c.said)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if c.objectKept {
|
||||
name := sdkstep.OutputObjectName(a.ID)
|
||||
if end["output-object"] != name || end["output"] != nil || string(kept[name]) != big {
|
||||
t.Fatalf("end %v and kept %d bytes, want the object %s holding the whole answer and no inline output", end["output-object"], len(kept[name]), name)
|
||||
}
|
||||
} else if end["output-object"] != nil || len(kept) != 0 {
|
||||
t.Fatalf("an object %v was named or kept (%d), want none", end["output-object"], len(kept))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
package steprun
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
@@ -50,3 +54,29 @@ func SeatTools(seat, what string, p *Puller) []stdio.Tool {
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// OutputTool serves `<seat>.output` (design 47 G A-254, A-257): a run's answer by its ask, as the object the holder keeps it in
|
||||
// (sdkstep.OutputObjectName), read with read, which answers nil for none. The verb's answer is marked chunks: the bundle answers
|
||||
// the whole answer as the member `file` (base64) and the tool runner sends it in chunks and answers their count; an ask
|
||||
// whose answer this node does not keep answers no chunks.
|
||||
func OutputTool(seat string, read func(object string) ([]byte, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: seat + ".output",
|
||||
Description: "A run's answer by its ask, when its step-ended named it as an output-object: the answer's bytes in chunks, of any size.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"ask", "call"}, "properties": map[string]any{
|
||||
"ask": map[string]any{"type": "string", "description": "the ask's id"},
|
||||
"call": map[string]any{"type": "string", "description": "the subject token of this send's chunks"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ask, _ := args["ask"].(string)
|
||||
if ask == "" {
|
||||
return nil, errors.New("the verb output is asked for no ask")
|
||||
}
|
||||
body, err := read(sdkstep.OutputObjectName(ask))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if body == nil {
|
||||
return map[string]any{"ask": ask, "chunks": 0}, nil
|
||||
}
|
||||
return map[string]any{"ask": ask, "file": base64.StdEncoding.EncodeToString(body)}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -50,3 +50,27 @@ func TestSeatToolsCurrentPauseResume(t *testing.T) {
|
||||
}
|
||||
waitFor(t, "the ask taken after the resume", func() bool { return src.Waiting(sdkstep.AskDelivery) == 0 })
|
||||
}
|
||||
|
||||
// Design 47 G A-254: `<seat>.output` answers a run's answer by its ask as the whole file (base64), which the tool runner sends in
|
||||
// chunks (A-257); an ask this node keeps no answer of answers no chunks.
|
||||
func TestOutputToolAnswersTheKeptAnswer(t *testing.T) {
|
||||
kept := map[string][]byte{sdkstep.OutputObjectName("a1"): []byte(`{"verdict":"x"}`)}
|
||||
tool := OutputTool("node-build-go", func(object string) ([]byte, error) { return kept[object], nil })
|
||||
if tool.Name != "node-build-go.output" {
|
||||
t.Fatalf("name %q", tool.Name)
|
||||
}
|
||||
got, err := tool.Run(map[string]any{"ask": "a1", "call": "c"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := got.(map[string]any); m["ask"] != "a1" || m["file"] != "eyJ2ZXJkaWN0IjoieCJ9" {
|
||||
t.Fatalf("answer %v, want the ask and its answer's bytes as the file", got)
|
||||
}
|
||||
got, _ = tool.Run(map[string]any{"ask": "none", "call": "c"})
|
||||
if m := got.(map[string]any); m["ask"] != "none" || m["chunks"] != 0 || m["file"] != nil {
|
||||
t.Fatalf("answer %v, want no chunks and no file", got)
|
||||
}
|
||||
if _, err := tool.Run(map[string]any{"call": "c"}); err == nil {
|
||||
t.Fatal("a call with no ask was answered")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -250,3 +250,15 @@ func SayStepOn(language, node string, publish func(subject string, body any, id
|
||||
return publish(subject, json.RawMessage(body), "step-ended."+end.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// SayOn is a holder's Puller.Say: an event of the language's build seat published on that seat's subject of node with
|
||||
// publish (stdio.SeatPublish), `mesh.seat.node-build-<language>.event.<event>.<node>`.
|
||||
func SayOn(language, node string, publish func(subject string, body any, id string) error) func(event string, body any, id string) error {
|
||||
return func(event string, body any, id string) error {
|
||||
seat, err := sdkstep.BuildSeat(language)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return publish("mesh.seat."+seat+".event."+event+"."+node, body, id)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user