You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
534 lines
14 KiB
534 lines
14 KiB
package org.wikimedia.integration
|
|
|
|
import com.cloudbees.groovy.cps.NonCPS
|
|
|
|
import static org.wikimedia.integration.Utility.timestampLabel
|
|
|
|
import org.wikimedia.integration.ExecutionContext
|
|
import org.wikimedia.integration.PatchSet
|
|
import org.wikimedia.integration.Pipeline
|
|
|
|
class PipelineStage implements Serializable {
|
|
static final String SETUP = 'setup'
|
|
static final String TEARDOWN = 'teardown'
|
|
static final List STEPS = ['build', 'run', 'publish', 'deploy', 'exports']
|
|
|
|
|
|
Pipeline pipeline
|
|
String name
|
|
Map config
|
|
|
|
private def context
|
|
|
|
/**
|
|
* Returns an config based on the given one but with default values
|
|
* inserted.
|
|
*
|
|
* @example Shorthand stage config (providing only a stage name)
|
|
* <pre><code>
|
|
* def cfg = [name: "foo"]
|
|
*
|
|
* assert PipelineStage.defaultConfig(cfg) == [
|
|
* name: "foo",
|
|
* build: '${.stage}', // builds a variant by the same name
|
|
* run: [
|
|
* image: '${.imageID}', // runs the variant built by this stage
|
|
* arguments: [],
|
|
* ],
|
|
* ]
|
|
* </code></pre>
|
|
*
|
|
* @example Configuring `run: true` means run the variant built by this
|
|
* stage
|
|
* <pre><code>
|
|
* def cfg = [name: "foo", build: "foo", run: true]
|
|
*
|
|
* assert PipelineStage.defaultConfig(cfg) == [
|
|
* name: "foo",
|
|
* build: "foo",
|
|
* run: [
|
|
* image: '${.imageID}', // runs the variant built by this stage
|
|
* arguments: [],
|
|
* ],
|
|
* ]
|
|
* </code></pre>
|
|
*
|
|
* @example Publish image default configuration
|
|
* <pre><code>
|
|
* def cfg = [image: true]
|
|
* def defaults = PipelineStage.defaultConfig(cfg)
|
|
*
|
|
* // publish.image.id defaults to the previously built image
|
|
* assert defaults.publish.image.id == '${.imageID}'
|
|
*
|
|
* // publish.image.name defaults to the project name
|
|
* assert defaults.publish.image.name == '${setup.project}'
|
|
*
|
|
* // publish.image.tag defaults to {timestamp}-{stage name}
|
|
* assert defaults.publish.image.tag == '${setup.timestamp}-${.stage}'
|
|
* </code></pre>
|
|
*/
|
|
@NonCPS
|
|
static Map defaultConfig(Map cfg) {
|
|
Map dcfg
|
|
|
|
// shorthand with just name is: build and run a variant
|
|
if (cfg.size() == 1 && cfg["name"]) {
|
|
dcfg = cfg + [
|
|
build: '${.stage}',
|
|
run: [
|
|
image: '${.imageID}',
|
|
]
|
|
]
|
|
} else {
|
|
dcfg = cfg.clone()
|
|
}
|
|
|
|
if (dcfg.run) {
|
|
// run: true means run the built image
|
|
if (dcfg.run == true) {
|
|
dcfg.run = [
|
|
image: '${.imageID}',
|
|
]
|
|
} else {
|
|
dcfg.run = dcfg.run.clone()
|
|
}
|
|
|
|
// run.image defaults to previously built image
|
|
dcfg.run.image = dcfg.run.image ?: '${.imageID}'
|
|
|
|
// run.arguments defaults to []
|
|
dcfg.run.arguments = dcfg.run.arguments ?: []
|
|
}
|
|
|
|
if (dcfg.publish) {
|
|
def pcfg = dcfg.publish.clone()
|
|
|
|
if (pcfg.image) {
|
|
if (pcfg.image == true) {
|
|
pcfg.image = [:]
|
|
} else {
|
|
pcfg.image = pcfg.image.clone()
|
|
}
|
|
|
|
// publish.image.id defaults to the previously built image
|
|
pcfg.image.id = pcfg.image.id ?: '${.imageID}'
|
|
|
|
// publish.image.name defaults to the project name
|
|
pcfg.image.name = pcfg.image.name ?: "\${${SETUP}.project}"
|
|
|
|
// publish.image.tag defaults to {timestamp}-{stage name}
|
|
pcfg.image.tag = pcfg.image.tag ?: "\${${SETUP}.timestamp}-\${.stage}"
|
|
|
|
pcfg.image.tags = (pcfg.image.tags ?: []).clone()
|
|
}
|
|
|
|
if (pcfg.files) {
|
|
pcfg.files.paths = pcfg.files.paths.clone()
|
|
}
|
|
|
|
dcfg.publish = pcfg
|
|
}
|
|
|
|
if (dcfg.deploy) {
|
|
dcfg.deploy = dcfg.deploy.clone()
|
|
|
|
dcfg.deploy.image = dcfg.deploy.image ?: '${.publishedImage}'
|
|
dcfg.deploy.cluster = dcfg.deploy.cluster ?: "ci"
|
|
dcfg.deploy.test = dcfg.deploy.test == null ? true : dcfg.deploy.test
|
|
}
|
|
|
|
dcfg
|
|
}
|
|
|
|
PipelineStage(Pipeline pline, String stageName, Map stageConfig, nodeContext) {
|
|
pipeline = pline
|
|
name = stageName
|
|
config = stageConfig
|
|
context = nodeContext
|
|
}
|
|
|
|
/**
|
|
* Constructs and retruns a closure for this pipeline stage using the given
|
|
* Jenkins workflow script object.
|
|
*/
|
|
Closure closure(ws) {
|
|
({
|
|
def runner = pipeline.runner(ws)
|
|
|
|
context["stage"] = name
|
|
|
|
switch (name) {
|
|
case SETUP:
|
|
setup(ws, runner)
|
|
break
|
|
case TEARDOWN:
|
|
teardown(ws, runner)
|
|
break
|
|
default:
|
|
ws.echo("running steps in ${pipeline.directory} with config: ${config.inspect()}")
|
|
|
|
ws.dir(pipeline.directory) {
|
|
for (def stageStep in STEPS) {
|
|
if (config[stageStep]) {
|
|
ws.echo("step: ${stageStep}, config: ${config.inspect()}")
|
|
this."${stageStep}"(ws, runner)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Returns a set of node labels that will be required for this stage to
|
|
* function correctly.
|
|
*/
|
|
Set getRequiredNodeLabels() {
|
|
def labels = [] as Set
|
|
|
|
if (config.build || config.run) {
|
|
labels.add("blubber")
|
|
}
|
|
|
|
if (config.publish) {
|
|
if (config.publish.files) {
|
|
labels.add("blubber")
|
|
}
|
|
|
|
if (config.publish.image) {
|
|
labels.add("dockerPublish")
|
|
}
|
|
}
|
|
|
|
labels
|
|
}
|
|
|
|
/**
|
|
* Performs setup steps, checkout out the repo and binding useful values to
|
|
* be used by all other stages (default image labels, project identifier,
|
|
* timestamp, etc).
|
|
*
|
|
* <h3>Exports</h3>
|
|
* <dl>
|
|
* <dt><code>${setup.project}</code></dt>
|
|
* <dd>ZUUL_PROJECT parameter value if getting a patchset from Zuul.</dd>
|
|
* <dd>Jenkins JOB_NAME value otherwise.</dd>
|
|
*
|
|
* <dt><code>${setup.timestamp}</code></dt>
|
|
* <dd>Timestamp at the start of pipeline execution. Used in image tags, etc.</dd>
|
|
*
|
|
* <dt><code>${setup.imageLabels}</code></dt>
|
|
* <dd>Default set of image labels:
|
|
* <code>jenkins.job</code>,
|
|
* <code>jenkins.build</code>,
|
|
* <code>ci.project</code>,
|
|
* <code>ci.pipeline</code>
|
|
* </dd>
|
|
* </dl>
|
|
*/
|
|
void setup(ws, runner) {
|
|
def imageLabels = [
|
|
"jenkins.job": ws.env.JOB_NAME,
|
|
"jenkins.build": ws.env.BUILD_ID,
|
|
]
|
|
|
|
if (ws.params.ZUUL_REF) {
|
|
def patchset = PatchSet.fromZuul(ws.params)
|
|
ws.checkout(patchset.getSCM())
|
|
context["project"] = patchset.project.replaceAll('/', '-')
|
|
imageLabels["zuul.commit"] = patchset.commit
|
|
} else {
|
|
ws.checkout(ws.scm)
|
|
context["project"] = ws.env.JOB_NAME
|
|
}
|
|
|
|
imageLabels["ci.project"] = context['project']
|
|
imageLabels["ci.pipeline"] = pipeline.name
|
|
|
|
context["timestamp"] = timestampLabel()
|
|
context["imageLabels"] = imageLabels
|
|
}
|
|
|
|
/**
|
|
* Performs teardown steps, removing images and helm releases, and reporting
|
|
* back to Gerrit.
|
|
*/
|
|
void teardown(ws, runner) {
|
|
try {
|
|
runner.removeImages(context.getAll("imageID").values())
|
|
} catch (all) {}
|
|
|
|
try {
|
|
runner.purgeReleases(context.getAll("releaseName").values())
|
|
} catch (all) {}
|
|
|
|
def imageTags = context.getAll("imageTags")
|
|
context.getAll("publishedImage").collect { stageName, image ->
|
|
try {
|
|
runner.reportToGerrit(image, imageTags[stageName] ?: [])
|
|
} catch (all) {}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Builds the configured Blubber variant.
|
|
*
|
|
* <h3>Configuration</h3>
|
|
* <dl>
|
|
* <dt><code>build</code></dt>
|
|
* <dd>Blubber variant name</dd>
|
|
* </dl>
|
|
*
|
|
* <h3>Example</h3>
|
|
* <pre><code>
|
|
* stages:
|
|
* - name: candidate
|
|
* build: production
|
|
* </code></pre>
|
|
*
|
|
* <h3>Exports</h3>
|
|
* <dl>
|
|
* <dt><code>${[stage].imageID}</code></dt>
|
|
* <dd>Image ID of built image.</dd>
|
|
* </dl>
|
|
*/
|
|
void build(ws, runner) {
|
|
def imageID = runner.build(context % config.build, context["setup.imageLabels"])
|
|
|
|
context["imageID"] = imageID
|
|
}
|
|
|
|
/**
|
|
* Runs the entry point of a built image variant.
|
|
*
|
|
* <h3>Configuration</h3>
|
|
* <dl>
|
|
* <dt><code>run</code></dt>
|
|
* <dd>Image to run and entry-point arguments</dd>
|
|
* <dd>Specifying <code>run: true</code> expands to
|
|
* <code>run: { image: '${.imageID}' }</code>
|
|
* (i.e. the image built in this stage)</dd>
|
|
* <dd>
|
|
* <dl>
|
|
* <dt><code>image</code></dt>
|
|
* <dd>An image to run</dd>
|
|
* <dd>Default: <code>{$.imageID}</code></dd>
|
|
*
|
|
* <dt><code>arguments</code></dt>
|
|
* <dd>Entry-point arguments</dd>
|
|
* <dd>Default: <code>[]</code></dd>
|
|
* </dl>
|
|
* </dd>
|
|
* </dl>
|
|
*
|
|
* <h3>Example</h3>
|
|
* <pre><code>
|
|
* stages:
|
|
* - name: test
|
|
* build: test
|
|
* run: true
|
|
* </code></pre>
|
|
*
|
|
* <h3>Example</h3>
|
|
* <pre><code>
|
|
* stages:
|
|
* - name: built
|
|
* - name: lint
|
|
* run:
|
|
* image: '${built.imageID}'
|
|
* arguments: [lint]
|
|
* - name: test
|
|
* run:
|
|
* image: '${built.imageID}'
|
|
* arguments: [test]
|
|
* </code></pre>
|
|
*/
|
|
void run(ws, runner) {
|
|
runner.run(
|
|
context % config.run.image,
|
|
config.run.arguments.collect { context % it },
|
|
)
|
|
}
|
|
|
|
/**
|
|
* Publish artifacts, either files or a built image variant (pushed to the
|
|
* WMF Docker registry).
|
|
*
|
|
* <h3>Configuration</h3>
|
|
* <dl>
|
|
* <dt><code>publish</code></dt>
|
|
* <dd>
|
|
* <dl>
|
|
* <dt><code>image</code></dt>
|
|
* <dd>Publish an to the WMF Docker registry</dd>
|
|
* <dd>
|
|
* <dl>
|
|
* <dt>id</dt>
|
|
* <dd>ID of a previously built image variant</dd>
|
|
* <dd>Default: <code>${.imageID}</code> (image built in this stage)</dd>
|
|
*
|
|
* <dt>name</dt>
|
|
* <dd>Published name of the image. Note that this base name will be
|
|
* prefixed with the globally configured registry/repository name
|
|
* before being pushed.</dd>
|
|
* <dd>Default: <code>${setup.project}</code> (project identifier;
|
|
* see {@link setup()})</dd>
|
|
*
|
|
* <dt>tag</dt>
|
|
* <dd>Primary tag under which the image is published</dd>
|
|
* <dd>Default: <code>${setup.timestamp}-${.stage}</code></dd>
|
|
*
|
|
* <dt>tags</dt>
|
|
* <dd>Additional tags under which to publish the image</dd>
|
|
* </dl>
|
|
* </dd>
|
|
* </dl>
|
|
* </dd>
|
|
* <dd>
|
|
* <dl>
|
|
* <dt><code>files</code></dt>
|
|
* <dd>Extract and save files from a previously built image variant</dd>
|
|
* <dd>
|
|
* <dl>
|
|
* <dt>paths</dt>
|
|
* <dd>Globbed file paths resolving any number of files under the
|
|
* image's root filesystem</dd>
|
|
* </dl>
|
|
* </dd>
|
|
* </dl>
|
|
* </dd>
|
|
* </dl>
|
|
*
|
|
* <h3>Exports</h3>
|
|
* <dl>
|
|
* <dt><code>${[stage].imageName}</code></dt>
|
|
* <dd>Short name under which the image was published</dd>
|
|
*
|
|
* <dt><code>${[stage].imageFullName}</code></dt>
|
|
* <dd>Fully qualified name (registry/repository/imageName) under which the
|
|
* image was published</dd>
|
|
*
|
|
* <dt><code>${[stage].imageTag}</code></dt>
|
|
* <dd>Primary tag under which the image was published</dd>
|
|
*
|
|
* <dt><code>${[stage].publishedImage}</code></dt>
|
|
* <dd>Full qualified name and tag (<code>${.imageFullName}:${.imageTag}</code>)</dd>
|
|
* </dl>
|
|
*/
|
|
void publish(ws, runner) {
|
|
if (config.publish.image) {
|
|
def publishImage = config.publish.image
|
|
|
|
def imageID = context % publishImage.id
|
|
def imageName = context % publishImage.name
|
|
def imageTags = ([publishImage.tag] + publishImage.tags).collect { context % it }
|
|
|
|
for (def tag in imageTags) {
|
|
runner.registerAs(
|
|
imageID,
|
|
imageName,
|
|
tag,
|
|
)
|
|
}
|
|
|
|
context["imageName"] = imageName
|
|
context["imageFullName"] = runner.qualifyRegistryPath(imageName)
|
|
context["imageTag"] = context % publishImage.tag
|
|
context["imageTags"] = imageTags
|
|
context["publishedImage"] = context % '${.imageFullName}:${.imageTag}'
|
|
}
|
|
|
|
if (config.publish.files) {
|
|
// TODO
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Deploy a published image to a WMF k8s cluster. (Currently only the "ci"
|
|
* cluster is supported for testing.)
|
|
*
|
|
* <h3>Configuration</h3>
|
|
* <dl>
|
|
* <dt><code>deploy</code></dt>
|
|
* <dd>
|
|
* <dl>
|
|
* <dt>image</dt>
|
|
* <dd>Reference to a previously published image</dd>
|
|
* <dd>Default: <code>${.publishedImage}</code> (image published in the
|
|
* {@link publish() publish step} of this stage)</dd>
|
|
*
|
|
* <dt>cluster</dt>
|
|
* <dd>Cluster to target</dd>
|
|
* <dd>Default: <code>"ci"</code></dd>
|
|
* <dd>Currently only "ci" is supported and this configuration is
|
|
* effectively ignored</dd>
|
|
*
|
|
* <dt>chart</dt>
|
|
* <dd>URL of Helm chart to use for deployment</dd>
|
|
* <dd>Required</dd>
|
|
*
|
|
* <dt>test</dt>
|
|
* <dd>Whether to run <code>helm test</code> against this deployment</dd>
|
|
* <dd>Default: <code>true</code></dd>
|
|
* </dl>
|
|
* </dd>
|
|
* </dl>
|
|
*
|
|
* <h3>Exports</h3>
|
|
* <dl>
|
|
* <dt><code>${[stage].releaseName}</code></dt>
|
|
* <dd>Release name of new deployment</dd>
|
|
* </dl>
|
|
*/
|
|
void deploy(ws, runner) {
|
|
def release = runner.deployWithChart(
|
|
context % config.deploy.chart,
|
|
context % config.deploy.image,
|
|
context % config.deploy.tag,
|
|
)
|
|
|
|
context["releaseName"] = release
|
|
|
|
if (config.deploy.test) {
|
|
runner.testRelease(release)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Binds a number of new values for reference in subsequent stages.
|
|
*
|
|
* <h3>Configuration</h3>
|
|
* <dl>
|
|
* <dt><code>exports</code></dt>
|
|
* <dd>Name/value pairs for additional exports.</dd>
|
|
* </dl>
|
|
*
|
|
* <h3>Example</h3>
|
|
* <pre><code>
|
|
* stages:
|
|
* - name: candidate
|
|
* build: production
|
|
* exports:
|
|
* image: '${.imageID}'
|
|
* tag: '${.imageTag}-my-tag'
|
|
* - name: published
|
|
* publish:
|
|
* image:
|
|
* id: '${candidate.image}'
|
|
* tags: ['${candidate.tag}']
|
|
* </code></pre>
|
|
*
|
|
* <h3>Exports</h3>
|
|
* <dl>
|
|
* <dt><code>${[name].[value]}</code></dt>
|
|
* <dd>Each configured name/value pair.</dd>
|
|
* </dl>
|
|
*/
|
|
void exports(ws, runner) {
|
|
config.exports.each { export, value ->
|
|
context[export] = context % value
|
|
ws.echo "exported ${name}.${export}=${context[export].inspect()}"
|
|
}
|
|
}
|
|
}
|