package org.wikimedia.integration import com.cloudbees.groovy.cps.NonCPS import static org.wikimedia.integration.Utility.timestampLabel import org.wikimedia.integration.ExecutionContext import org.wikimedia.integration.PatchSet import org.wikimedia.integration.Pipeline class PipelineStage implements Serializable { static final String SETUP = 'setup' static final String TEARDOWN = 'teardown' static final List STEPS = ['build', 'run', 'publish', 'deploy', 'exports'] Pipeline pipeline String name Map config private def context /** * Returns an config based on the given one but with default values * inserted. * * @example Shorthand stage config (providing only a stage name) *

   *   def cfg = [name: "foo"]
   *
   *   assert PipelineStage.defaultConfig(cfg) == [
   *     name: "foo",
   *     build: '${.stage}',     // builds a variant by the same name
   *     run: [
   *       image: '${.imageID}', // runs the variant built by this stage
   *       arguments: [],
   *     ],
   *   ]
   * 
* * @example Configuring `run: true` means run the variant built by this * stage *

   *   def cfg = [name: "foo", build: "foo", run: true]
   *
   *   assert PipelineStage.defaultConfig(cfg) == [
   *     name: "foo",
   *     build: "foo",
   *     run: [
   *       image: '${.imageID}', // runs the variant built by this stage
   *       arguments: [],
   *     ],
   *   ]
   * 
* * @example Publish image default configuration *

   *   def cfg = [image: true]
   *   def defaults = PipelineStage.defaultConfig(cfg)
   *
   *   // publish.image.id defaults to the previously built image
   *   assert defaults.publish.image.id == '${.imageID}'
   *
   *   // publish.image.name defaults to the project name
   *   assert defaults.publish.image.name == '${setup.project}'
   *
   *   // publish.image.tag defaults to {timestamp}-{stage name}
   *   assert defaults.publish.image.tag == '${setup.timestamp}-${.stage}'
   * 
*/ @NonCPS static Map defaultConfig(Map cfg) { Map dcfg // shorthand with just name is: build and run a variant if (cfg.size() == 1 && cfg["name"]) { dcfg = cfg + [ build: '${.stage}', run: [ image: '${.imageID}', ] ] } else { dcfg = cfg.clone() } if (dcfg.run) { // run: true means run the built image if (dcfg.run == true) { dcfg.run = [ image: '${.imageID}', ] } else { dcfg.run = dcfg.run.clone() } // run.image defaults to previously built image dcfg.run.image = dcfg.run.image ?: '${.imageID}' // run.arguments defaults to [] dcfg.run.arguments = dcfg.run.arguments ?: [] } if (dcfg.publish) { def pcfg = dcfg.publish.clone() if (pcfg.image) { if (pcfg.image == true) { pcfg.image = [:] } else { pcfg.image = pcfg.image.clone() } // publish.image.id defaults to the previously built image pcfg.image.id = pcfg.image.id ?: '${.imageID}' // publish.image.name defaults to the project name pcfg.image.name = pcfg.image.name ?: "\${${SETUP}.project}" // publish.image.tag defaults to {timestamp}-{stage name} pcfg.image.tag = pcfg.image.tag ?: "\${${SETUP}.timestamp}-\${.stage}" pcfg.image.tags = (pcfg.image.tags ?: []).clone() } if (pcfg.files) { pcfg.files.paths = pcfg.files.paths.clone() } dcfg.publish = pcfg } if (dcfg.deploy) { dcfg.deploy = dcfg.deploy.clone() dcfg.deploy.image = dcfg.deploy.image ?: '${.publishedImage}' dcfg.deploy.cluster = dcfg.deploy.cluster ?: "ci" dcfg.deploy.test = dcfg.deploy.test == null ? true : dcfg.deploy.test } dcfg } PipelineStage(Pipeline pline, String stageName, Map stageConfig, nodeContext) { pipeline = pline name = stageName config = stageConfig context = nodeContext } /** * Constructs and retruns a closure for this pipeline stage using the given * Jenkins workflow script object. */ Closure closure(ws) { ({ def runner = pipeline.runner(ws) context["stage"] = name switch (name) { case SETUP: setup(ws, runner) break case TEARDOWN: teardown(ws, runner) break default: ws.echo("running steps in ${pipeline.directory} with config: ${config.inspect()}") ws.dir(pipeline.directory) { for (def stageStep in STEPS) { if (config[stageStep]) { ws.echo("step: ${stageStep}, config: ${config.inspect()}") this."${stageStep}"(ws, runner) } } } } }) } /** * Returns a set of node labels that will be required for this stage to * function correctly. */ Set getRequiredNodeLabels() { def labels = [] as Set if (config.build || config.run) { labels.add("blubber") } if (config.publish) { if (config.publish.files) { labels.add("blubber") } if (config.publish.image) { labels.add("dockerPublish") } } labels } /** * Performs setup steps, checkout out the repo and binding useful values to * be used by all other stages (default image labels, project identifier, * timestamp, etc). * *

Exports

*
*
${setup.project}
*
ZUUL_PROJECT parameter value if getting a patchset from Zuul.
*
Jenkins JOB_NAME value otherwise.
* *
${setup.timestamp}
*
Timestamp at the start of pipeline execution. Used in image tags, etc.
* *
${setup.imageLabels}
*
Default set of image labels: * jenkins.job, * jenkins.build, * ci.project, * ci.pipeline *
*
*/ void setup(ws, runner) { def imageLabels = [ "jenkins.job": ws.env.JOB_NAME, "jenkins.build": ws.env.BUILD_ID, ] if (ws.params.ZUUL_REF) { def patchset = PatchSet.fromZuul(ws.params) ws.checkout(patchset.getSCM()) context["project"] = patchset.project.replaceAll('/', '-') imageLabels["zuul.commit"] = patchset.commit } else { ws.checkout(ws.scm) context["project"] = ws.env.JOB_NAME } imageLabels["ci.project"] = context['project'] imageLabels["ci.pipeline"] = pipeline.name context["timestamp"] = timestampLabel() context["imageLabels"] = imageLabels } /** * Performs teardown steps, removing images and helm releases, and reporting * back to Gerrit. */ void teardown(ws, runner) { try { runner.removeImages(context.getAll("imageID").values()) } catch (all) {} try { runner.purgeReleases(context.getAll("releaseName").values()) } catch (all) {} def imageTags = context.getAll("imageTags") context.getAll("publishedImage").collect { stageName, image -> try { runner.reportToGerrit(image, imageTags[stageName] ?: []) } catch (all) {} } } /** * Builds the configured Blubber variant. * *

Configuration

*
*
build
*
Blubber variant name
*
* *

Example

*

   *   stages:
   *     - name: candidate
   *       build: production
   * 
* *

Exports

*
*
${[stage].imageID}
*
Image ID of built image.
*
*/ void build(ws, runner) { def imageID = runner.build(context % config.build, context["setup.imageLabels"]) context["imageID"] = imageID } /** * Runs the entry point of a built image variant. * *

Configuration

*
*
run
*
Image to run and entry-point arguments
*
Specifying run: true expands to * run: { image: '${.imageID}' } * (i.e. the image built in this stage)
*
*
*
image
*
An image to run
*
Default: {$.imageID}
* *
arguments
*
Entry-point arguments
*
Default: []
*
*
*
* *

Example

*

   *   stages:
   *     - name: test
   *       build: test
   *       run: true
   * 
* *

Example

*

   *   stages:
   *     - name: built
   *     - name: lint
   *       run:
   *         image: '${built.imageID}'
   *         arguments: [lint]
   *     - name: test
   *       run:
   *         image: '${built.imageID}'
   *         arguments: [test]
   * 
*/ void run(ws, runner) { runner.run( context % config.run.image, config.run.arguments.collect { context % it }, ) } /** * Publish artifacts, either files or a built image variant (pushed to the * WMF Docker registry). * *

Configuration

*
*
publish
*
*
*
image
*
Publish an to the WMF Docker registry
*
*
*
id
*
ID of a previously built image variant
*
Default: ${.imageID} (image built in this stage)
* *
name
*
Published name of the image. Note that this base name will be * prefixed with the globally configured registry/repository name * before being pushed.
*
Default: ${setup.project} (project identifier; * see {@link setup()})
* *
tag
*
Primary tag under which the image is published
*
Default: ${setup.timestamp}-${.stage}
* *
tags
*
Additional tags under which to publish the image
*
*
*
*
*
*
*
files
*
Extract and save files from a previously built image variant
*
*
*
paths
*
Globbed file paths resolving any number of files under the * image's root filesystem
*
*
*
*
*
* *

Exports

*
*
${[stage].imageName}
*
Short name under which the image was published
* *
${[stage].imageFullName}
*
Fully qualified name (registry/repository/imageName) under which the * image was published
* *
${[stage].imageTag}
*
Primary tag under which the image was published
* *
${[stage].publishedImage}
*
Full qualified name and tag (${.imageFullName}:${.imageTag})
*
*/ void publish(ws, runner) { if (config.publish.image) { def publishImage = config.publish.image def imageID = context % publishImage.id def imageName = context % publishImage.name def imageTags = ([publishImage.tag] + publishImage.tags).collect { context % it } for (def tag in imageTags) { runner.registerAs( imageID, imageName, tag, ) } context["imageName"] = imageName context["imageFullName"] = runner.qualifyRegistryPath(imageName) context["imageTag"] = context % publishImage.tag context["imageTags"] = imageTags context["publishedImage"] = context % '${.imageFullName}:${.imageTag}' } if (config.publish.files) { // TODO } } /** * Deploy a published image to a WMF k8s cluster. (Currently only the "ci" * cluster is supported for testing.) * *

Configuration

*
*
deploy
*
*
*
image
*
Reference to a previously published image
*
Default: ${.publishedImage} (image published in the * {@link publish() publish step} of this stage)
* *
cluster
*
Cluster to target
*
Default: "ci"
*
Currently only "ci" is supported and this configuration is * effectively ignored
* *
chart
*
URL of Helm chart to use for deployment
*
Required
* *
test
*
Whether to run helm test against this deployment
*
Default: true
*
*
*
* *

Exports

*
*
${[stage].releaseName}
*
Release name of new deployment
*
*/ void deploy(ws, runner) { def release = runner.deployWithChart( context % config.deploy.chart, context % config.deploy.image, context % config.deploy.tag, ) context["releaseName"] = release if (config.deploy.test) { runner.testRelease(release) } } /** * Binds a number of new values for reference in subsequent stages. * *

Configuration

*
*
exports
*
Name/value pairs for additional exports.
*
* *

Example

*

   *   stages:
   *     - name: candidate
   *       build: production
   *       exports:
   *         image: '${.imageID}'
   *         tag: '${.imageTag}-my-tag'
   *     - name: published
   *       publish:
   *         image:
   *           id: '${candidate.image}'
   *           tags: ['${candidate.tag}']
   * 
* *

Exports

*
*
${[name].[value]}
*
Each configured name/value pair.
*
*/ void exports(ws, runner) { config.exports.each { export, value -> context[export] = context % value ws.echo "exported ${name}.${export}=${context[export].inspect()}" } } }