You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
331 lines
9.3 KiB
331 lines
9.3 KiB
package org.wikimedia.integration
|
|
|
|
import java.io.FileNotFoundException
|
|
|
|
import static org.wikimedia.integration.Utility.*
|
|
|
|
import org.wikimedia.integration.GerritPipelineComment
|
|
import org.wikimedia.integration.GerritReview
|
|
|
|
/**
|
|
* Provides an interface to common pipeline build/run/deploy functions.
|
|
*
|
|
* You must provide the Jenkins workflow script sandbox object that will be
|
|
* used to declare Jenkins pipeline steps.
|
|
*
|
|
* {@code
|
|
* // With just the Jenkins context
|
|
* def pipeline = new PipelineRunner(this)
|
|
*
|
|
* // Or with a map of additional settings
|
|
* def pipeline = new PipelineRunner(this, configPath: "dist/pipeline", registry: "foo.registry")
|
|
* }
|
|
*/
|
|
class PipelineRunner implements Serializable {
|
|
/**
|
|
* Relative path to Blubber config file.
|
|
*/
|
|
def blubberConfig = "blubber.yaml"
|
|
|
|
/**
|
|
* Base URL for the Blubberoid service.
|
|
*/
|
|
def blubberoidURL = "https://blubberoid.wikimedia.org/v1/"
|
|
|
|
/**
|
|
* Directory in which pipeline configuration is stored.
|
|
*/
|
|
def configPath = ".pipeline"
|
|
|
|
/**
|
|
* Relative path to Helm config file.
|
|
*/
|
|
def helmConfig = "helm.yaml"
|
|
|
|
/**
|
|
* Absolute path to a Kubernetes config file to specify when executing
|
|
* `helm` or other k8s related commands. By default, none will be specified.
|
|
*/
|
|
def kubeConfig = null
|
|
|
|
/**
|
|
* Namespace used for Helm/Kubernetes deployments.
|
|
*/
|
|
def namespace = "ci"
|
|
|
|
/**
|
|
* Docker pull policy when deploying.
|
|
*/
|
|
def pullPolicy = "IfNotPresent"
|
|
|
|
/**
|
|
* Default Docker registry host used when qualifying public image URLs.
|
|
*/
|
|
def registry = "docker-registry.wikimedia.org"
|
|
|
|
/**
|
|
* Alternative Docker registry host used only when registering images.
|
|
*/
|
|
def registryInternal = "docker-registry.discovery.wmnet"
|
|
|
|
/**
|
|
* Default Docker registry repository used for tagging and registering images.
|
|
*/
|
|
def repository = "wikimedia"
|
|
|
|
/**
|
|
* Timeout for deployment using Helm.
|
|
*/
|
|
def timeout = 120
|
|
|
|
/**
|
|
* Jenkins Pipeline Workflow context.
|
|
*/
|
|
final def workflowScript
|
|
|
|
/**
|
|
* Constructor with Jenkins workflow script context and settings.
|
|
*
|
|
* @param settings Property map.
|
|
* @param workflowScript Jenkins workflow script sandbox object.
|
|
*/
|
|
PipelineRunner(Map settings = [:], workflowScript) {
|
|
this.workflowScript = workflowScript
|
|
|
|
settings.each { prop, value -> this.@"${prop}" = value }
|
|
}
|
|
|
|
/**
|
|
* Builds the given image variant and returns an ID for the resulting image.
|
|
*
|
|
* @param variant Image variant name that should be built.
|
|
* @param labels Additional name/value labels to add to the image metadata.
|
|
*/
|
|
String build(String variant, Map labels = [:]) {
|
|
def cfg = getConfigFile(blubberConfig)
|
|
|
|
if (!workflowScript.fileExists(cfg)) {
|
|
throw new FileNotFoundException("failed to build image: no Blubber config found at ${cfg}")
|
|
}
|
|
|
|
def blubber = new Blubber(workflowScript, cfg, blubberoidURL)
|
|
def dockerfile = getTempFile("Dockerfile.")
|
|
|
|
workflowScript.writeFile(text: blubber.generateDockerfile(variant), file: dockerfile)
|
|
|
|
def labelFlags = labels.collect { k, v -> "--label ${arg(k + "=" + v)}" }.join(" ")
|
|
def dockerBuild = "docker build --pull ${labelFlags} --file ${arg(dockerfile)} ."
|
|
|
|
def output = workflowScript.sh(returnStdout: true, script: dockerBuild)
|
|
|
|
// Return just the image ID from `docker build` output
|
|
output.substring(output.lastIndexOf(" ") + 1).trim()
|
|
}
|
|
|
|
/**
|
|
* Deploys the given registered image using the configured Helm chart and
|
|
* returns the name of the release.
|
|
*
|
|
* @param imageName Name of the registered image to deploy.
|
|
* @param imageTag Tag of the registered image to use.
|
|
* @param overrides Additional Helm value overrides to set.
|
|
*/
|
|
String deploy(String imageName, String imageTag, Map overrides = [:]) {
|
|
def cfg = workflowScript.readYaml(file: getConfigFile(helmConfig))
|
|
|
|
assert cfg instanceof Map && cfg.chart : "you must define 'chart: <helm chart url>' in ${cfg}"
|
|
|
|
deployWithChart(cfg.chart, imageName, imageTag, overrides)
|
|
}
|
|
|
|
/**
|
|
* Deploys the given registered image using the given Helm chart and returns
|
|
* the name of the release.
|
|
*
|
|
* @param chart Chart URL.
|
|
* @param imageName Name of the registered image to deploy.
|
|
* @param imageTag Tag of the registered image to use.
|
|
* @param overrides Additional Helm value overrides to set.
|
|
*/
|
|
String deployWithChart(String chart, String imageName, String imageTag, Map overrides = [:]) {
|
|
def values = [
|
|
"docker.registry": registry,
|
|
"docker.pull_policy": pullPolicy,
|
|
"main_app.image": [repository, imageName].join("/"),
|
|
"main_app.version": imageTag,
|
|
] + overrides
|
|
|
|
values = values.collect { k, v -> arg(k + "=" + v) }.join(",")
|
|
|
|
def release = imageName + "-" + randomAlphanum(8)
|
|
|
|
helm("install --namespace=${arg(namespace)} --set ${values} -n ${arg(release)} " +
|
|
"--debug --wait --timeout ${timeout} ${arg(chart)}")
|
|
|
|
release
|
|
}
|
|
|
|
/**
|
|
* Returns a path under configPath to the given config file.
|
|
*
|
|
* @param filePath Relative file path.
|
|
*/
|
|
String getConfigFile(String filePath) {
|
|
[configPath, filePath].join("/")
|
|
}
|
|
|
|
/**
|
|
* Returns a path under configPath to a temp file with the given base name.
|
|
*
|
|
* @param baseName File base name.
|
|
*/
|
|
String getTempFile(String baseName) {
|
|
getConfigFile("${baseName}${randomAlphanum(8)}")
|
|
}
|
|
|
|
/**
|
|
* Deletes and purges the given Helm release.
|
|
*
|
|
* @param release Previously deployed release name.
|
|
*/
|
|
void purgeRelease(String release) {
|
|
purgeReleases([release])
|
|
}
|
|
|
|
/**
|
|
* Deletes and purges the given Helm release.
|
|
*
|
|
* @param release Previously deployed release name.
|
|
*/
|
|
void purgeReleases(List releases) {
|
|
if (releases.size() > 0) {
|
|
helm("delete --purge ${args(releases)}")
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Name and push an image specified by the given image ID to the WMF Docker
|
|
* registry.
|
|
*
|
|
* The repo name is enforced as "docker-registry.wikimedia.org", and the
|
|
* remote path prefix is enforced as "/wikimedia/".
|
|
*
|
|
* {@code
|
|
* // Pushes built image to docker-registry.wikimedia.org/wikimedia/mathoid:build-123
|
|
* def image = pipeline.build("production")
|
|
* pipeline.registerAs(image, "mathoid", "build-123")
|
|
* }
|
|
* @param imageID Image ID.
|
|
* @param name Remote name to use for the image.
|
|
* @param tag Remote tag to use for the image.
|
|
*/
|
|
String registerAs(String imageID, String name, String tag) {
|
|
def nameAndTag = qualifyRegistryPath(name, registryInternal) + ":" + tag
|
|
|
|
workflowScript.sh("docker tag ${arg(imageID)} ${arg(nameAndTag)} && " +
|
|
"sudo /usr/local/bin/docker-pusher ${arg(nameAndTag)}")
|
|
|
|
nameAndTag
|
|
}
|
|
|
|
/**
|
|
* Removes the given image from the local cache. All tags are removed from
|
|
* the image as well.
|
|
*
|
|
* @param imageID ID of the image to remove.
|
|
*/
|
|
void removeImage(String imageID) {
|
|
removeImages([imageID])
|
|
}
|
|
|
|
/**
|
|
* Removes the given images from the local cache.
|
|
*
|
|
* @param imageIDs IDs of images to remove.
|
|
*/
|
|
void removeImages(List imageIDs) {
|
|
if (imageIDs.size() > 0) {
|
|
workflowScript.sh("docker rmi --force ${args(imageIDs)}")
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Submits a comment to Gerrit with the build result and links to published
|
|
* images.
|
|
*
|
|
* @param imageName Fully qualified name of published image.
|
|
* @param imageTags Image tags.
|
|
*/
|
|
void reportToGerrit(imageName, imageTags = []) {
|
|
def comment
|
|
|
|
if (workflowScript.currentBuild.result == 'SUCCESS' && imageName) {
|
|
comment = new GerritPipelineComment(
|
|
jobName: workflowScript.env.JOB_NAME,
|
|
buildNumber: workflowScript.env.BUILD_NUMBER,
|
|
jobStatus: workflowScript.currentBuild.result,
|
|
image: imageName,
|
|
tags: imageTags,
|
|
)
|
|
} else {
|
|
comment = new GerritPipelineComment(
|
|
jobName: workflowScript.env.JOB_NAME,
|
|
buildNumber: workflowScript.env.BUILD_NUMBER,
|
|
jobStatus: workflowScript.currentBuild.result,
|
|
)
|
|
}
|
|
|
|
GerritReview.post(workflowScript, comment)
|
|
}
|
|
|
|
/**
|
|
* Runs a container using the image specified by the given ID.
|
|
*
|
|
* @param imageID Image ID.
|
|
* @param arguments Entry-point arguments.
|
|
*/
|
|
void run(String imageID, List arguments = []) {
|
|
workflowScript.timeout(time: 20, unit: "MINUTES") {
|
|
workflowScript.sh("exec docker run --rm ${args([imageID] + arguments)}")
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fully qualifies an image name to a public registry path.
|
|
*
|
|
* @param name Image name.
|
|
* @param registryName Alternative registry. Defaults to {@link registry}.
|
|
*/
|
|
String qualifyRegistryPath(String name, String registryName = "") {
|
|
assert !name.contains("/") : "image name ${name} cannot contain slashes"
|
|
|
|
[registryName ?: registry, repository, name].join("/")
|
|
}
|
|
|
|
/**
|
|
* Runs end-to-end tests for the given release via `helm test`.
|
|
*
|
|
* @param release Previously deployed release name.
|
|
*/
|
|
void testRelease(String release) {
|
|
helm("test --cleanup ${arg(release)}")
|
|
}
|
|
|
|
private
|
|
|
|
/**
|
|
* Execute a helm command, specifying the right tiller namespace.
|
|
*/
|
|
void helm(String cmd) {
|
|
kubeCmd("helm --tiller-namespace=${arg(namespace)} ${cmd}")
|
|
}
|
|
|
|
/**
|
|
* Execute a Kubernetes related command, specifying the configured
|
|
* kubeConfig.
|
|
*/
|
|
void kubeCmd(String cmd) {
|
|
def env = kubeConfig ? "KUBECONFIG=${arg(kubeConfig)} " : ""
|
|
workflowScript.sh("${env}${cmd}")
|
|
}
|
|
}
|