Browse Source

Use blubberoid.wikimedia.org to process blubber.yaml

Simplified `Blubber` client to handle only the transcompilation of Blubber
configuration into Dockerfile and not the actual building of images.
Building was moved into `PipelineRunner` where all other Docker commands
are invoked.

Blubber client was refactored to use the WMF production deployment of
Blubberoid instead of relying on a locally installed binary of the
`blubber` CLI.

Bug: T212247
Change-Id: Ib403786af7af6ce9d469798452da512fa535f2b4
master
Dan Duvall 2 years ago
parent
commit
ebcb93b0ac
5 changed files with 152 additions and 33 deletions
  1. +28
    -1
      Jenkinsfile
  2. +42
    -13
      src/org/wikimedia/integration/Blubber.groovy
  3. +16
    -2
      src/org/wikimedia/integration/PipelineRunner.groovy
  4. +46
    -12
      test/org/wikimedia/integration/BlubberTest.groovy
  5. +20
    -5
      test/org/wikimedia/integration/PipelineRunnerTest.groovy

+ 28
- 1
Jenkinsfile View File

@ -1,10 +1,37 @@
/**
* Functionally tests pipelinelib by retrieving the patchset referenced by
* `scm` and importing the library into the current context before making some
* basic assertions about its methods behaviors. Note that the Jenkins job
* that runs this Jenkinsfile must already define `scm` with the correct Zuul
* parameters.
*/
def plib = library(identifier: 'pipelinelib@FETCH_HEAD', retriever: legacySCM(scm)).org.wikimedia.integration
def prunner = plib.PipelineRunner.new(this)
def imageID
node('blubber') {
def blubberoidURL = "https://blubberoid.wikimedia.org/v1/"
stage('Test checkout') {
stage('Checkout SCM') {
def patchset = plib.PatchSet.fromZuul(params)
checkout(patchset.getSCM())
}
stage('Generate Dockerfile') {
def blubber = plib.Blubber.new(this, '.pipeline/blubber.yaml', blubberoidURL)
def dockerfile = blubber.generateDockerfile("test")
echo 'Checking that Dockerfile was correctly generated'
assert dockerfile.contains('LABEL blubber.variant="test"')
}
stage('Build test image') {
imageID = prunner.build('test')
echo 'Successfully built image "${imageID}" from "test" variant'
}
stage('Remove test image') {
prunner.removeImage(imageID)
echo 'Removed test image "${imageID}"'
}
}

+ 42
- 13
src/org/wikimedia/integration/Blubber.groovy View File

@ -1,9 +1,11 @@
package org.wikimedia.integration
import java.net.URLEncoder
import static org.wikimedia.integration.Utility.arg
/**
* Provides an interface to Blubber for building container images.
* Provides an interface to Blubber for generating Dockerfiles.
*/
class Blubber implements Serializable {
/**
@ -16,33 +18,60 @@ class Blubber implements Serializable {
*/
final def workflowScript
/**
* Blubberoid base service URL.
*/
final String blubberoidURL
/**
* Blubber constructor.
*
* @param workflowScript Jenkins workflow script context.
* @param configPath Blubber config path.
* @param blubberoidURL Blubberoid service URL.
*/
Blubber(workflowScript, String configPath) {
Blubber(workflowScript, String configPath, String blubberoidURL) {
this.workflowScript = workflowScript
this.configPath = configPath
this.blubberoidURL = blubberoidURL
}
/**
* Builds the given variant and tags the image with the given tag name and
* labels.
* Returns a valid Dockerfile for the given variant.
*
* @param variant Blubber variant name that should be built.
* @param labels Additional name/value labels to add to the image.
* @param variant Blubber variant name.
*/
String build(String variant, Map labels = [:]) {
def labelFlags = labels.collect { k, v -> "--label ${arg(k + "=" + v)}" }.join(" ")
String generateDockerfile(String variant) {
def config = workflowScript.readFile(file: configPath)
def headers = [[name: "content-type", value: getConfigMediaType()]]
def response = workflowScript.httpRequest(url: getRequestURL(variant),
httpMode: "POST",
customHeaders: headers,
requestBody: config,
consoleLogResponseBody: true,
validResponseCodes: "200")
def cmd = "blubber ${arg(configPath)} ${arg(variant)} | " +
"docker build --pull ${labelFlags} --file - ."
response.content
}
def output = workflowScript.sh(returnStdout: true, script: cmd)
/**
* Returns a request media type based on the config file extension.
*/
String getConfigMediaType() {
def ext = configPath.substring(configPath.lastIndexOf(".") + 1)
switch (ext) {
case ["yaml", "yml"]:
return "application/yaml"
default:
return "application/json"
}
}
// Return just the image ID from `docker build` output
output.substring(output.lastIndexOf(" ") + 1).trim()
/**
* Return a request URL for the given variant.
*/
String getRequestURL(String variant) {
blubberoidURL + URLEncoder.encode(variant, "UTF-8")
}
}

+ 16
- 2
src/org/wikimedia/integration/PipelineRunner.groovy View File

@ -24,6 +24,11 @@ class PipelineRunner implements Serializable {
*/
def blubberConfig = "blubber.yaml"
/**
* Base URL for the Blubberoid service.
*/
def blubberoidURL = "https://blubberoid.wikimedia.org/v1/"
/**
* Directory in which pipeline configuration is stored.
*/
@ -95,9 +100,18 @@ class PipelineRunner implements Serializable {
throw new FileNotFoundException("failed to build image: no Blubber config found at ${cfg}")
}
def blubber = new Blubber(workflowScript, cfg)
def blubber = new Blubber(workflowScript, cfg, blubberoidURL)
def dockerfile = getConfigFile("Dockerfile")
workflowScript.writeFile(text: blubber.generateDockerfile(variant), file: dockerfile)
def labelFlags = labels.collect { k, v -> "--label ${arg(k + "=" + v)}" }.join(" ")
def dockerBuild = "docker build --pull ${labelFlags} --file ${arg(dockerfile)} ."
def output = workflowScript.sh(returnStdout: true, script: dockerBuild)
blubber.build(variant, labels)
// Return just the image ID from `docker build` output
output.substring(output.lastIndexOf(" ") + 1).trim()
}
/**


+ 46
- 12
test/org/wikimedia/integration/BlubberTest.groovy View File

@ -6,25 +6,59 @@ import org.wikimedia.integration.Blubber
class BlubberTestCase extends GroovyTestCase {
private class WorkflowScript {} // Mock for Jenkins Pipeline workflow context
void testBuildCommand() {
def blubberConfig = ".pipeline/blubber.yaml"
def blubberoidURL = "https://an.example/blubberoid/v1/"
void testGenerateDockerfile() {
def mock = new MockFor(WorkflowScript)
def config = "version: v3\n" +
"base: foo"
mock.demand.readFile { args ->
assert args.file == blubberConfig
config
}
mock.demand.sh { args ->
assert args.returnStdout
assert args.script == "blubber 'foo/blubber.yaml' 'foo' | " +
"docker build --pull --label 'foo=a' --label 'bar=b' --file - ."
mock.demand.httpRequest { args ->
assert args.httpMode == "POST"
assert args.customHeaders == [[name: "content-type", value: "application/yaml"]]
assert args.requestBody == config
assert args.consoleLogResponseBody == true
assert args.validResponseCodes == "200"
// Mock `docker build` output to test that we correctly parse the image ID
return "Removing intermediate container foo\n" +
" ---> bf1e86190382\n" +
"Successfully built bf1e86190382\n"
[content: "BASE foo\n"]
}
mock.use {
def blubber = new Blubber(new WorkflowScript(), "foo/blubber.yaml")
def imageID = blubber.build("foo", [foo: "a", bar: "b"])
def blubber = new Blubber(new WorkflowScript(), blubberConfig, blubberoidURL)
def dockerfile = blubber.generateDockerfile("foo")
assert imageID == "bf1e86190382"
assert dockerfile == "BASE foo\n"
}
}
void testGetConfigMediaType_yaml() {
def blubber = new Blubber(new WorkflowScript(), blubberConfig, blubberoidURL)
assert blubber.getConfigMediaType() == "application/yaml"
}
void testGetConfigMediaType_yml() {
def blubber = new Blubber(new WorkflowScript(), blubberConfig, blubberoidURL)
assert blubber.getConfigMediaType() == "application/yaml"
}
void testGetConfigMediaType_json() {
def blubber = new Blubber(new WorkflowScript(), ".pipeline/blubber.json", blubberoidURL)
assert blubber.getConfigMediaType() == "application/json"
}
void testGetRequestURL() {
def blubber = new Blubber(new WorkflowScript(), blubberConfig, blubberoidURL)
assert blubber.getRequestURL("foo bar") == "https://an.example/blubberoid/v1/foo+bar"
}
}

+ 20
- 5
test/org/wikimedia/integration/PipelineRunnerTest.groovy View File

@ -71,23 +71,38 @@ class PipelineRunnerTest extends GroovyTestCase {
}
}
void testBuild_delegatesToBlubber() {
void testBuild_generatesDockerfileAndBuilds() {
def mockWorkflow = new MockFor(WorkflowScript)
def mockBlubber = new MockFor(Blubber)
mockWorkflow.demand.fileExists { true }
mockBlubber.demand.build { variant, labels ->
mockBlubber.demand.generateDockerfile { variant ->
assert variant == "foo"
assert labels == [bar: "baz"]
"fooimageID"
"BASE: foo\n"
}
mockWorkflow.demand.writeFile { args ->
assert args.text == "BASE: foo\n"
assert args.file == ".pipeline/Dockerfile"
}
mockWorkflow.demand.sh { args ->
assert args.returnStdout
assert args.script == "docker build --pull --label 'foo=a' --label 'bar=b' --file '.pipeline/Dockerfile' ."
// Mock `docker build` output to test that we correctly parse the image ID
return "Removing intermediate container foo\n" +
" ---> bf1e86190382\n" +
"Successfully built bf1e86190382\n"
}
mockWorkflow.use {
mockBlubber.use {
def runner = new PipelineRunner(new WorkflowScript())
runner.build("foo", [bar: "baz"])
assert runner.build("foo", [foo: "a", bar: "b"]) == "bf1e86190382"
}
}
}


Loading…
Cancel
Save