Browse Source

Use blubberoid.wikimedia.org to process blubber.yaml

Simplified `Blubber` client to handle only the transcompilation of Blubber
configuration into Dockerfile and not the actual building of images.
Building was moved into `PipelineRunner` where all other Docker commands
are invoked.

Blubber client was refactored to use the WMF production deployment of
Blubberoid instead of relying on a locally installed binary of the
`blubber` CLI.

Bug: T212247
Change-Id: Ib403786af7af6ce9d469798452da512fa535f2b4
Dan Duvall 9 months ago
parent
commit
ebcb93b0ac

+ 28
- 1
Jenkinsfile View File

@@ -1,10 +1,37 @@
1
+/**
2
+ * Functionally tests pipelinelib by retrieving the patchset referenced by
3
+ * `scm` and importing the library into the current context before making some
4
+ * basic assertions about its methods behaviors. Note that the Jenkins job
5
+ * that runs this Jenkinsfile must already define `scm` with the correct Zuul
6
+ * parameters.
7
+ */
1 8
 def plib = library(identifier: 'pipelinelib@FETCH_HEAD', retriever: legacySCM(scm)).org.wikimedia.integration
9
+def prunner = plib.PipelineRunner.new(this)
10
+def imageID
2 11
 
3 12
 node('blubber') {
4 13
   def blubberoidURL = "https://blubberoid.wikimedia.org/v1/"
5 14
 
6
-  stage('Test checkout') {
15
+  stage('Checkout SCM') {
7 16
     def patchset = plib.PatchSet.fromZuul(params)
8 17
     checkout(patchset.getSCM())
9 18
   }
19
+
20
+  stage('Generate Dockerfile') {
21
+    def blubber = plib.Blubber.new(this, '.pipeline/blubber.yaml', blubberoidURL)
22
+    def dockerfile = blubber.generateDockerfile("test")
23
+
24
+    echo 'Checking that Dockerfile was correctly generated'
25
+    assert dockerfile.contains('LABEL blubber.variant="test"')
26
+  }
27
+
28
+  stage('Build test image') {
29
+    imageID = prunner.build('test')
30
+    echo 'Successfully built image "${imageID}" from "test" variant'
31
+  }
32
+
33
+  stage('Remove test image') {
34
+    prunner.removeImage(imageID)
35
+    echo 'Removed test image "${imageID}"'
36
+  }
10 37
 }

+ 42
- 13
src/org/wikimedia/integration/Blubber.groovy View File

@@ -1,9 +1,11 @@
1 1
 package org.wikimedia.integration
2 2
 
3
+import java.net.URLEncoder
4
+
3 5
 import static org.wikimedia.integration.Utility.arg
4 6
 
5 7
 /**
6
- * Provides an interface to Blubber for building container images.
8
+ * Provides an interface to Blubber for generating Dockerfiles.
7 9
  */
8 10
 class Blubber implements Serializable {
9 11
   /**
@@ -17,32 +19,59 @@ class Blubber implements Serializable {
17 19
   final def workflowScript
18 20
 
19 21
   /**
22
+   * Blubberoid base service URL.
23
+   */
24
+  final String blubberoidURL
25
+
26
+  /**
20 27
    * Blubber constructor.
21 28
    *
22 29
    * @param workflowScript Jenkins workflow script context.
23 30
    * @param configPath Blubber config path.
31
+   * @param blubberoidURL Blubberoid service URL.
24 32
    */
25
-  Blubber(workflowScript, String configPath) {
33
+  Blubber(workflowScript, String configPath, String blubberoidURL) {
26 34
     this.workflowScript = workflowScript
27 35
     this.configPath = configPath
36
+    this.blubberoidURL = blubberoidURL
28 37
   }
29 38
 
30 39
   /**
31
-   * Builds the given variant and tags the image with the given tag name and
32
-   * labels.
40
+   * Returns a valid Dockerfile for the given variant.
33 41
    *
34
-   * @param variant Blubber variant name that should be built.
35
-   * @param labels Additional name/value labels to add to the image.
42
+   * @param variant Blubber variant name.
36 43
    */
37
-  String build(String variant, Map labels = [:]) {
38
-    def labelFlags = labels.collect { k, v -> "--label ${arg(k + "=" + v)}" }.join(" ")
44
+  String generateDockerfile(String variant) {
45
+    def config = workflowScript.readFile(file: configPath)
46
+    def headers = [[name: "content-type", value: getConfigMediaType()]]
47
+    def response = workflowScript.httpRequest(url: getRequestURL(variant),
48
+                                              httpMode: "POST",
49
+                                              customHeaders: headers,
50
+                                              requestBody: config,
51
+                                              consoleLogResponseBody: true,
52
+                                              validResponseCodes: "200")
39 53
 
40
-    def cmd = "blubber ${arg(configPath)} ${arg(variant)} | " +
41
-              "docker build --pull ${labelFlags} --file - ."
54
+    response.content
55
+  }
42 56
 
43
-    def output = workflowScript.sh(returnStdout: true, script: cmd)
57
+  /**
58
+   * Returns a request media type based on the config file extension.
59
+   */
60
+  String getConfigMediaType() {
61
+    def ext = configPath.substring(configPath.lastIndexOf(".") + 1)
62
+
63
+    switch (ext) {
64
+      case ["yaml", "yml"]:
65
+        return "application/yaml"
66
+      default:
67
+        return "application/json"
68
+    }
69
+  }
44 70
 
45
-    // Return just the image ID from `docker build` output
46
-    output.substring(output.lastIndexOf(" ") + 1).trim()
71
+  /**
72
+   * Return a request URL for the given variant.
73
+   */
74
+  String getRequestURL(String variant) {
75
+    blubberoidURL + URLEncoder.encode(variant, "UTF-8")
47 76
   }
48 77
 }

+ 16
- 2
src/org/wikimedia/integration/PipelineRunner.groovy View File

@@ -25,6 +25,11 @@ class PipelineRunner implements Serializable {
25 25
   def blubberConfig = "blubber.yaml"
26 26
 
27 27
   /**
28
+   * Base URL for the Blubberoid service.
29
+   */
30
+  def blubberoidURL = "https://blubberoid.wikimedia.org/v1/"
31
+
32
+  /**
28 33
    * Directory in which pipeline configuration is stored.
29 34
    */
30 35
   def configPath = ".pipeline"
@@ -95,9 +100,18 @@ class PipelineRunner implements Serializable {
95 100
       throw new FileNotFoundException("failed to build image: no Blubber config found at ${cfg}")
96 101
     }
97 102
 
98
-    def blubber = new Blubber(workflowScript, cfg)
103
+    def blubber = new Blubber(workflowScript, cfg, blubberoidURL)
104
+    def dockerfile = getConfigFile("Dockerfile")
105
+
106
+    workflowScript.writeFile(text: blubber.generateDockerfile(variant), file: dockerfile)
107
+
108
+    def labelFlags = labels.collect { k, v -> "--label ${arg(k + "=" + v)}" }.join(" ")
109
+    def dockerBuild = "docker build --pull ${labelFlags} --file ${arg(dockerfile)} ."
110
+
111
+    def output = workflowScript.sh(returnStdout: true, script: dockerBuild)
99 112
 
100
-    blubber.build(variant, labels)
113
+    // Return just the image ID from `docker build` output
114
+    output.substring(output.lastIndexOf(" ") + 1).trim()
101 115
   }
102 116
 
103 117
   /**

+ 46
- 12
test/org/wikimedia/integration/BlubberTest.groovy View File

@@ -6,25 +6,59 @@ import org.wikimedia.integration.Blubber
6 6
 class BlubberTestCase extends GroovyTestCase {
7 7
   private class WorkflowScript {} // Mock for Jenkins Pipeline workflow context
8 8
 
9
-  void testBuildCommand() {
9
+  def blubberConfig = ".pipeline/blubber.yaml"
10
+  def blubberoidURL = "https://an.example/blubberoid/v1/"
11
+
12
+  void testGenerateDockerfile() {
10 13
     def mock = new MockFor(WorkflowScript)
14
+    def config = "version: v3\n" +
15
+                 "base: foo"
16
+
17
+    mock.demand.readFile { args ->
18
+      assert args.file == blubberConfig
19
+
20
+      config
21
+    }
11 22
 
12
-    mock.demand.sh { args ->
13
-      assert args.returnStdout
14
-      assert args.script == "blubber 'foo/blubber.yaml' 'foo' | " +
15
-                            "docker build --pull --label 'foo=a' --label 'bar=b' --file - ."
23
+    mock.demand.httpRequest { args ->
24
+      assert args.httpMode == "POST"
25
+      assert args.customHeaders == [[name: "content-type", value: "application/yaml"]]
26
+      assert args.requestBody == config
27
+      assert args.consoleLogResponseBody == true
28
+      assert args.validResponseCodes == "200"
16 29
 
17
-      // Mock `docker build` output to test that we correctly parse the image ID
18
-      return "Removing intermediate container foo\n" +
19
-             " ---> bf1e86190382\n" +
20
-             "Successfully built bf1e86190382\n"
30
+      [content: "BASE foo\n"]
21 31
     }
22 32
 
23 33
     mock.use {
24
-      def blubber = new Blubber(new WorkflowScript(), "foo/blubber.yaml")
25
-      def imageID = blubber.build("foo", [foo: "a", bar: "b"])
34
+      def blubber = new Blubber(new WorkflowScript(), blubberConfig, blubberoidURL)
35
+      def dockerfile = blubber.generateDockerfile("foo")
26 36
 
27
-      assert imageID == "bf1e86190382"
37
+      assert dockerfile == "BASE foo\n"
28 38
     }
29 39
   }
40
+
41
+  void testGetConfigMediaType_yaml() {
42
+    def blubber = new Blubber(new WorkflowScript(), blubberConfig, blubberoidURL)
43
+
44
+    assert blubber.getConfigMediaType() == "application/yaml"
45
+  }
46
+
47
+  void testGetConfigMediaType_yml() {
48
+    def blubber = new Blubber(new WorkflowScript(), blubberConfig, blubberoidURL)
49
+
50
+    assert blubber.getConfigMediaType() == "application/yaml"
51
+  }
52
+
53
+  void testGetConfigMediaType_json() {
54
+    def blubber = new Blubber(new WorkflowScript(), ".pipeline/blubber.json", blubberoidURL)
55
+
56
+    assert blubber.getConfigMediaType() == "application/json"
57
+  }
58
+
59
+  void testGetRequestURL() {
60
+    def blubber = new Blubber(new WorkflowScript(), blubberConfig, blubberoidURL)
61
+
62
+    assert blubber.getRequestURL("foo bar") == "https://an.example/blubberoid/v1/foo+bar"
63
+  }
30 64
 }

+ 20
- 5
test/org/wikimedia/integration/PipelineRunnerTest.groovy View File

@@ -71,23 +71,38 @@ class PipelineRunnerTest extends GroovyTestCase {
71 71
     }
72 72
   }
73 73
 
74
-  void testBuild_delegatesToBlubber() {
74
+  void testBuild_generatesDockerfileAndBuilds() {
75 75
     def mockWorkflow = new MockFor(WorkflowScript)
76 76
     def mockBlubber = new MockFor(Blubber)
77 77
 
78 78
     mockWorkflow.demand.fileExists { true }
79
-    mockBlubber.demand.build { variant, labels ->
79
+
80
+    mockBlubber.demand.generateDockerfile { variant ->
80 81
       assert variant == "foo"
81
-      assert labels == [bar: "baz"]
82 82
 
83
-      "fooimageID"
83
+      "BASE: foo\n"
84
+    }
85
+
86
+    mockWorkflow.demand.writeFile { args ->
87
+      assert args.text == "BASE: foo\n"
88
+      assert args.file == ".pipeline/Dockerfile"
89
+    }
90
+
91
+    mockWorkflow.demand.sh { args ->
92
+      assert args.returnStdout
93
+      assert args.script == "docker build --pull --label 'foo=a' --label 'bar=b' --file '.pipeline/Dockerfile' ."
94
+
95
+      // Mock `docker build` output to test that we correctly parse the image ID
96
+      return "Removing intermediate container foo\n" +
97
+             " ---> bf1e86190382\n" +
98
+             "Successfully built bf1e86190382\n"
84 99
     }
85 100
 
86 101
     mockWorkflow.use {
87 102
       mockBlubber.use {
88 103
         def runner = new PipelineRunner(new WorkflowScript())
89 104
 
90
-        runner.build("foo", [bar: "baz"])
105
+        assert runner.build("foo", [foo: "a", bar: "b"]) == "bf1e86190382"
91 106
       }
92 107
     }
93 108
   }

Loading…
Cancel
Save