You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

90 lines
2.6 KiB

pipeline: Builder and stage implementation Provides `PipelineBuilder` for reading `.pipeline/config.yaml` and mapping user-defined pipelines, stages, and execution graphs to actual Jenkins Pipeline stage definitions. Provides `Pipeline` class that constructs a "stack" of `PipelineStage` objects from the user-provided configs, each with its own `NodeContext` for binding output values to names and consuming bound values from previous stages. Provides `PipelineStage` that contains core stage step implementations based on the existing `service-pipeline` JJB job definition in `integration/config`. A closure is returned by each stage for passing off to Jenkins Pipeline stage definitions by the builder. Steps have a fixed order within a given stage: build, run, publish, deploy, exports. This allows for concise definition of a stage that performs multiple steps, and deterministic behavior of default configuration that references locally bound output values (e.g. the default configuration of `image:` for an `publish: { type: image }` publish entry is `${.imageID}`, referencing the image built in the current stage's `build` step.) If the user needs to change ordering, they can simply break the stage out into multiple stages. See the `Pipeline` class for currently supported configuration. Note that the aforementioned context system allows for user's to make use of the same value bindings that step implementations use internally. They can also use the `exports` configuration field to bind new values. To illustrate the minimally required configuration, the following would approximate the current `service-pipeline-test-and-publish` JJB job for a project named "foo". pipelines: foo: directory: src/foo stages: - name: test # builds/runs "test" variant - name: candidate build: production publish: image: true deploy: # currently only the "ci" cluster chart: https://releases.wikimedia.org/charts/foo-0.0.1.tgz test: true And to illustrate how the "candidate" stage in this example could be expressed as multiple stages using references to the output names that steps bind/export: pipelines: foo: directory: src/foo stages: - name: tested - name: built build: production - name: published publish: image: id: '${built.imageID}' exports: image: '${.imageFullName}:${.imageTag}' - name: staged deploy: image: '${published.image}' chart: https://releases.wikimedia.org/charts/foo-0.0.1.tgz test: true Bug: T210267 Change-Id: I5a41d0d33ed7e9174db6178ab7921f5143296c75
5 years ago
  1. SHELL := /bin/bash
  2. GRADLE := $(shell command -v gradle)
  3. BLUBBER := $(shell command -v blubber)
  4. DOCKER := $(shell command -v docker)
  5. DOCKER_TAG := piplinelib-tests-$(shell date -u +%Y%m%d-%H%M%S)
  6. DOCKER_LABEL := wmf.gc=pipelinelib-tests
  7. DOCKER_BUILD := docker build --label $(DOCKER_LABEL) --tag $(DOCKER_TAG)
  8. DOCKER_RUN := docker run --rm --label $(DOCKER_LABEL) --name $(DOCKER_TAG)
  9. DOCKER_STOP := docker stop "$(DOCKER_TAG)"
  10. DOCKER_STOP_ALL = docker stop $(shell docker ps -qf label=$(DOCKER_LABEL))
  11. DOCKER_RMI = docker rmi $(shell docker images -qf label=$(DOCKER_LABEL))
  12. .PHONY: test
  13. clean:
  14. ifneq (,$(DOCKER))
  15. $(DOCKER_STOP_ALL) 2> /dev/null || true
  16. $(DOCKER_RMI) 2> /dev/null || true
  17. else
  18. @echo "Not using Docker. Nothing to do."
  19. endif
  20. doc: docs
  21. docs:
  22. gradle groovydoc
  23. test:
  24. ifneq (,$(GRADLE))
  25. gradle test
  26. @exit 0
  27. else ifneq (,$(and $(BLUBBER), $(DOCKER)))
  28. blubber .pipeline/blubber.yaml test | docker build -t "$(DOCKER_TAG)" -f - .
  29. docker run --rm -it "$(DOCKER_TAG)"
  30. @exit 0
  31. else
  32. @echo "Can't find Gradle or Blubber/Docker. Install one to run tests."
  33. @exit 1
  34. endif
  35. systemtest:
  36. ifneq (,$(and $(DOCKER), $(DOCKER_HOST)))
  37. $(eval JENKINS_HOST := $(patsubst tcp://%,%,$(DOCKER_HOST)))
  38. $(eval JENKINS_HOST := $(word 1, $(subst :, ,$(JENKINS_HOST))))
  39. $(eval JENKINS_URL := http://docker:docker@$(JENKINS_HOST):8080)
  40. $(eval JENKINS_BLUE_URL := $(JENKINS_URL)/blue/organizations/jenkins)
  41. $(eval BUILD_OUTPUT := $(shell mktemp))
  42. $(DOCKER_BUILD) -f systemtests/jenkins/Dockerfile .
  43. $(DOCKER_RUN) -d \
  44. -p 8080:8080 \
  45. -v /var/run/docker.sock:/var/run/docker.sock \
  46. $(DOCKER_TAG)
  47. @while ! curl -s http://$(JENKINS_HOST):8080/ > /dev/null; do \
  48. echo "waiting for jenkins..."; \
  49. sleep 1; \
  50. done
  51. @while curl -s http://$(JENKINS_HOST):8080/ | grep -q "is getting ready to work"; do \
  52. echo "waiting for jenkins..."; \
  53. sleep 1; \
  54. done
  55. curl -X POST $(JENKINS_URL)/job/repo1/build
  56. @echo "Build $(JENKINS_URL)/job/repo1/1 created"
  57. @while curl -sw %%{http_code} $(JENKINS_URL)/job/repo1/1/api/json | grep -q '404'; do \
  58. echo "waiting for build to start..."; \
  59. sleep 1; \
  60. done
  61. @while curl -s $(JENKINS_URL)/job/repo1/1/api/json | grep -q '"building":true'; do \
  62. sleep 1; \
  63. curl -s $(JENKINS_URL)/job/repo1/1/consoleText | \
  64. tail -n +$$(wc -l $(BUILD_OUTPUT) | awk '{ print $$1 }') | \
  65. tee -a $(BUILD_OUTPUT); \
  66. done
  67. ifeq (1,$(DEBUG))
  68. @echo "DEBUG: Build $(JENKINS_URL)/job/repo1/1 completed"
  69. @echo -n "DEBUG: Press <enter> to continue: "
  70. @read
  71. endif
  72. $(DOCKER_STOP)
  73. else
  74. @echo "Can't find Docker and your DOCKER_HOST. Set up both to run system tests."
  75. @exit 1
  76. endif