build for ppc64le architecture via gitlab ci

- automatically build and deploy gitlab helper image
- create debian package for ppc64el

.gitlab-ci.yml

 stages:
-- prepare
-- prebuild
-- test
-- coverage
-- build
-- package
-- release
-- cleanup
+  - build
+  - deploy
 
-variables:
-  CI_IMAGE: registry.gitlab.com/gitlab-org/gitlab-runner/ci:1.8.7-0
-  GOROOT: /usr/local/go
-
-before_script:
-  - unset GPG_KEY
-
-image: $CI_IMAGE
-
-# Skip all jobs except the ones that begin and end with 'docs'.
-# Used for commits including ONLY documentation changes.
-# https://docs.gitlab.com/ce/development/writing_documentation.html#testing
-.except_docs: &except_docs
-  except:
-  - /(^docs[\/-].*|.*-docs$)/
-
-.docker: &docker
-  <<: *except_docs
-  services:
-  - docker:dind
-  variables: &docker_variables
-    CI_IMAGE: registry.gitlab.com/gitlab-org/gitlab-runner/ci:1.8.7-0
-    DOCKER_DRIVER: overlay2
-    DOCKER_HOST: tcp://docker:2375
-  tags:
-  - docker
-
-#
-# Prepare stage
-#
-
-.image_builder: &image_builder
-  <<: *docker
-  stage: prepare
-  image: docker:git
-  script:
-  - source ./ci/build_ci_image
-
-prepare ci image:
-  <<: *image_builder
-  variables:
-    BUILD_IMAGE: $CI_IMAGE
-    BUILD_DOCKERFILE: ./dockerfiles/ci/Dockerfile
-  only:
-  - /\Adocker-ci-image\/.*/
-
-prepare alpine-no-root image:
-  <<: *image_builder
-  variables:
-    BUILD_IMAGE: registry.gitlab.com/gitlab-org/gitlab-runner/alpine-no-root:latest
-    BUILD_DOCKERFILE: ./tests/dockerfiles/alpine-no-root/Dockerfile
-  only:
-  - /\Adocker-alpine-no-root-image\/.*/
-
-#
-# Prebuild stage
-#
-
-helper images:
-  <<: *docker
-  stage: prebuild
-  script:
-  - make docker
-  artifacts:
-    paths:
-    - out/helper-images/
-    - .gopath/bin
-    - .gopath/.ok
-    expire_in: 7d
-
-clone test repo:
-  <<: *except_docs
-  stage: prebuild
-  script:
-  - mkdir tmp
-  - succeed=0
-  - for i in {1..3}; do git clone https://gitlab.com/gitlab-org/gitlab-test.git tmp/gitlab-test && succeed=1 && break; echo "retrying"; done
-  - '[[ "$succeed" -eq 1 ]]'
-  artifacts:
-    paths:
-    - tmp/gitlab-test
-    expire_in: 7d
-
-# Review docs base
-.review-docs: &review-docs
-  image: ruby:2.4-alpine
-  before_script:
-    - unset GPG_KEY
-    - gem install gitlab --no-doc
-    # We need to download the script rather than clone the repo since the
-    # review-docs-cleanup job will not be able to run when the branch gets
-    # deleted (when merging the MR).
-    - apk add --update openssl
-    - wget https://gitlab.com/gitlab-org/gitlab-ce/raw/master/scripts/trigger-build-docs
-    - chmod 755 trigger-build-docs
-  cache: {}
-  dependencies: []
-  variables:
-    GIT_STRATEGY: none
-  when: manual
-  only:
-    - branches
-
-# Trigger a docs build in gitlab-docs
-# Useful to preview the docs changes live
-# https://docs.gitlab.com/ee/development/writing_documentation.html#previewing-the-changes-live
-review-docs-deploy:
-  <<: *review-docs
-  stage: prebuild
-  environment:
-    name: review-docs/$CI_COMMIT_REF_NAME
-    # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are secret variables
-    # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693
-    url: http://$DOCS_GITLAB_REPO_SUFFIX-$CI_COMMIT_REF_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX
-    on_stop: review-docs-cleanup
-  script:
-    - ./trigger-build-docs deploy
-
-# Cleanup remote environment of gitlab-docs
-review-docs-cleanup:
-  <<: *review-docs
-  stage: cleanup
-  environment:
-    name: review-docs/$CI_COMMIT_REF_NAME
-    action: stop
-  script:
-    - ./trigger-build-docs cleanup
-
-tests definitions:
-  <<: *except_docs
-  stage: prebuild
-  # Using Go 1.10, because the `go test -list` is unavailable in <1.9
-  # and it's much faster in 1.10. And since there is no real test execution
-  # in this step, the version difference doesn't matter
-  image: golang:1.10
+build-ppc64le:
+  image: golang:1.11.1
+  stage: build
   script:
-  - apt update
-  - apt install make
-  - source ci/touch_make_dependencies
-  - make parallel_test_prepare
+    - make deps
+    - make BUILD_PLATFORMS="-osarch='linux/ppc64le'" build
   artifacts:
     paths:
-    - testsdefinitions.txt
-    expire_in: 7d
-  tags:
-  - gitlab-org
-
-#
-# Test stage
-#
+      - out/binaries/
 
-code_quality:
-  <<: *except_docs
-  stage: test
-  image: docker:git
+helper-image-ppc64le:
+  image: ubuntu:bionic
+  stage: build
   services:
     - docker:dind
   variables:
-    DOCKER_DRIVER: overlay2
-    CODECLIMATE_FORMAT: json
-  cache: {}
-  dependencies: []
-  script:
-    - ./scripts/codequality analyze -f json --dev | tee gl-code-quality-report.json
-  artifacts:
-    paths: [gl-code-quality-report.json]
-    expire_in: 7d
-
-.unit_tests: &unit_tests
-  <<: *docker
-  stage: test
-  script:
-    - JOB_NAME=( $CI_JOB_NAME )
-    - export SUITE_INDEX=${JOB_NAME[-2]}
-    - export SUITE_TOTAL=${JOB_NAME[-1]}
-    - source ci/touch_make_dependencies
-    - make parallel_test_execute
-  artifacts:
-    paths:
-      - .cover/*
-      - .testoutput/*
-    when: always
-    expire_in: 7d
-
-.unit_tests_with_race: &unit_tests_with_race
-  <<: *unit_tests
-  allow_failure: true
-  variables:
-    <<: *docker_variables
-    CGO_ENABLED: "1"
-    TESTFLAGS: "-cover -race"
-
-unit tests 0 5: *unit_tests
-unit tests 1 5: *unit_tests
-unit tests 2 5: *unit_tests
-unit tests 3 5: *unit_tests
-unit tests 4 5: *unit_tests
-
-unit tests with race 0 5: *unit_tests_with_race
-unit tests with race 1 5: *unit_tests_with_race
-unit tests with race 2 5: *unit_tests_with_race
-unit tests with race 3 5: *unit_tests_with_race
-unit tests with race 4 5: *unit_tests_with_race
-
-docs check links:
-  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
-  stage: test
-  cache: {}
-  dependencies: []
-  before_script:
-    - unset GPG_KEY
-  script:
-    - mv docs/ /nanoc/content/
-    - cd /nanoc
-    # Build HTML from Markdown
-    - bundle exec nanoc
-    # Check the internal links
-    - bundle exec nanoc check internal_links
-
-integration windows:
-  <<: *except_docs
-  stage: test
+    DOCKER_DRIVER: vfs
   before_script:
-  - $GODIR = "$env:GOPATH\src\gitlab.com\gitlab-org"
-  - $target_path = "$GODIR\gitlab-runner"
-  - $current_path = pwd
-  - md $GODIR -Force
-  - if (Test-Path $target_path) { cmd /c rmdir $target_path }
-  - cmd /c mklink /D $target_path $current_path
-  - cd $GODIR\gitlab-runner
-  script:
-  - $packages = (go list ./... | findstr -v "\/vendor\/" | Out-String | %{$_ -replace "\r\n"," "})
-  - $testCommand = "go test $packages"
-  - Invoke-Expression $testCommand
-  - if ( $LASTEXITCODE -ne 0 ) { exit $LASTEXITCODE }
-  allow_failure: true
-  only:
-  - /.*/@gitlab-org/gitlab-runner
+    - apt-get -qy update
+    - apt-get install -qy make xz-utils
+    - apt-get install -qy apt-transport-https ca-certificates curl gnupg2 software-properties-common
+    # install docker-ce
+    - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+    - add-apt-repository -y "deb [arch=ppc64el] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    - apt-get -qy update && apt-get -qy install docker-ce=18.06\*
+    - dockerd &
+    # install golang
+    - curl https://dl.google.com/go/go1.11.1.linux-ppc64le.tar.gz --output go.1.11.1.tar.gz -s
+    - tar -C /usr/local -xzf go.1.11.1.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+  script:
+    - make deps
+    - make docker-ppc64le
   tags:
-  - integration
-  - windows
-
-git 1.7.1:
-  <<: *except_docs
-  image: centos:6
-  stage: test
-  variables:
-    GOROOT: /usr/local/go
-    GOPATH: /go
-    CGO_ENABLED: 0
-    OUR_PACKAGES: gitlab.com/gitlab-org/gitlab-runner/executors/shell gitlab.com/gitlab-org/gitlab-runner/shells
-  before_script:
-    - unset GPG_KEY
-    - yum install -y -q git && git version
-    - curl https://storage.googleapis.com/golang/go1.8.7.linux-amd64.tar.gz | tar zx -C $(dirname "$GOROOT")
-    - export PATH=${GOROOT}/bin:$PATH
-    - source ci/touch_make_dependencies
-  script:
-    - make test
-
-#
-# Coverage stage
-#
-
-test coverage report: &test_coverage_report
-  <<: *except_docs
-  stage: coverage
-  image: golang:1.10
-  coverage: /regular total:\s+\(statements\)\s+\d+.\d+\%/
-  script:
-    - source ci/touch_make_dependencies
-    - make parallel_test_coverage_report
-    - make parallel_test_coverage_report TESTFLAGS="-cover -race"
+    - ppc64le
   artifacts:
     paths:
-      - out/coverage/*
-    expire_in: 7d
+      - out/helper-images
 
-race conditions detector:
-  <<: *except_docs
-  stage: coverage
-  image: golang:1.10
-  script:
-  - source ci/touch_make_dependencies
-  - make check_race_conditions
-
-#
-# Build stage
-#
-
-.binaries: &binaries
-  <<: *except_docs
-  stage: build
+deploy-helper-image:
+  image: docker:latest
+  stage: deploy
+  services:
+    - docker:dind
+  dependencies:
+    - helper-image-ppc64le
+  before_script:
+    - apk add git
+    - export REVISION=$(git rev-parse --short=8 HEAD || echo "unknown")
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN gitlab.hzdr.de:5005
   script:
-  - source ci/touch_make_dependencies
-  - export platforms=$(echo $CI_JOB_NAME | sed 's|binaries ||')
-  - make build BUILD_PLATFORMS="-osarch='$platforms'"
-  artifacts:
-    paths:
-    - out/binaries/
-    expire_in: 7d
-  tags:
-  - gitlab-org
-
-binaries darwin/386 darwin/amd64: *binaries
-binaries freebsd/386 freebsd/amd64 freebsd/arm: *binaries
-binaries linux/386 linux/amd64 linux/arm: *binaries
-binaries windows/386 windows/amd64: *binaries
-
-#
-# Package stage
-#
+    - cat out/helper-images/prebuilt-ppc64le.tar.xz | docker import - gitlab.hzdr.de:5005/fwcc/gitlab-runner/helper-image:ppc64le-$REVISION
+    - docker push gitlab.hzdr.de:5005/fwcc/gitlab-runner/helper-image:ppc64le-$REVISION
+  only:
+    - /^v.*$/
 
-.packages: &packages
-  <<: *except_docs
-  stage: package
+deploy-deb-package:
+  image: golang:1.11.1
+  stage: deploy
+  dependencies:
+    - build-ppc64le
+    - helper-image-ppc64le
   before_script:
-  - |
-    # checking GPG signing support
-    if [ -n "$GPG_KEY" ]; then
-      echo "$GPG_KEY" | gpg --batch --no-tty --allow-secret-key-import --import -
-      unset GPG_KEY
-      export GPG_KEYID=$(gpg --with-colon --list-secret-keys | head -n1 | cut -d : -f 5)
-    else
-      echo -e "\033[0;31m****** GPG signing disabled ******\033[0m"
-    fi
+    - apt-get -qy update
+    - apt-get -qy install locales make xz-utils ruby ruby-dev python-pip dpkg-sig createrepo rpm zip libffi-dev
   script:
-  - source ci/touch_make_dependencies
-  - make ${CI_JOB_NAME}
+    - make package-deb-ppc64le
   artifacts:
     paths:
-    - out/deb/
-    - out/rpm/
-    expire_in: 7d
-  tags:
-  - gitlab-org
-
-package-deb: *packages
-package-rpm: *packages
-
-#
-# Release stage
-#
-
-.release_docker_images: &release_docker_images
-  <<: *docker
-  stage: release
-  variables: &release_docker_images_variables
-    <<: *docker_variables
-    PUBLISH_IMAGES: "true"
-    PUSH_TO_DOCKER_HUB: "true"
-    DOCKER_MACHINE_VERSION: "0.14.0"
-    DOCKER_MACHINE_CHECKSUM: "a4c69bffb78d3cfe103b89dae61c3ea11cc2d1a91c4ff86e630c9ae88244db02"
-    DUMB_INIT_VERSION: "1.0.2"
-    DUMB_INIT_CHECKSUM: "a8defac40aaca2ca0896c7c5adbc241af60c7c3df470c1a4c469a860bd805429"
-  script:
-  - source ci/touch_make_dependencies
-  - make release_docker_images
-  tags:
-  - release
-
-.release: &release
-  <<: *except_docs
-  stage: release
-  before_script:
-  - unset GPG_KEY
-  - source ci/touch_make_dependencies
-  tags:
-  - release
-
-.release_s3: &release_s3
-  <<: *release
-  script:
-  - make release_s3
-
-.release_packagecloud: &release_packagecloud
-  <<: *release
-  script:
-  - make release_packagecloud
-
-.release_development: &release_development
-  only:
-  - branches@gitlab-org/gitlab-runner
-  except:
-  - master@gitlab-org/gitlab-runner
-  - /(^docs[\/-].*|.*-docs$)/
-
-.release_bleeding_edge: &release_bleeding_edge
-  only:
-  - master@gitlab-org/gitlab-runner
-  - /\Av[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+\Z/@gitlab-org/gitlab-runner
-
-.release_stable: &release_stable
+      - out/deb/
   only:
-  - /\Av[0-9]+\.[0-9]+\.[0-9]+\Z/@gitlab-org/gitlab-runner
-
-development S3:
-  <<: *release_s3
-  <<: *release_development
-  environment:
-    name: development/s3/${CI_COMMIT_REF_NAME}
-    url: https://gitlab-runner-downloads.s3.amazonaws.com/${CI_COMMIT_REF_NAME}/index.html
-
-development docker images:
-  <<: *release_docker_images
-  variables:
-    <<: *release_docker_images_variables
-    PUBLISH_IMAGES: "false"
-    PUSH_TO_DOCKER_HUB: "false"
-  only:
-  - branches
-  except:
-  - master
-  - /(^docs[\/-].*|.*-docs$)/
-  tags:
-  - docker
-
-bleeding edge S3:
-  <<: *release_s3
-  <<: *release_bleeding_edge
-  environment:
-    name: bleeding_edge/s3
-    url: https://gitlab-runner-downloads.s3.amazonaws.com/${CI_COMMIT_REF_NAME}/index.html
-
-bleeding edge packagecloud:
-  <<: *release_packagecloud
-  <<: *release_bleeding_edge
-  environment:
-    name: bleeding_edge/packagecloud
-    url: https://packages.gitlab.com/runner/unstable
-
-bleeding edge docker images:
-  <<: *release_docker_images
-  <<: *release_bleeding_edge
-  environment:
-    name: bleeding_edge/docker_images
-    url: https://hub.docker.com/r/gitlab/gitlab-runner/tags/
-
-stable S3:
-  <<: *release_s3
-  <<: *release_stable
-  environment:
-    name: stable/s3
-    url: https://gitlab-runner-downloads.s3.amazonaws.com/${CI_COMMIT_REF_NAME}/index.html
-
-stable packagecloud:
-  <<: *release_packagecloud
-  <<: *release_stable
-  environment:
-    name: stable/packagecloud
-    url: https://packages.gitlab.com/runner/gitlab-runner
-
-stable docker images:
-  <<: *release_docker_images
-  <<: *release_stable
-  environment:
-    name: stable/docker_images
-    url: https://hub.docker.com/r/gitlab/gitlab-runner/tags/
-
-static QA:
-  <<: *except_docs
-  stage: release
-  image: alpine:3.7
-  cache: {}
-  dependencies:
-  - code_quality
-  script: |
-    if [ "$(cat gl-code-quality-report.json)" != "[]" ] ; then
-      apk add -U --no-cache jq > /dev/null
-      jq -C . gl-code-quality-report.json
-      exit 1
-    fi
+    - /^v.*$/

Makefile

@@ -199,6 +199,9 @@ package-deb: package-deps package-prepare
 	make package-deb-fpm ARCH=arm PACKAGE_ARCH=armel
 	make package-deb-fpm ARCH=arm PACKAGE_ARCH=armhf
 
+package-deb-ppc64le: package-deps package-prepare
+	make package-deb-fpm ARCH=ppc64le PACKAGE_ARCH=ppc64el
+
 package-rpm: package-deps package-prepare
 	# Building RedHat compatible packages...
 	make package-rpm-fpm ARCH=amd64 PACKAGE_ARCH=amd64

Makefile.docker.mk

 ifeq (, $(shell docker info 2>/dev/null))
-USE_PRECOMPILED_IMAGES ?= 1
+USE_PRECOMPILED_IMAGES ?= 0
 endif
 
 docker: out/helper-images/prebuilt-x86_64.tar.xz out/helper-images/prebuilt-arm.tar.xz
+docker-ppc64le: out/helper-images/prebuilt-ppc64le.tar.xz
 
 ifeq (, $(USE_PRECOMPILED_IMAGES))
 HELPER_GO_FILES ?= $(shell find common network vendor -name '*.go')
 
 GO_x86_64_ARCH = amd64
 GO_arm_ARCH = arm
+GO_ppc64le_ARCH = ppc64le
 
 dockerfiles/build/binaries/gitlab-runner-helper.%: $(HELPER_GO_FILES) $(GOX)
 	gox -osarch=linux/$(GO_$*_ARCH) -ldflags "$(GO_LDFLAGS)" -output=$@ $(PKG)/apps/gitlab-runner-helper

dockerfiles/build/Dockerfile.ppc64le

+FROM ppc64le/alpine:3.8
+
+RUN apk add --no-cache bash ca-certificates git git-lfs miniperl \
+	&& ln -s miniperl /usr/bin/perl
+
+RUN git lfs install
+
+COPY ./scripts/ /usr/bin
+COPY ./binaries/gitlab-runner-helper.ppc64le /usr/bin/gitlab-runner-helper
+
+RUN echo 'hosts: files dns' >> /etc/nsswitch.conf