diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aac748b..060ffd3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,110 +1,157 @@ +# we support merge request workflow only +include: + - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' + default: - image: docker:git + image: docker:20.10.9-git services: - - docker:19.03.0-dind + - docker:20.10.9-dind before_script: - - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" "$CI_REGISTRY" + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - source scripts/build-helpers.sh + - apk add -U bash tags: - gitlab-org-docker stages: - - build - test + - deploy - automation variables: DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 - FORCE_BUILD: "false" + +# Run test job always +.test: + stage: test + rules: + - when: always + +# Run build job only on a default branch when triggered manually +.deploy: + stage: deploy + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual .test_build: &test_build - stage: test + extends: .test script: # Hack to set an array in /bin/sh # http://unix.stackexchange.com/a/137571 - set -- $CI_JOB_NAME - - docker build -f "Dockerfile.$1" . - except: - - master + - docker pull --quiet "$CI_REGISTRY_IMAGE:$1" || true + - docker build --cache-from="$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" . .build_and_deploy: &build_and_deploy - stage: build - script: - - docker build --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$CI_JOB_NAME" -f "Dockerfile.$CI_JOB_NAME" . - - docker push "$CI_REGISTRY_IMAGE:$CI_JOB_NAME" - only: - - master - -.test_custom: &test_custom - stage: test + extends: .deploy script: # Hack to set an array in /bin/sh # http://unix.stackexchange.com/a/137571 - set -- $CI_JOB_NAME - - ./scripts/custom-docker-build $1 - except: - - master + - docker pull --quiet "$CI_REGISTRY_IMAGE:$1" || true + - docker build --cache-from="$CI_REGISTRY_IMAGE:$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" . + - docker push "$CI_REGISTRY_IMAGE:$1" + +.test_custom: &test_custom + extends: .test + script: + # Hack to set an array in /bin/sh + # http://unix.stackexchange.com/a/137571 + - set -- $CI_JOB_NAME + - ./scripts/custom-docker-build "$1" + rules: + - when: always .build_and_deploy_custom: &build_and_deploy_custom - stage: build + extends: .deploy script: - - ./scripts/custom-docker-build $CI_JOB_NAME --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$CI_JOB_NAME" - - push_if_needed "$CI_REGISTRY_IMAGE:$CI_JOB_NAME" - only: - - master + # Hack to set an array in /bin/sh + # http://unix.stackexchange.com/a/137571 + - set -- $CI_JOB_NAME + - ./scripts/custom-docker-build "$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1" + - docker push "$CI_REGISTRY_IMAGE:$1" # Tests -# Used by GDK verify step: https://gitlab.com/gitlab-org/gitlab-development-kit -node-10 test: *test_custom - # Used by Gitaly: https://gitlab.com/gitlab-org/gitaly/blob/master/.gitlab-ci.yml -ruby-2.6-golang-1.14-git-2.28 test: *test_custom -ruby-2.6-golang-1.15-git-2.28 test: *test_custom -ruby-2.6-golang-1.15-git-2.28-pgbouncer-1.14 test: *test_custom -ruby-2.7-golang-1.14-git-2.28 test: *test_custom -ruby-2.7-golang-1.15-git-2.28 test: *test_custom -ruby-2.7-golang-1.15-git-2.28-pgbouncer-1.14 test: *test_custom -ruby-2.7-golang-1.14-git-2.29 test: *test_custom -ruby-2.7-golang-1.15-git-2.29 test: *test_custom -ruby-2.7-golang-1.15-git-2.29-pgbouncer-1.14 test: *test_custom +ruby-2.7-golang-1.16-git-2.31 test: *test_custom +ruby-2.7-golang-1.16-git-2.31-pgbouncer-1.14 test: *test_custom +ruby-2.7-golang-1.17-git-2.31 test: *test_custom +ruby-2.7-golang-1.17-git-2.31-pgbouncer-1.14 test: *test_custom +ruby-3.0-golang-1.16-git-2.31 test: *test_custom +ruby-3.0-golang-1.16-git-2.31-pgbouncer-1.14 test: *test_custom +ruby-3.0-golang-1.17-git-2.31 test: *test_custom +ruby-3.0-golang-1.17-git-2.31-pgbouncer-1.14 test: *test_custom +ruby-2.7-golang-1.16-git-2.33 test: *test_custom +ruby-2.7-golang-1.16-git-2.33-pgbouncer-1.14 test: *test_custom +ruby-2.7-golang-1.17-git-2.33 test: *test_custom +ruby-2.7-golang-1.17-git-2.33-pgbouncer-1.14 test: *test_custom +ruby-3.0-golang-1.16-git-2.33 test: *test_custom +ruby-3.0-golang-1.16-git-2.33-pgbouncer-1.14 test: *test_custom +ruby-3.0-golang-1.17-git-2.33 test: *test_custom +ruby-3.0-golang-1.17-git-2.33-pgbouncer-1.14 test: *test_custom + # Used by GitLab: https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml -ruby-2.7.2-golang-1.14-git-2.28-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-11-graphicsmagick-1.3.34 test: *test_custom -ruby-2.7.2-golang-1.14-git-2.28-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-12-graphicsmagick-1.3.34 test: *test_custom -ruby-2.7.2-golang-1.14-git-2.29-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-11-graphicsmagick-1.3.34 test: *test_custom -ruby-2.7.2-golang-1.14-git-2.29-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-12-graphicsmagick-1.3.34 test: *test_custom -ruby-3.0.0-golang-1.14-git-2.29-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-11-graphicsmagick-1.3.34 test: *test_custom -ruby-3.0.0-golang-1.14-git-2.29-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-12-graphicsmagick-1.3.34 test: *test_custom +ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom +ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom +ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom +ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom +ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom +ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom +ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom +ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom +ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom +ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom +ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom +ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom + + +# Used by GitLab/KAS integration tests +ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 test: *test_custom +ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 test: *test_custom +ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 test: *test_custom # Used by GitLab's compile-production-assets and compile-test-assets jobs -ruby-2.7.2-git-2.28-lfs-2.9-node-12.18-yarn-1.22-graphicsmagick-1.3.34 test: *test_custom -ruby-2.7.2-git-2.29-lfs-2.9-node-12.18-yarn-1.22-graphicsmagick-1.3.34 test: *test_custom -ruby-3.0.0-git-2.29-lfs-2.9-node-12.18-yarn-1.22-graphicsmagick-1.3.34 test: *test_custom - -# Used by www-gitlab-com's rspec job: https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/.gitlab-ci.yml -ruby-2.6.5-git-2.28-chrome-74.0-node-12.18-yarn-1.22-docker-19.03.1 test: *test_custom -ruby-2.6.5-git-2.28-chrome-85-node-12.18-yarn-1.22-docker-19.03.1 test: *test_custom -ruby-2.6.5-git-2.29-chrome-74.0-node-12.18-yarn-1.22-docker-19.03.1 test: *test_custom -ruby-2.6.5-git-2.29-chrome-85-node-12.18-yarn-1.22-docker-19.03.1 test: *test_custom -ruby-2.7.2-git-2.29-chrome-85-node-12.18-yarn-1.22-docker-19.03.1 test: *test_custom +ruby-2.7.2-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom +ruby-2.7-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom +ruby-3.0-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom +ruby-2.7.2-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom +ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom +ruby-3.0-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom # Used by customers-gitlab-com rspec job: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab-ci.yml -ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-9.5-graphicsmagick-1.3.34: *test_custom +ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom +ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1: *test_custom +ruby-2.7.2-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom +ruby-2.7-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom +ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom +ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1: *test_custom +ruby-2.7.2-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom +ruby-2.7-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom + +# Used by customers-gitlab-com qa e2e test: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab/ci/e2e.gitlab-ci.yml +ruby-2.7.2-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom +ruby-2.7-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom +ruby-2.7.2-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom +ruby-2.7-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom + +# Used by https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/blob/master/.gitlab-ci.yml +alpine-helm test: *test_build danger test: *test_build -alpine-aws test: *test_build -alpine-helm test: *test_build -ruby-alpine-aws test: *test_build +danger-ruby-2.6.6 test: *test_build gitlab-charts-build-base test: *test_build gitlab-charts-build-base-helm-3 test: *test_build -gitlab-helm3-kubectl1.14 test: *test_build +gitlab-helm3.5-kubectl1.17 test: *test_build gitlab-qa-ruby-2.7 test: *test_build gitlab-qa-ruby-3.0 test: *test_build gitlab-qa-alpine-ruby-2.7 test: *test_build gitlab-qa-alpine-ruby-3.0 test: *test_build gitlab-puppeteer test: *test_build +kaniko test: *test_build omnibus-gitlab-bionic test: *test_build omnibus-gitlab-centos7 test: *test_build omnibus-gitlab-centos8 test: *test_build @@ -121,124 +168,129 @@ omnibus-gitlab-cve-search test: *test_build release-tools test: *test_build sitespeed-gitlab test: *test_build ubi-release test: *test_build -www-gitlab-com-2.6 test: *test_build +www-gitlab-com-3.0 test: *test_build build-git: *test_build +# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml terraform test: *test_build # Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting golangci-lint-alpine test: *test_build # Used by gitlab-operator: https://gitlab.com/gitlab-org/gl-openshift/gitlab-operator gitlab-operator-build-base test: *test_build +alpine-bash-jq-curl-git test: *test_build + # Builds -# Used by GDK verify step: https://gitlab.com/gitlab-org/gitlab-development-kit -node-10: *build_and_deploy_custom - # Used by Gitaly: https://gitlab.com/gitlab-org/gitaly/blob/master/.gitlab-ci.yml -ruby-2.6-golang-1.14-git-2.28: *build_and_deploy_custom -ruby-2.6-golang-1.15-git-2.28: *build_and_deploy_custom -ruby-2.6-golang-1.15-git-2.28-pgbouncer-1.14: *build_and_deploy_custom -ruby-2.7-golang-1.14-git-2.28: *build_and_deploy_custom -ruby-2.7-golang-1.15-git-2.28: *build_and_deploy_custom -ruby-2.7-golang-1.15-git-2.28-pgbouncer-1.14: *build_and_deploy_custom -ruby-2.7-golang-1.14-git-2.29: *build_and_deploy_custom -ruby-2.7-golang-1.15-git-2.29: *build_and_deploy_custom -ruby-2.7-golang-1.15-git-2.29-pgbouncer-1.14: *build_and_deploy_custom +ruby-2.7-golang-1.16-git-2.31 push: *build_and_deploy_custom +ruby-2.7-golang-1.16-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom +ruby-2.7-golang-1.17-git-2.31 push: *build_and_deploy_custom +ruby-2.7-golang-1.17-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom +ruby-3.0-golang-1.16-git-2.31 push: *build_and_deploy_custom +ruby-3.0-golang-1.16-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom +ruby-3.0-golang-1.17-git-2.31 push: *build_and_deploy_custom +ruby-3.0-golang-1.17-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom +ruby-2.7-golang-1.16-git-2.33 push: *build_and_deploy_custom +ruby-2.7-golang-1.16-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom +ruby-2.7-golang-1.17-git-2.33 push: *build_and_deploy_custom +ruby-2.7-golang-1.17-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom +ruby-3.0-golang-1.16-git-2.33 push: *build_and_deploy_custom +ruby-3.0-golang-1.16-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom +ruby-3.0-golang-1.17-git-2.33 push: *build_and_deploy_custom +ruby-3.0-golang-1.17-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom # Used by GitLab CE/EE: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/.gitlab-ci.yml -ruby-2.7.2-golang-1.14-git-2.28-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-11-graphicsmagick-1.3.34: *build_and_deploy_custom -ruby-2.7.2-golang-1.14-git-2.28-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-12-graphicsmagick-1.3.34: *build_and_deploy_custom -ruby-2.7.2-golang-1.14-git-2.29-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-11-graphicsmagick-1.3.34: *build_and_deploy_custom -ruby-2.7.2-golang-1.14-git-2.29-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-12-graphicsmagick-1.3.34: *build_and_deploy_custom -ruby-3.0.0-golang-1.14-git-2.29-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-11-graphicsmagick-1.3.34: *build_and_deploy_custom -ruby-3.0.0-golang-1.14-git-2.29-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-12-graphicsmagick-1.3.34: *build_and_deploy_custom +ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom + +# Used by GitLab/KAS integration tests +ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom +ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom +ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom +ruby-2.7.2.patched-golang-1.16-git-2.33-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom +ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom +ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom # Used by GitLab's compile-production-assets and compile-test-assets jobs -ruby-2.7.2-git-2.28-lfs-2.9-node-12.18-yarn-1.22-graphicsmagick-1.3.34: *build_and_deploy_custom -ruby-2.7.2-git-2.29-lfs-2.9-node-12.18-yarn-1.22-graphicsmagick-1.3.34: *build_and_deploy_custom -ruby-3.0.0-git-2.29-lfs-2.9-node-12.18-yarn-1.22-graphicsmagick-1.3.34: *build_and_deploy_custom - -# Used by www-gitlab-com's rspec job: https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/.gitlab-ci.yml -ruby-2.6.5-git-2.28-chrome-74.0-node-12.18-yarn-1.22-docker-19.03.1: *build_and_deploy_custom -ruby-2.6.5-git-2.28-chrome-85-node-12.18-yarn-1.22-docker-19.03.1: *build_and_deploy_custom -ruby-2.6.5-git-2.29-chrome-74.0-node-12.18-yarn-1.22-docker-19.03.1: *build_and_deploy_custom -ruby-2.6.5-git-2.29-chrome-85-node-12.18-yarn-1.22-docker-19.03.1: *build_and_deploy_custom -ruby-2.7.2-git-2.29-chrome-85-node-12.18-yarn-1.22-docker-19.03.1: *build_and_deploy_custom +ruby-2.7-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-3.0-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom +ruby-3.0-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom # Used by customers-gitlab-com rspec job: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab-ci.yml -ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-9.5-graphicsmagick-1.3.34: *build_and_deploy_custom +ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom +ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1 push: *build_and_deploy_custom +ruby-2.7.2-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom +ruby-2.7-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom +ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom +ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1 push: *build_and_deploy_custom +ruby-2.7.2-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom +ruby-2.7-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom -danger: *build_and_deploy -alpine-aws: *build_and_deploy -alpine-helm: *build_and_deploy -ruby-alpine-aws: *build_and_deploy -gitlab-charts-build-base: *build_and_deploy -gitlab-charts-build-base-helm-3: *build_and_deploy -gitlab-helm3-kubectl1.14: *build_and_deploy -gitlab-qa-ruby-2.7: *build_and_deploy -gitlab-qa-ruby-3.0: *build_and_deploy -gitlab-qa-alpine-ruby-2.7: *build_and_deploy -gitlab-qa-alpine-ruby-3.0: *build_and_deploy -gitlab-puppeteer: *build_and_deploy -omnibus-gitlab-bionic: *build_and_deploy -omnibus-gitlab-centos7: *build_and_deploy -omnibus-gitlab-centos8: *build_and_deploy -omnibus-gitlab-jessie: *build_and_deploy -omnibus-gitlab-opensuse15.1: *build_and_deploy -omnibus-gitlab-opensuse15.2: *build_and_deploy -omnibus-gitlab-stretch: *build_and_deploy -omnibus-gitlab-buster: *build_and_deploy -omnibus-gitlab-xenial: *build_and_deploy -omnibus-gitlab-focal: *build_and_deploy -omnibus-gitlab-depscan: *build_and_deploy -omnibus-gitlab-cve-search: *build_and_deploy -release-tools: *build_and_deploy -sitespeed-gitlab: *build_and_deploy -ubi-release: *build_and_deploy -www-gitlab-com-2.6: *build_and_deploy -build-git: *build_and_deploy -terraform: *build_and_deploy +# Used by customers-gitlab-com qa e2e test: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab/ci/e2e.gitlab-ci.yml +ruby-2.7.2-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom +ruby-2.7-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom +ruby-2.7.2-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom +ruby-2.7-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom + +# Used by https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/blob/master/.gitlab-ci.yml +alpine-helm push: *build_and_deploy + +danger push: *build_and_deploy +danger-ruby-2.6.6 push: *build_and_deploy +gitlab-charts-build-base push: *build_and_deploy +gitlab-charts-build-base-helm-3 push: *build_and_deploy +gitlab-helm3.5-kubectl1.17 push: *build_and_deploy +gitlab-qa-ruby-2.7 push: *build_and_deploy +gitlab-qa-ruby-3.0 push: *build_and_deploy +gitlab-qa-alpine-ruby-2.7 push: *build_and_deploy +gitlab-qa-alpine-ruby-3.0 push: *build_and_deploy +gitlab-puppeteer push: *build_and_deploy +kaniko push: *build_and_deploy +omnibus-gitlab-bionic push: *build_and_deploy +omnibus-gitlab-centos7 push: *build_and_deploy +omnibus-gitlab-centos8 push: *build_and_deploy +omnibus-gitlab-jessie push: *build_and_deploy +omnibus-gitlab-opensuse15.1 push: *build_and_deploy +omnibus-gitlab-opensuse15.2 push: *build_and_deploy +omnibus-gitlab-stretch push: *build_and_deploy +omnibus-gitlab-buster push: *build_and_deploy +omnibus-gitlab-xenial push: *build_and_deploy +omnibus-gitlab-focal push: *build_and_deploy +omnibus-gitlab-depscan push: *build_and_deploy +omnibus-gitlab-cve-search push: *build_and_deploy +release-tools push: *build_and_deploy +sitespeed-gitlab push: *build_and_deploy +ubi-release push: *build_and_deploy +www-gitlab-com-3.0 push: *build_and_deploy +build-git push: *build_and_deploy +# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +terraform push: *build_and_deploy # Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting -golangci-lint-alpine: *build_and_deploy +golangci-lint-alpine push: *build_and_deploy # Used by gitlab-operator: https://gitlab.com/gitlab-org/gl-openshift/gitlab-operator -gitlab-operator-build-base: *build_and_deploy +gitlab-operator-build-base push: *build_and_deploy +alpine-bash-jq-curl-git push: *build_and_deploy cache-google-chrome: stage: automation + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + # Starts the job immediately + needs: [] variables: MOUNT_POINT: /builds/$CI_PROJECT_PATH/mnt script: - mkdir -p "$MOUNT_POINT" - cp scripts/cache-google-chrome $MOUNT_POINT - docker run -e "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" -e "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" -v "$MOUNT_POINT:/mnt" ubuntu:16.04 /mnt/cache-google-chrome - -# Old Google Chrome images are listed here for reference: - -# These still exist in our docker repository, but they cannot be built anymore since -# Google yanks them from their PPA. To build variations of these requires an image based on -# one of the images below. - -# See https://gitlab.com/gitlab-org/gitlab-build-images/merge_requests/54 for an example - -# ruby-2.3.3-golang-1.8-git-2.7-chrome-59.0-node-7.1-postgresql-9.6: *build_and_deploy_custom -# ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6: *build_and_deploy_custom -# ruby-2.3.3-golang-1.8-git-2.13-chrome-59.0-node-7.1-postgresql-9.6: *build_and_deploy_custom -# ruby-2.3.3-golang-1.8-git-2.13-chrome-60.0-node-7.1-postgresql-9.6: *build_and_deploy_custom -# ruby-2.3.3-golang-1.8-git-2.13-chrome-61.0-node-8.x-yarn-1.0-postgresql-9.6: *build_and_deploy_custom -# ruby-2.3.5-golang-1.8-git-2.13-chrome-61.0-node-8.x-yarn-1.0-postgresql-9.6: *build_and_deploy_custom -# ruby-2.3.5-golang-1.8-git-2.13-chrome-62.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-84-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-84-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.7.1-golang-1.14-git-2.28-lfs-2.9-chrome-84-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34: *build_and_deploy_custom -# ruby-2.7.1-golang-1.14-git-2.28-lfs-2.9-chrome-84-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34: *build_and_deploy_custom - -# ruby-2.6.5-git-2.27-chrome-83-node-12.x-yarn-1.21-docker-19.03.1: *build_and_deploy_custom -# ruby-2.6.5-git-2.28-chrome-84-node-12.x-yarn-1.21-docker-19.03.1: *build_and_deploy_custom diff --git a/Dockerfile.alpine-aws b/Dockerfile.alpine-aws deleted file mode 100644 index 773467b..0000000 --- a/Dockerfile.alpine-aws +++ /dev/null @@ -1,4 +0,0 @@ -FROM alpine:latest - -RUN apk --no-cache add py-pip \ - && pip install --no-cache-dir awscli diff --git a/Dockerfile.alpine-bash-jq-curl-git b/Dockerfile.alpine-bash-jq-curl-git new file mode 100644 index 0000000..c1c4ea9 --- /dev/null +++ b/Dockerfile.alpine-bash-jq-curl-git @@ -0,0 +1,3 @@ +FROM alpine:latest + +RUN apk --no-cache add bash jq curl git diff --git a/Dockerfile.custom b/Dockerfile.custom index c44c031..ccbb140 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -13,6 +13,12 @@ RUN /scripts/install-essentials ENV PATH $PATH:/usr/local/go/bin +# Ruby +ARG RUBY_VERSION +ARG RUBY_DOWNLOAD_SHA256 + +RUN if [ -n "$RUBY_VERSION" ]; then /scripts/install-ruby $RUBY_VERSION $RUBY_DOWNLOAD_SHA256 && ruby --version; fi + # Git ARG GIT_VERSION ARG GIT_DOWNLOAD_URL @@ -24,16 +30,19 @@ RUN if [ -n "$GIT_VERSION" ]; then /scripts/install-git && git --version; fi # Chrome ARG CHROME_VERSION ARG CHROME_DRIVER_VERSION + RUN if [ -n "$CHROME_VERSION" ]; then /scripts/install-chrome $CHROME_VERSION $CHROME_DRIVER_VERSION && google-chrome --version; fi # NodeJS and Yarn ARG NODE_INSTALL_VERSION ARG YARN_INSTALL_VERSION + RUN if [ -n "$NODE_INSTALL_VERSION" ] ; then /scripts/install-node $NODE_INSTALL_VERSION $YARN_INSTALL_VERSION && node --version && yarn --version; fi # Golang ARG INSTALL_GOLANG_VERSION ARG GOLANG_DOWNLOAD_SHA256 + RUN if [ -n "$INSTALL_GOLANG_VERSION" ] ; then /scripts/install-golang "${INSTALL_GOLANG_VERSION}" "${GOLANG_DOWNLOAD_SHA256}" && go version; fi # Git LFS (https://git-lfs.github.com/) @@ -45,33 +54,48 @@ RUN if [ -n "$LFS_VERSION" ]; then /scripts/install-lfs && git lfs --version; fi # Postgres ARG POSTGRES_VERSION + RUN if [ -n "$POSTGRES_VERSION" ] ; then /scripts/install-postgresql $POSTGRES_VERSION; fi -# Ansible -ARG ANSIBLE_VERSION -RUN if [ -n "$ANSIBLE_VERSION" ] ; then /scripts/install-ansible $ANSIBLE_VERSION; fi - -# Terraform -ARG TERRAFORM_VERSION -ARG TERRAFORM_DOWNLOAD_SHA256 -RUN if [ -n "$TERRAFORM_VERSION" ] ; then /scripts/install-terraform $TERRAFORM_VERSION $TERRAFORM_DOWNLOAD_SHA256; fi - # GraphicsMagick ARG GRAPHISMAGICK_VERSION ARG GRAPHISMAGICK_DOWNLOAD_URL=https://sourceforge.net/projects/graphicsmagick/files/graphicsmagick/${GRAPHISMAGICK_VERSION}/GraphicsMagick-${GRAPHISMAGICK_VERSION}.tar.gz ARG GRAPHISMAGICK_DOWNLOAD_SHA256 + RUN if [ -n "$GRAPHISMAGICK_VERSION" ]; then /scripts/install-graphicsmagick && gm version; fi # Docker ARG DOCKER_VERSION + RUN if [ -n "$DOCKER_VERSION" ]; then /scripts/install-docker $DOCKER_VERSION; fi # PgBouncer ARG PGBOUNCER_VERSION ARG PGBOUNCER_DOWNLOAD_SHA256 + RUN if [ -n "$PGBOUNCER_VERSION" ] ; then /scripts/install-pgbouncer $PGBOUNCER_VERSION $PGBOUNCER_DOWNLOAD_SHA256; fi +# Bazelisk +ARG BAZELISK_VERSION +ARG BAZELISK_DOWNLOAD_SHA256 + +RUN if [ -n "$BAZELISK_VERSION" ] ; then /scripts/install-bazelisk $BAZELISK_VERSION $BAZELISK_DOWNLOAD_SHA256; fi + RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 + +# Set as env variables all versions configured +ENV RUBY_VERSION=${RUBY_VERSION} \ + GIT_VERSION=${GIT_VERSION} \ + CHROME_VERSION=${CHROME_VERSION} \ + NODE_VERSION=${NODE_INSTALL_VERSION} \ + YARN_VERSION=${YARN_INSTALL_VERSION} \ + GOLANG_VERSION=${INSTALL_GOLANG_VERSION} \ + LFS_VERSION=${LFS_VERSION} \ + POSTGRES_VERSION=${POSTGRES_VERSION} \ + GRAPHISMAGICK_VERSION=${GRAPHISMAGICK_VERSION} \ + DOCKER_VERSION=${DOCKER_VERSION} \ + PGBOUNCER_VERSION=${PGBOUNCER_VERSION} \ + BAZELISK_VERSION=${BAZELISK_VERSION} diff --git a/Dockerfile.danger b/Dockerfile.danger index 9d0c045..26e8909 100644 --- a/Dockerfile.danger +++ b/Dockerfile.danger @@ -1,15 +1,15 @@ -FROM ruby:2.7.2-buster +FROM ruby:2.7.4-buster MAINTAINER GitLab Quality Team -# Danger manipulates git diff output. When an unicode char is present in the diff, it chokes with +# Danger manipulates git diff output. When a unicode char is present in the diff, it chokes with # (Danger::DSLError) [!] Invalid `Dangerfile` file: invalid byte sequence in US-ASCII -# eg. https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/275286625 +# eg. https://gitlab.com/gitlab-org/gitlab-foss/-/jobs/275286625 ENV LC_ALL "C.UTF-8" -ENV NODE_VERSION=12.4.0 -ENV YARN_VERSION=1.21.1-1 -ENV GITLAB_GEM_VERSION=4.16.1 -ENV DANGER_GEM_VERSION=8.2.1 +ENV NODE_VERSION=14.17.2 +ENV YARN_VERSION=1.22.10 +ENV GITLAB_GEM_VERSION=4.17.0 +ENV DANGER_GEM_VERSION=8.3.1 ENV DANGER_GITLAB_GEM_VERSION=8.0.0 ADD /scripts/ /scripts/ @@ -17,7 +17,7 @@ ADD /scripts/ /scripts/ RUN apt-get update \ && apt-get install -y git \ && /scripts/install-node ${NODE_VERSION} ${YARN_VERSION} \ - && yarn global add alex@7 \ + && yarn global add alex@9 \ && yarn cache clean \ && apt-get autoremove -yq \ && apt-get clean -yqq \ diff --git a/Dockerfile.danger-ruby-2.6.6 b/Dockerfile.danger-ruby-2.6.6 new file mode 100644 index 0000000..ae74d26 --- /dev/null +++ b/Dockerfile.danger-ruby-2.6.6 @@ -0,0 +1,31 @@ +FROM ruby:2.6.6-buster +MAINTAINER GitLab Quality Team + +# Danger manipulates git diff output. When a unicode char is present in the diff, it chokes with +# (Danger::DSLError) [!] Invalid `Dangerfile` file: invalid byte sequence in US-ASCII +# eg. https://gitlab.com/gitlab-org/gitlab-foss/-/jobs/275286625 +ENV LC_ALL "C.UTF-8" + +ENV NODE_VERSION=14.17.2 +ENV YARN_VERSION=1.22.10 +ENV GITLAB_GEM_VERSION=4.17.0 +ENV DANGER_GEM_VERSION=8.3.1 +ENV DANGER_GITLAB_GEM_VERSION=8.0.0 + +ADD /scripts/ /scripts/ + +RUN apt-get update \ + && apt-get install -y git \ + && /scripts/install-node ${NODE_VERSION} ${YARN_VERSION} \ + && yarn global add alex@9 \ + && yarn cache clean \ + && apt-get autoremove -yq \ + && apt-get clean -yqq \ + && rm -rf /var/lib/apt/lists/* \ + && gem install gitlab --version ${GITLAB_GEM_VERSION} --no-document \ + && gem install danger --version ${DANGER_GEM_VERSION} --no-document \ + && gem install danger-gitlab --version ${DANGER_GITLAB_GEM_VERSION} --no-document \ + && git version \ + && echo "node version " $(node --version) \ + && echo "yarn version " $(yarn --version) \ + && echo "Danger version " $(danger --version) diff --git a/Dockerfile.gitlab-charts-build-base-helm-3 b/Dockerfile.gitlab-charts-build-base-helm-3 index e716198..2424a84 100644 --- a/Dockerfile.gitlab-charts-build-base-helm-3 +++ b/Dockerfile.gitlab-charts-build-base-helm-3 @@ -10,7 +10,7 @@ ENV KUBECTL_VERSION=1.16.4 ENV KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl # Install dependencies -RUN apk --no-cache add -U openssl curl tar gzip bash ca-certificates git python2\ +RUN apk --no-cache add -U openssl curl tar gzip bash ca-certificates git python2 py-pip groff\ && mkdir -p /opt # Install kubectl @@ -24,6 +24,9 @@ RUN wget -q -O - ${HELM_URL} | tar zxf - \ && chmod +x /usr/bin/helm \ && helm version --client -#Install kubeval +# Install kubeval RUN mkdir -p $HELM_HOME/plugins && \ helm plugin install https://github.com/instrumenta/helm-kubeval + +# Install aws cli +RUN pip install awscli \ No newline at end of file diff --git a/Dockerfile.gitlab-gcloud-helm3.5-kubectl1.17 b/Dockerfile.gitlab-gcloud-helm3.5-kubectl1.17 new file mode 100644 index 0000000..495a8a1 --- /dev/null +++ b/Dockerfile.gitlab-gcloud-helm3.5-kubectl1.17 @@ -0,0 +1,39 @@ +FROM ruby:3.0.0-alpine +MAINTAINER GitLab Engineering Productivity Team + +ENV GCLOUD_VERSION=354.0.0 +ENV GCLOUD_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz + +ENV KUBECTL_VERSION=1.17.17 + +ENV HELM_VERSION=3.5.3 +ENV HELM_URL=https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz +ENV HELM_HOME=/root/.helm + +# Install dependencies +RUN apk --no-cache -U add openssl curl tar gzip bash ca-certificates git \ + && mkdir -p /opt + +# Install Google Cloud SDK +RUN curl ${GCLOUD_URL} > /tmp/google-cloud-sdk.tar.gz +RUN mkdir -p /usr/local/gcloud \ + && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \ + && /usr/local/gcloud/google-cloud-sdk/install.sh +ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin + +RUN gcloud version + +# Install kubectl +RUN wget https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ + && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \ + && rm -f kubectl + +# Install Helm +RUN wget -q -O - ${HELM_URL} | tar zxf - \ + && mv linux-amd64/helm /usr/bin/ \ + && chmod +x /usr/bin/helm \ + && helm version --client + +# Install kubeval +RUN mkdir -p $HELM_HOME/plugins \ + && helm plugin install https://github.com/instrumenta/helm-kubeval diff --git a/Dockerfile.gitlab-helm3.5-kubectl1.17 b/Dockerfile.gitlab-helm3.5-kubectl1.17 new file mode 100644 index 0000000..1ef2539 --- /dev/null +++ b/Dockerfile.gitlab-helm3.5-kubectl1.17 @@ -0,0 +1,27 @@ +FROM ruby:3.0.0-alpine +MAINTAINER GitLab Engineering Productivity Team + +ENV KUBECTL_VERSION=1.17.17 + +ENV HELM_VERSION=3.5.3 +ENV HELM_URL=https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz +ENV HELM_HOME=/root/.helm + +# Install dependencies +RUN apk --no-cache -U add openssl curl tar gzip bash ca-certificates git \ + && mkdir -p /opt + +# Install kubectl +RUN wget https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ + && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \ + && rm -f kubectl + +# Install Helm +RUN wget -q -O - ${HELM_URL} | tar zxf - \ + && mv linux-amd64/helm /usr/bin/ \ + && chmod +x /usr/bin/helm \ + && helm version --client + +# Install kubeval +RUN mkdir -p $HELM_HOME/plugins \ + && helm plugin install https://github.com/instrumenta/helm-kubeval diff --git a/Dockerfile.gitlab-operator-build-base b/Dockerfile.gitlab-operator-build-base index 0311a2b..09a9e28 100644 --- a/Dockerfile.gitlab-operator-build-base +++ b/Dockerfile.gitlab-operator-build-base @@ -1,33 +1,75 @@ -FROM golang:1.17 +ARG GOLANG_VERSION=1.16 + +## Kustomize +FROM golang:${GOLANG_VERSION}-alpine as kustomize + +ARG KUSTOMIZE_VERSION=3.8.10 +ENV GOPROXY="https://proxy.golang.org/" + +RUN apk add --no-cache gcc musl-dev +RUN mkdir /src && cd /src; \ + go mod init tmp && \ + go get sigs.k8s.io/kustomize/kustomize/v3@v${KUSTOMIZE_VERSION} + +## Controller-gen +FROM golang:${GOLANG_VERSION}-alpine as controller-gen + +ARG CONTROLLER_GEN_VERSION=0.3.0 +ENV GOPROXY="https://proxy.golang.org/" + +RUN mkdir /src && cd /src; \ + go mod init tmp \ + && go get sigs.k8s.io/controller-tools/cmd/controller-gen@v${CONTROLLER_GEN_VERSION} + +## Build +FROM golang:${GOLANG_VERSION}-alpine as build LABEL maintainer="GitLab Distribution Team" \ description="Build base image for the GitLab Operator project." -RUN apt-get update && apt-get install -y \ - buildah \ - && apt-get clean - -RUN go get -u \ - github.com/onsi/ginkgo/ginkgo \ - golang.org/x/lint/golint - ARG KUBEBUILDER_VERSION=2.3.1 ARG OPERATOR_SDK_VERSION=1.13.1 ARG OPM_VERSION=1.19.1 -ARG YQ_VERSION=2.4.1 +ARG YQ_VERSION=4.7.0 +ARG HELM_VERSION=3.5.2 +RUN apk add --no-cache podman buildah curl make gcc musl-dev git bash coreutils +RUN go get -u \ + github.com/onsi/ginkgo/ginkgo \ + golang.org/x/lint/golint + +# Kubebuilder RUN curl --retry 6 -Ls https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz | tar -xz -C /tmp/ \ && mv /tmp/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder \ && ln -sfv /usr/local/kubebuilder/bin/* /usr/local/bin +# Operator SDK RUN curl --retry 6 -LsO https://github.com/operator-framework/operator-sdk/releases/download/v${OPERATOR_SDK_VERSION}/operator-sdk_linux_amd64 \ && chmod +x operator-sdk_linux_amd64 \ && mv operator-sdk_linux_amd64 /usr/local/bin/operator-sdk +# OPM RUN curl --retry 6 -LsO https://github.com/operator-framework/operator-registry/releases/download/v${OPM_VERSION}/linux-amd64-opm \ && chmod +x linux-amd64-opm \ && mv linux-amd64-opm /usr/local/bin/opm -RUN curl --retry 6 -LsO https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 \ - && chmod +x yq_linux_amd64 \ - && mv yq_linux_amd64 /usr/local/bin/yq +# Yq +RUN curl --retry 6 -LsO https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64 \ + && chmod +x yq_linux_amd64 \ + && mv yq_linux_amd64 /usr/local/bin/yq + +# Helm +RUN curl --retry 6 -Ls "https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar -xz -C /tmp/ \ + && chmod +x /tmp/linux-amd64/helm \ + && mv /tmp/linux-amd64/helm /usr/local/bin/helm + +# Kustomize +COPY --from=kustomize /go/bin/kustomize /usr/local/bin/kustomize +# Controller-gen +COPY --from=controller-gen /go/bin/controller-gen /usr/local/bin/controller-gen + +## Final image +FROM golang:${GOLANG_VERSION}-alpine +LABEL maintainer="GitLab Distribution Team" \ + description="Build base image for the GitLab Operator project." +COPY --from=build / / diff --git a/Dockerfile.gitlab-puppeteer b/Dockerfile.gitlab-puppeteer index ad6f274..0bc6b38 100644 --- a/Dockerfile.gitlab-puppeteer +++ b/Dockerfile.gitlab-puppeteer @@ -1,4 +1,4 @@ -FROM node:12-stretch +FROM node:14 ADD /scripts/ /scripts/ diff --git a/Dockerfile.gitlab-qa-alpine-ruby-2.7 b/Dockerfile.gitlab-qa-alpine-ruby-2.7 index 96991ff..639d6c2 100644 --- a/Dockerfile.gitlab-qa-alpine-ruby-2.7 +++ b/Dockerfile.gitlab-qa-alpine-ruby-2.7 @@ -8,10 +8,10 @@ MAINTAINER GitLab Quality Team WORKDIR /home/qa -ENV DOCKER_VERSION="18.06.1" -ENV DOCKER_ARCHIVE="docker-${DOCKER_VERSION}-ce.tgz" +ENV DOCKER_VERSION="20.10.2" +ENV DOCKER_ARCHIVE="docker-${DOCKER_VERSION}.tgz" ENV DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/${DOCKER_ARCHIVE}" -ENV DOCKER_CHECKSUM="2d92728714d794f78619785a2c638b58b0b15e60b340de51886bf6d3730f31f0" +ENV DOCKER_CHECKSUM="893e0b05e347dee1d857022181831c6ec3798e9a49fda9a319b2f5a6fb79c42d" ENV DOCKER_BIN="/usr/local/bin/docker" # Install dependencies diff --git a/Dockerfile.gitlab-qa-alpine-ruby-3.0 b/Dockerfile.gitlab-qa-alpine-ruby-3.0 index 89b61c0..87e6901 100644 --- a/Dockerfile.gitlab-qa-alpine-ruby-3.0 +++ b/Dockerfile.gitlab-qa-alpine-ruby-3.0 @@ -8,10 +8,10 @@ MAINTAINER GitLab Quality Team WORKDIR /home/qa -ENV DOCKER_VERSION="18.06.1" -ENV DOCKER_ARCHIVE="docker-${DOCKER_VERSION}-ce.tgz" +ENV DOCKER_VERSION="20.10.2" +ENV DOCKER_ARCHIVE="docker-${DOCKER_VERSION}.tgz" ENV DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/${DOCKER_ARCHIVE}" -ENV DOCKER_CHECKSUM="2d92728714d794f78619785a2c638b58b0b15e60b340de51886bf6d3730f31f0" +ENV DOCKER_CHECKSUM="893e0b05e347dee1d857022181831c6ec3798e9a49fda9a319b2f5a6fb79c42d" ENV DOCKER_BIN="/usr/local/bin/docker" # Install dependencies diff --git a/Dockerfile.gitlab-qa-ruby-2.7 b/Dockerfile.gitlab-qa-ruby-2.7 index f23c553..9c5293a 100644 --- a/Dockerfile.gitlab-qa-ruby-2.7 +++ b/Dockerfile.gitlab-qa-ruby-2.7 @@ -6,7 +6,14 @@ ENV LANG C.UTF-8 WORKDIR /home/qa -RUN wget -q https://download.docker.com/linux/static/stable/x86_64/docker-19.03.1.tgz && \ - tar -zxf docker-19.03.1.tgz && mv docker/docker /usr/local/bin/docker && \ - echo "9b6191f64cb89e706dc29390408260d643e88f7f853d8878b2fb0360186b2ac3 /usr/local/bin/docker" | sha256sum -c - && \ - rm docker-19.03.1.tgz +ENV DOCKER_VERSION="20.10.2" +ENV DOCKER_ARCHIVE="docker-${DOCKER_VERSION}.tgz" +ENV DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/${DOCKER_ARCHIVE}" +ENV DOCKER_CHECKSUM="893e0b05e347dee1d857022181831c6ec3798e9a49fda9a319b2f5a6fb79c42d" +ENV DOCKER_BIN="/usr/local/bin/docker" + +RUN wget -q ${DOCKER_URL} && \ + tar -zxf ${DOCKER_ARCHIVE} && \ + mv docker/docker ${DOCKER_BIN} && \ + echo "${DOCKER_CHECKSUM} ${DOCKER_BIN}" | sha256sum -c - && \ + rm ${DOCKER_ARCHIVE} diff --git a/Dockerfile.gitlab-qa-ruby-3.0 b/Dockerfile.gitlab-qa-ruby-3.0 index cda9991..39340f1 100644 --- a/Dockerfile.gitlab-qa-ruby-3.0 +++ b/Dockerfile.gitlab-qa-ruby-3.0 @@ -6,7 +6,14 @@ ENV LANG C.UTF-8 WORKDIR /home/qa -RUN wget -q https://download.docker.com/linux/static/stable/x86_64/docker-19.03.1.tgz && \ - tar -zxf docker-19.03.1.tgz && mv docker/docker /usr/local/bin/docker && \ - echo "9b6191f64cb89e706dc29390408260d643e88f7f853d8878b2fb0360186b2ac3 /usr/local/bin/docker" | sha256sum -c - && \ - rm docker-19.03.1.tgz +ENV DOCKER_VERSION="20.10.2" +ENV DOCKER_ARCHIVE="docker-${DOCKER_VERSION}.tgz" +ENV DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/${DOCKER_ARCHIVE}" +ENV DOCKER_CHECKSUM="893e0b05e347dee1d857022181831c6ec3798e9a49fda9a319b2f5a6fb79c42d" +ENV DOCKER_BIN="/usr/local/bin/docker" + +RUN wget -q ${DOCKER_URL} && \ + tar -zxf ${DOCKER_ARCHIVE} && \ + mv docker/docker ${DOCKER_BIN} && \ + echo "${DOCKER_CHECKSUM} ${DOCKER_BIN}" | sha256sum -c - && \ + rm ${DOCKER_ARCHIVE} diff --git a/Dockerfile.golangci-lint-alpine b/Dockerfile.golangci-lint-alpine index 8942129..65de71a 100644 --- a/Dockerfile.golangci-lint-alpine +++ b/Dockerfile.golangci-lint-alpine @@ -1,5 +1,5 @@ -FROM golangci/golangci-lint:v1.28.1-alpine +FROM golangci/golangci-lint:v1.41.1-alpine # git must be installed for golangci-lint's --new-from-rev flag to work. -RUN apk --no-cache add git jq +RUN apk --no-cache add git jq bash # Include a default .golangci.yml COPY /assets/.golangci.yml /golangci/.golangci.yml diff --git a/Dockerfile.kaniko b/Dockerfile.kaniko new file mode 100644 index 0000000..82d57b6 --- /dev/null +++ b/Dockerfile.kaniko @@ -0,0 +1,16 @@ +FROM gcr.io/kaniko-project/executor:debug-v1.3.0 AS kaniko + +FROM alpine/git + +# See https://github.com/GoogleContainerTools/kaniko/blob/master/deploy/Dockerfile#L65-L70 +COPY --from=kaniko /kaniko/ /kaniko/ +COPY --from=kaniko /etc/nsswitch.conf /etc/nsswitch.conf + +ENV HOME /root +ENV USER root +ENV PATH /usr/local/bin:/kaniko:$PATH +ENV SSL_CERT_DIR=/kaniko/ssl/certs +ENV DOCKER_CONFIG /kaniko/.docker/ +ENV DOCKER_CREDENTIAL_GCR_CONFIG /kaniko/.config/gcloud/docker_credential_gcr_config.json + +ENTRYPOINT ["/kaniko/executor"] diff --git a/Dockerfile.omnibus-gitlab-bionic b/Dockerfile.omnibus-gitlab-bionic index f58df9c..45d9b09 100644 --- a/Dockerfile.omnibus-gitlab-bionic +++ b/Dockerfile.omnibus-gitlab-bionic @@ -1,7 +1,7 @@ FROM ubuntu:18.04 RUN apt-get update -q \ - && apt-get install -y g++ gcc git make curl zlib1g-dev libssl1.0-dev locales \ + && apt-get install -y g++ gcc git make curl zlib1g-dev libssl-dev locales \ && apt-get autoremove -yq \ && apt-get clean -yqq \ && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.omnibus-gitlab-stretch b/Dockerfile.omnibus-gitlab-stretch index c3a7ea1..076638c 100644 --- a/Dockerfile.omnibus-gitlab-stretch +++ b/Dockerfile.omnibus-gitlab-stretch @@ -1,7 +1,7 @@ FROM debian:stretch RUN apt-get update -q \ - && apt-get install -y g++ gcc git make curl zlib1g-dev libssl1.0-dev locales \ + && apt-get install -y g++ gcc git make curl zlib1g-dev libssl-dev locales \ && apt-get autoremove -yq \ && apt-get clean -yqq \ && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.ruby-alpine-aws b/Dockerfile.ruby-alpine-aws deleted file mode 100644 index 8df8c36..0000000 --- a/Dockerfile.ruby-alpine-aws +++ /dev/null @@ -1,4 +0,0 @@ -FROM ruby:2.5-alpine - -RUN apk --no-cache add py-pip && \ - pip install --no-cache-dir awscli diff --git a/Dockerfile.www-gitlab-com-2.4 b/Dockerfile.www-gitlab-com-2.4 deleted file mode 100644 index f85d945..0000000 --- a/Dockerfile.www-gitlab-com-2.4 +++ /dev/null @@ -1,10 +0,0 @@ -FROM ruby:2.4-slim - -ADD /scripts/ /scripts/ -RUN /scripts/install-www-gitlab-com - -# Set UTF-8 http://jaredmarkell.com/docker-and-locales/ -# Must be set after install-essentials is run -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 diff --git a/Dockerfile.www-gitlab-com-2.6 b/Dockerfile.www-gitlab-com-2.6 deleted file mode 100644 index 27d59b1..0000000 --- a/Dockerfile.www-gitlab-com-2.6 +++ /dev/null @@ -1,10 +0,0 @@ -FROM ruby:2.6.6-slim-stretch - -ADD /scripts/ /scripts/ -RUN /scripts/install-www-gitlab-com - -# Set UTF-8 http://jaredmarkell.com/docker-and-locales/ -# Must be set after install-essentials is run -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 diff --git a/Dockerfile.www-gitlab-com-3.0 b/Dockerfile.www-gitlab-com-3.0 new file mode 100644 index 0000000..9e8694b --- /dev/null +++ b/Dockerfile.www-gitlab-com-3.0 @@ -0,0 +1,17 @@ +FROM gcr.io/google.com/cloudsdktool/cloud-sdk as gcloud-sdk + +FROM ruby:3.0.0-slim + +ADD /scripts/ /scripts/ +RUN /scripts/install-www-gitlab-com + +# Install Google Cloud SDK for deploys via rsync +COPY --from=gcloud-sdk /usr/lib/google-cloud-sdk /usr/lib/google-cloud-sdk +COPY --from=gcloud-sdk /usr/share/google-cloud-sdk /usr/share/google-cloud-sdk +RUN cd /usr/bin && find ../lib/google-cloud-sdk/bin -type f -executable -exec ln -s {} \;; cd - + +# Set UTF-8 http://jaredmarkell.com/docker-and-locales/ +# Must be set after install-essentials is run +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 diff --git a/README.md b/README.md index 71dc6a7..1b1c916 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ various parts of GitLab: There are two methods to adding a new image: -1. Use a docker buildfile. +1. Use a Dockerfile. 1. Use a custom image with versioned features. -### Use a docker buildfile +### Use a Dockerfile A bespoke image is one that uses it's own Dockerfile. In the interests of avoiding repetition, it's better to use the custom docker build if possible. @@ -37,46 +37,47 @@ The name is defined as `[feature]-[version]` pairs, separated by `-`. Here are some example build names: -1. `ruby-2.1-git-2.7` -1. `ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6` -1. `ruby-2.4.1-golang-1.8-git-2.7-node-7.1-postgresql-9.6` +1. `ruby-2.7-golang-1.15-git-2.29` +1. `ruby-3.0.0-git-2.29-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.34` +1. `ruby-3.0.0.patched-golang-1.14-git-2.29-lfs-2.9-chrome-87-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.34` -The first pair represents the base image and version. So `ruby-2.1-...` will use the `ruby:2.1` base image, while -`golang-1.8` will use the `golang:1.8` base image. +The first pair represents the base image and version. So `ruby-3.0-...` will use the `ruby:3.0` base image, while +`golang-1.15` will use the `golang:1.15` base image. Each of the following parts represents another feature and version. Available options are: -1. `golang` -1. `git` +1. `bazelisk` 1. `chrome` -1. `node` -1. `yarn` -1. `postgres` -1. `terraform` -1. `ansible` +1. `docker` +1. `git` +1. `golang` 1. `graphicsmagick` +1. `lfs` +1. `node` 1. `pgbouncer` +1. `postgresql` +1. `ruby` +1. `yarn` #### Adding a new build -As an example, if you want to add new image for Ruby 2.4 with `git` 2.14 and `golang` 1.9, -the name would be `ruby-2.4-golang-1.9-git-2.14`. +As an example, if you want to add new image for Ruby 3.0 with `git` 2.29 and `golang` 1.15, +the name would be `ruby-3.0-golang-1.15-git-2.29`. -1. Add a test task: `ruby-2.4-golang-1.9-git-2.14 test: *test_custom` -1. Add a new build task: `ruby-2.4-golang-1.9-git-2.14: *build_and_deploy_custom` +1. Add a test task: `ruby-3.0-golang-1.15-git-2.29 test: *test_custom` +1. Add a new build task: `ruby-3.0-golang-1.15-git-2.29 push: *build_and_deploy_custom` -#### Forcing custom images to be rebuilt +Note that the build name is also used for tagging the generated image, which means it cannot be +longer than 128 characters. -By default, once a custom image is built, tagged and pushed to the registry, it's -not rebuilt to ensure an upstream dependency doesn't end up breaking our images -unexpectedly. +#### Pushing a rebuild image -For reference, this happened in the past: https://gitlab.com/gitlab-org/gitlab/issues/205192 +To build (or rebuild) a given image and push it, you need to enable manual action for a given +job after it is merged to master. -In the rare case where the `Dockerfile.custom` file is updated and all custom -images shoulld be rebuild, you can start a new pipeline and set the variable -`FORCE_BUILD` to `true`. +By default we don't do it to ensure that an upstream dependency doesn't end up breaking our images +unexpectedly. For reference, this happened in the past: https://gitlab.com/gitlab-org/gitlab/issues/205192 ## Note regarding Google Chrome diff --git a/patches/ruby/2.7.2/debug-segfault.patch b/patches/ruby/2.7.2/debug-segfault.patch new file mode 100644 index 0000000..72c1e41 --- /dev/null +++ b/patches/ruby/2.7.2/debug-segfault.patch @@ -0,0 +1,25 @@ +diff --git a/class.c b/class.c +index c866d1d727..37ff3c5ade 100644 +--- a/class.c ++++ b/class.c +@@ -27,6 +27,7 @@ + #include "ruby/st.h" + #include "constant.h" + #include "vm_core.h" ++#include "vm_debug.h" + #include "id_table.h" + #include + +@@ -119,6 +120,12 @@ rb_class_foreach_subclass(VALUE klass, void (*f)(VALUE, VALUE), VALUE arg) + while (cur) { + VALUE curklass = cur->klass; + cur = cur->next; ++ ++ if (curklass == 0) { ++ fprintf(stderr, "=== Detected NULL subclass:\n"); ++ dp(curklass); ++ } ++ + f(curklass, arg); + } + } diff --git a/patches/ruby/2.7.2/thread-memory-allocations-2.7.patch b/patches/ruby/2.7.2/thread-memory-allocations-2.7.patch new file mode 100644 index 0000000..ca0a87e --- /dev/null +++ b/patches/ruby/2.7.2/thread-memory-allocations-2.7.patch @@ -0,0 +1,258 @@ +From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= +Date: Tue, 22 Dec 2020 15:33:08 +0100 +Subject: [PATCH] Expose `Thread#memory_allocations` counters + +This provides currently a per-thread GC heap slots +and malloc allocations statistics. + +This is designed to measure a memory allocations +in a multi-threaded environments (concurrent requests +processing) with an accurate information about allocated +memory within a given execution context. + +Example: Measure memory pressure generated by a given +requests to easier find requests with a lot of allocations. + +Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530 +--- + gc.c | 20 ++++++ + .../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++ + thread.c | 55 +++++++++++++++ + vm_core.h | 17 +++++ + 4 files changed, 159 insertions(+) + create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb + +diff --git a/gc.c b/gc.c +index 73faf46b128b..f2dcd2935052 100644 +--- a/gc.c ++++ b/gc.c +@@ -2172,6 +2172,13 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote + GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */ + #endif + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects); ++ } ++#endif ++ + objspace->total_allocated_objects++; + + gc_report(5, objspace, "newobj: %s\n", obj_info(obj)); +@@ -9732,6 +9739,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si + #endif + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ if (new_size > old_size) { ++ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size); ++ } ++ ++ if (type == MEMOP_TYPE_MALLOC) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs); ++ } ++ } ++#endif ++ + if (type == MEMOP_TYPE_MALLOC) { + retry: + if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc) { +diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb +new file mode 100644 +index 000000000000..2e281513578b +--- /dev/null ++++ b/test/ruby/test_thread_trace_memory_allocations.rb +@@ -0,0 +1,67 @@ ++# frozen_string_literal: true ++ ++require 'test/unit' ++ ++class TestThreadTraceMemoryAllocations < Test::Unit::TestCase ++ def test_disabled_trace_memory_allocations ++ Thread.trace_memory_allocations = false ++ ++ assert_predicate Thread.current.memory_allocations, :nil? ++ end ++ ++ def test_enabled_trace_memory_allocations ++ Thread.trace_memory_allocations = true ++ ++ assert_not_nil(Thread.current.memory_allocations) ++ end ++ ++ def test_only_this_thread_allocations_are_counted ++ changed = { ++ total_allocated_objects: 1000, ++ total_malloc_bytes: 1_000_000, ++ total_mallocs: 100 ++ } ++ ++ Thread.trace_memory_allocations = true ++ ++ assert_less_than(changed) do ++ Thread.new do ++ assert_greater_than(changed) do ++ # This will allocate: 5k objects, 5k mallocs, 5MB ++ allocate(5000, 1000) ++ end ++ end.join ++ ++ # This will allocate: 50 objects, 50 mallocs, 500 bytes ++ allocate(50, 10) ++ end ++ end ++ ++ private ++ ++ def allocate(slots, bytes) ++ Array.new(slots).map do ++ '0' * bytes ++ end ++ end ++ ++ def assert_greater_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}") ++ end ++ end ++ ++ def assert_less_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}") ++ end ++ end ++end +diff --git a/thread.c b/thread.c +index 708aaa471d99..d68a59e9f2d6 100644 +--- a/thread.c ++++ b/thread.c +@@ -5143,6 +5143,55 @@ rb_thread_backtrace_locations_m(int argc, VALUE *argv, VALUE thval) + return rb_vm_thread_backtrace_locations(argc, argv, thval); + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++rb_thread_t * ++ruby_threadptr_for_trace_memory_allocations(void) ++{ ++ // The order of this checks is important due ++ // to how Ruby VM is initialized ++ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) { ++ return GET_THREAD(); ++ } ++ ++ return NULL; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations(VALUE _) ++{ ++ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val) ++{ ++ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val); ++ return val; ++} ++ ++static VALUE ++rb_thread_memory_allocations(VALUE self) ++{ ++ rb_thread_t *th = rb_thread_ptr(self); ++ ++ if (!th->vm->thread_trace_memory_allocations) { ++ return Qnil; ++ } ++ ++ VALUE ret = rb_hash_new(); ++ ++ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects")); ++ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes")); ++ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs")); ++ ++ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects)); ++ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes)); ++ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs)); ++ ++ return ret; ++} ++#endif ++ + /* + * Document-class: ThreadError + * +@@ -5230,6 +5279,12 @@ Init_Thread(void) + rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0); + rb_define_alias(rb_cThread, "inspect", "to_s"); + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0); ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1); ++ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0); ++#endif ++ + rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError, + "stream closed in another thread"); + +diff --git a/vm_core.h b/vm_core.h +index 12c3ac377551..63cdf55fa6ed 100644 +--- a/vm_core.h ++++ b/vm_core.h +@@ -69,6 +69,13 @@ + # define VM_INSN_INFO_TABLE_IMPL 2 + #endif + ++/* ++ * track a per thread memory allocations ++ */ ++#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS ++# define THREAD_TRACE_MEMORY_ALLOCATIONS 1 ++#endif ++ + #include "ruby/ruby.h" + #include "ruby/st.h" + +@@ -602,6 +609,7 @@ typedef struct rb_vm_struct { + unsigned int running: 1; + unsigned int thread_abort_on_exception: 1; + unsigned int thread_report_on_exception: 1; ++ unsigned int thread_trace_memory_allocations: 1; + + unsigned int safe_level_: 1; + int sleeper; +@@ -960,6 +968,14 @@ typedef struct rb_thread_struct { + + rb_thread_list_t *join_list; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ struct { ++ size_t total_allocated_objects; ++ size_t total_malloc_bytes; ++ size_t total_mallocs; ++ } memory_allocations; ++#endif ++ + union { + struct { + VALUE proc; +@@ -1852,6 +1868,7 @@ void rb_threadptr_interrupt(rb_thread_t *th); + void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th); + void rb_threadptr_pending_interrupt_clear(rb_thread_t *th); + void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v); ++rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void); + VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec); + void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo); + void rb_execution_context_update(const rb_execution_context_t *ec); diff --git a/patches/ruby/2.7.4/debug-segfault.patch b/patches/ruby/2.7.4/debug-segfault.patch new file mode 100644 index 0000000..72c1e41 --- /dev/null +++ b/patches/ruby/2.7.4/debug-segfault.patch @@ -0,0 +1,25 @@ +diff --git a/class.c b/class.c +index c866d1d727..37ff3c5ade 100644 +--- a/class.c ++++ b/class.c +@@ -27,6 +27,7 @@ + #include "ruby/st.h" + #include "constant.h" + #include "vm_core.h" ++#include "vm_debug.h" + #include "id_table.h" + #include + +@@ -119,6 +120,12 @@ rb_class_foreach_subclass(VALUE klass, void (*f)(VALUE, VALUE), VALUE arg) + while (cur) { + VALUE curklass = cur->klass; + cur = cur->next; ++ ++ if (curklass == 0) { ++ fprintf(stderr, "=== Detected NULL subclass:\n"); ++ dp(curklass); ++ } ++ + f(curklass, arg); + } + } diff --git a/patches/ruby/2.7.4/thread-memory-allocations-2.7.patch b/patches/ruby/2.7.4/thread-memory-allocations-2.7.patch new file mode 100644 index 0000000..ca0a87e --- /dev/null +++ b/patches/ruby/2.7.4/thread-memory-allocations-2.7.patch @@ -0,0 +1,258 @@ +From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= +Date: Tue, 22 Dec 2020 15:33:08 +0100 +Subject: [PATCH] Expose `Thread#memory_allocations` counters + +This provides currently a per-thread GC heap slots +and malloc allocations statistics. + +This is designed to measure a memory allocations +in a multi-threaded environments (concurrent requests +processing) with an accurate information about allocated +memory within a given execution context. + +Example: Measure memory pressure generated by a given +requests to easier find requests with a lot of allocations. + +Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530 +--- + gc.c | 20 ++++++ + .../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++ + thread.c | 55 +++++++++++++++ + vm_core.h | 17 +++++ + 4 files changed, 159 insertions(+) + create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb + +diff --git a/gc.c b/gc.c +index 73faf46b128b..f2dcd2935052 100644 +--- a/gc.c ++++ b/gc.c +@@ -2172,6 +2172,13 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote + GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */ + #endif + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects); ++ } ++#endif ++ + objspace->total_allocated_objects++; + + gc_report(5, objspace, "newobj: %s\n", obj_info(obj)); +@@ -9732,6 +9739,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si + #endif + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ if (new_size > old_size) { ++ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size); ++ } ++ ++ if (type == MEMOP_TYPE_MALLOC) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs); ++ } ++ } ++#endif ++ + if (type == MEMOP_TYPE_MALLOC) { + retry: + if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc) { +diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb +new file mode 100644 +index 000000000000..2e281513578b +--- /dev/null ++++ b/test/ruby/test_thread_trace_memory_allocations.rb +@@ -0,0 +1,67 @@ ++# frozen_string_literal: true ++ ++require 'test/unit' ++ ++class TestThreadTraceMemoryAllocations < Test::Unit::TestCase ++ def test_disabled_trace_memory_allocations ++ Thread.trace_memory_allocations = false ++ ++ assert_predicate Thread.current.memory_allocations, :nil? ++ end ++ ++ def test_enabled_trace_memory_allocations ++ Thread.trace_memory_allocations = true ++ ++ assert_not_nil(Thread.current.memory_allocations) ++ end ++ ++ def test_only_this_thread_allocations_are_counted ++ changed = { ++ total_allocated_objects: 1000, ++ total_malloc_bytes: 1_000_000, ++ total_mallocs: 100 ++ } ++ ++ Thread.trace_memory_allocations = true ++ ++ assert_less_than(changed) do ++ Thread.new do ++ assert_greater_than(changed) do ++ # This will allocate: 5k objects, 5k mallocs, 5MB ++ allocate(5000, 1000) ++ end ++ end.join ++ ++ # This will allocate: 50 objects, 50 mallocs, 500 bytes ++ allocate(50, 10) ++ end ++ end ++ ++ private ++ ++ def allocate(slots, bytes) ++ Array.new(slots).map do ++ '0' * bytes ++ end ++ end ++ ++ def assert_greater_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}") ++ end ++ end ++ ++ def assert_less_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}") ++ end ++ end ++end +diff --git a/thread.c b/thread.c +index 708aaa471d99..d68a59e9f2d6 100644 +--- a/thread.c ++++ b/thread.c +@@ -5143,6 +5143,55 @@ rb_thread_backtrace_locations_m(int argc, VALUE *argv, VALUE thval) + return rb_vm_thread_backtrace_locations(argc, argv, thval); + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++rb_thread_t * ++ruby_threadptr_for_trace_memory_allocations(void) ++{ ++ // The order of this checks is important due ++ // to how Ruby VM is initialized ++ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) { ++ return GET_THREAD(); ++ } ++ ++ return NULL; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations(VALUE _) ++{ ++ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val) ++{ ++ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val); ++ return val; ++} ++ ++static VALUE ++rb_thread_memory_allocations(VALUE self) ++{ ++ rb_thread_t *th = rb_thread_ptr(self); ++ ++ if (!th->vm->thread_trace_memory_allocations) { ++ return Qnil; ++ } ++ ++ VALUE ret = rb_hash_new(); ++ ++ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects")); ++ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes")); ++ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs")); ++ ++ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects)); ++ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes)); ++ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs)); ++ ++ return ret; ++} ++#endif ++ + /* + * Document-class: ThreadError + * +@@ -5230,6 +5279,12 @@ Init_Thread(void) + rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0); + rb_define_alias(rb_cThread, "inspect", "to_s"); + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0); ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1); ++ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0); ++#endif ++ + rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError, + "stream closed in another thread"); + +diff --git a/vm_core.h b/vm_core.h +index 12c3ac377551..63cdf55fa6ed 100644 +--- a/vm_core.h ++++ b/vm_core.h +@@ -69,6 +69,13 @@ + # define VM_INSN_INFO_TABLE_IMPL 2 + #endif + ++/* ++ * track a per thread memory allocations ++ */ ++#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS ++# define THREAD_TRACE_MEMORY_ALLOCATIONS 1 ++#endif ++ + #include "ruby/ruby.h" + #include "ruby/st.h" + +@@ -602,6 +609,7 @@ typedef struct rb_vm_struct { + unsigned int running: 1; + unsigned int thread_abort_on_exception: 1; + unsigned int thread_report_on_exception: 1; ++ unsigned int thread_trace_memory_allocations: 1; + + unsigned int safe_level_: 1; + int sleeper; +@@ -960,6 +968,14 @@ typedef struct rb_thread_struct { + + rb_thread_list_t *join_list; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ struct { ++ size_t total_allocated_objects; ++ size_t total_malloc_bytes; ++ size_t total_mallocs; ++ } memory_allocations; ++#endif ++ + union { + struct { + VALUE proc; +@@ -1852,6 +1868,7 @@ void rb_threadptr_interrupt(rb_thread_t *th); + void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th); + void rb_threadptr_pending_interrupt_clear(rb_thread_t *th); + void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v); ++rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void); + VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec); + void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo); + void rb_execution_context_update(const rb_execution_context_t *ec); diff --git a/patches/ruby/3.0.2/dont_alias_array_size.patch b/patches/ruby/3.0.2/dont_alias_array_size.patch new file mode 100644 index 0000000..c299faf --- /dev/null +++ b/patches/ruby/3.0.2/dont_alias_array_size.patch @@ -0,0 +1,22 @@ +commit fa0279d947c3962c3f8c32852278d3ebb964cb19 +Author: Koichi Sasada +Date: Wed Jul 28 13:40:30 2021 +0900 + + should not share same `def` for specialized method + + Because the key of redefine table is `def`, `def` should be + unique for each optimized method (`alias` is not allowed). + +diff --git a/array.c b/array.c +index 3cb57a0872..36f712bcac 100644 +--- a/array.c ++++ b/array.c +@@ -8384,7 +8384,7 @@ Init_Array(void) + rb_define_method(rb_cArray, "each_index", rb_ary_each_index, 0); + rb_define_method(rb_cArray, "reverse_each", rb_ary_reverse_each, 0); + rb_define_method(rb_cArray, "length", rb_ary_length, 0); +- rb_define_alias(rb_cArray, "size", "length"); ++ rb_define_method(rb_cArray, "size", rb_ary_length, 0); + rb_define_method(rb_cArray, "empty?", rb_ary_empty_p, 0); + rb_define_method(rb_cArray, "find_index", rb_ary_index, -1); + rb_define_method(rb_cArray, "index", rb_ary_index, -1); diff --git a/patches/ruby/3.0.2/override_optimized_methods.patch b/patches/ruby/3.0.2/override_optimized_methods.patch new file mode 100644 index 0000000..43b492b --- /dev/null +++ b/patches/ruby/3.0.2/override_optimized_methods.patch @@ -0,0 +1,85 @@ +commit fb4cf204a662a8cd9dafef6f31f2bd0db9129abe +Author: Koichi Sasada +Date: Thu May 13 03:10:18 2021 +0900 + + use me->def instead of me for opt_table + + `vm_opt_method_table` is me=>bop table to manage the optimized + methods (by specialized instruction). However, `me` can be invalidated + to invalidate the method cache entry. + [Bug #17725] + + To solve the issue, use `me-def` instead of `me` which simply copied + at invalidation timing. + + A test by @jeremyevans https://github.com/ruby/ruby/pull/4376 + +diff --git a/test/ruby/test_method.rb b/test/ruby/test_method.rb +index 240821c9e2..0bd5dc63dd 100644 +--- a/test/ruby/test_method.rb ++++ b/test/ruby/test_method.rb +@@ -1303,6 +1303,21 @@ class TestMethod < Test::Unit::TestCase + end; + end + ++ def test_override_optimized_method_on_class_using_prepend ++ assert_separately(%w(--disable-gems), <<-'end;', timeout: 30) ++ # Bug #17725 [ruby-core:102884] ++ $VERBOSE = nil ++ String.prepend(Module.new) ++ class String ++ def + other ++ 'blah blah' ++ end ++ end ++ ++ assert_equal('blah blah', 'a' + 'b') ++ end; ++ end ++ + def test_eqq + assert_operator(0.method(:<), :===, 5) + assert_not_operator(0.method(:<), :===, -5) +diff --git a/vm.c b/vm.c +index accd12644e..8a044efa1f 100644 +--- a/vm.c ++++ b/vm.c +@@ -1798,7 +1798,7 @@ rb_iter_break_value(VALUE val) + + /* optimization: redefine management */ + +-static st_table *vm_opt_method_table = 0; ++static st_table *vm_opt_method_def_table = 0; + static st_table *vm_opt_mid_table = 0; + + static int +@@ -1852,9 +1852,8 @@ rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VALUE klass) + klass = RBASIC_CLASS(klass); + } + if (vm_redefinition_check_method_type(me->def)) { +- if (st_lookup(vm_opt_method_table, (st_data_t)me, &bop)) { +- int flag = vm_redefinition_check_flag(klass); +- ++ if (st_lookup(vm_opt_method_def_table, (st_data_t)me->def, &bop)) { ++ int flag = vm_redefinition_check_flag(klass); + ruby_vm_redefined_flag[bop] |= flag; + } + } +@@ -1885,7 +1884,7 @@ add_opt_method(VALUE klass, ID mid, VALUE bop) + const rb_method_entry_t *me = rb_method_entry_at(klass, mid); + + if (me && vm_redefinition_check_method_type(me->def)) { +- st_insert(vm_opt_method_table, (st_data_t)me, (st_data_t)bop); ++ st_insert(vm_opt_method_def_table, (st_data_t)me->def, (st_data_t)bop); + st_insert(vm_opt_mid_table, (st_data_t)mid, (st_data_t)Qtrue); + } + else { +@@ -1899,7 +1898,7 @@ vm_init_redefined_flag(void) + ID mid; + VALUE bop; + +- vm_opt_method_table = st_init_numtable(); ++ vm_opt_method_def_table = st_init_numtable(); + vm_opt_mid_table = st_init_numtable(); + + #define OP(mid_, bop_) (mid = id##mid_, bop = BOP_##bop_, ruby_vm_redefined_flag[bop] = 0) diff --git a/patches/ruby/3.0.2/thread-memory-allocations-3.0.patch b/patches/ruby/3.0.2/thread-memory-allocations-3.0.patch new file mode 100644 index 0000000..ba3a743 --- /dev/null +++ b/patches/ruby/3.0.2/thread-memory-allocations-3.0.patch @@ -0,0 +1,258 @@ +From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= +Date: Tue, 22 Dec 2020 15:33:08 +0100 +Subject: [PATCH] Expose `Thread#memory_allocations` counters + +This provides currently a per-thread GC heap slots +and malloc allocations statistics. + +This is designed to measure a memory allocations +in a multi-threaded environments (concurrent requests +processing) with an accurate information about allocated +memory within a given execution context. + +Example: Measure memory pressure generated by a given +requests to easier find requests with a lot of allocations. + +Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530 +--- + gc.c | 20 ++++++ + .../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++ + thread.c | 55 +++++++++++++++ + vm_core.h | 17 +++++ + 4 files changed, 159 insertions(+) + create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb + +diff --git a/gc.c b/gc.c +index 27cf65b196a3..280c62fbe341 100644 +--- a/gc.c ++++ b/gc.c +@@ -2123,6 +2123,13 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, + // TODO: make it atomic, or ractor local + objspace->total_allocated_objects++; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects); ++ } ++#endif ++ + #if RGENGC_PROFILE + if (wb_protected) { + objspace->profile.total_generated_normal_object_count++; +@@ -10487,6 +10494,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si + #endif + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ if (new_size > old_size) { ++ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size); ++ } ++ ++ if (type == MEMOP_TYPE_MALLOC) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs); ++ } ++ } ++#endif ++ + if (type == MEMOP_TYPE_MALLOC) { + retry: + if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) { +diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb +new file mode 100644 +index 000000000000..2e281513578b +--- /dev/null ++++ b/test/ruby/test_thread_trace_memory_allocations.rb +@@ -0,0 +1,67 @@ ++# frozen_string_literal: true ++ ++require 'test/unit' ++ ++class TestThreadTraceMemoryAllocations < Test::Unit::TestCase ++ def test_disabled_trace_memory_allocations ++ Thread.trace_memory_allocations = false ++ ++ assert_predicate Thread.current.memory_allocations, :nil? ++ end ++ ++ def test_enabled_trace_memory_allocations ++ Thread.trace_memory_allocations = true ++ ++ assert_not_nil(Thread.current.memory_allocations) ++ end ++ ++ def test_only_this_thread_allocations_are_counted ++ changed = { ++ total_allocated_objects: 1000, ++ total_malloc_bytes: 1_000_000, ++ total_mallocs: 100 ++ } ++ ++ Thread.trace_memory_allocations = true ++ ++ assert_less_than(changed) do ++ Thread.new do ++ assert_greater_than(changed) do ++ # This will allocate: 5k objects, 5k mallocs, 5MB ++ allocate(5000, 1000) ++ end ++ end.join ++ ++ # This will allocate: 50 objects, 50 mallocs, 500 bytes ++ allocate(50, 10) ++ end ++ end ++ ++ private ++ ++ def allocate(slots, bytes) ++ Array.new(slots).map do ++ '0' * bytes ++ end ++ end ++ ++ def assert_greater_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}") ++ end ++ end ++ ++ def assert_less_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}") ++ end ++ end ++end +diff --git a/thread.c b/thread.c +index dce181d24e02..247440766cdf 100644 +--- a/thread.c ++++ b/thread.c +@@ -5412,6 +5412,55 @@ Init_Thread_Mutex(void) + rb_native_mutex_initialize(&th->interrupt_lock); + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++rb_thread_t * ++ruby_threadptr_for_trace_memory_allocations(void) ++{ ++ // The order of this checks is important due ++ // to how Ruby VM is initialized ++ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) { ++ return GET_THREAD(); ++ } ++ ++ return NULL; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations(VALUE _) ++{ ++ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val) ++{ ++ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val); ++ return val; ++} ++ ++static VALUE ++rb_thread_memory_allocations(VALUE self) ++{ ++ rb_thread_t *th = rb_thread_ptr(self); ++ ++ if (!th->vm->thread_trace_memory_allocations) { ++ return Qnil; ++ } ++ ++ VALUE ret = rb_hash_new(); ++ ++ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects")); ++ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes")); ++ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs")); ++ ++ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects)); ++ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes)); ++ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs)); ++ ++ return ret; ++} ++#endif ++ + /* + * Document-class: ThreadError + * +@@ -5497,6 +5546,12 @@ Init_Thread(void) + rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0); + rb_define_alias(rb_cThread, "inspect", "to_s"); + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0); ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1); ++ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0); ++#endif ++ + rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError, + "stream closed in another thread"); + +diff --git a/vm_core.h b/vm_core.h +index 5f8d4ab87670..ac15f72fa25b 100644 +--- a/vm_core.h ++++ b/vm_core.h +@@ -97,6 +97,13 @@ + # define VM_INSN_INFO_TABLE_IMPL 2 + #endif + ++/* ++ * track a per thread memory allocations ++ */ ++#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS ++# define THREAD_TRACE_MEMORY_ALLOCATIONS 1 ++#endif ++ + #if defined(NSIG_MAX) /* POSIX issue 8 */ + # undef NSIG + # define NSIG NSIG_MAX +@@ -606,6 +613,7 @@ typedef struct rb_vm_struct { + unsigned int thread_abort_on_exception: 1; + unsigned int thread_report_on_exception: 1; + unsigned int thread_ignore_deadlock: 1; ++ unsigned int thread_trace_memory_allocations: 1; + + /* object management */ + VALUE mark_object_ary; +@@ -981,6 +989,14 @@ typedef struct rb_thread_struct { + + struct rb_waiting_list *join_list; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ struct { ++ size_t total_allocated_objects; ++ size_t total_malloc_bytes; ++ size_t total_mallocs; ++ } memory_allocations; ++#endif ++ + union { + struct { + VALUE proc; +@@ -1901,6 +1917,7 @@ void rb_threadptr_interrupt(rb_thread_t *th); + void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th); + void rb_threadptr_pending_interrupt_clear(rb_thread_t *th); + void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v); ++rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void); + VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec); + void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo); + void rb_execution_context_update(const rb_execution_context_t *ec); diff --git a/scripts/build-helpers.sh b/scripts/build-helpers.sh index 52b25b0..e69de29 100644 --- a/scripts/build-helpers.sh +++ b/scripts/build-helpers.sh @@ -1,25 +0,0 @@ -function image_already_exists() { - image=$1 - - $(docker pull $image > /dev/null) -} - -function build_if_needed() { - image=$1 - - if ! image_already_exists $image; then - docker build -t "$image" -f "Dockerfile.$image" . - else - echo "$image already exists, skipping build." - fi -} - -function push_if_needed() { - image=$1 - - if ! image_already_exists $image; then - docker push "$image" - else - echo "$image already exists, skipping push." - fi -} diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index 85a21a5..dd01d73 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e IFS=$'\n\t' @@ -7,21 +7,21 @@ source scripts/build-helpers.sh function print_golang_args() { case "$1" in - 1.12) - INSTALL_GOLANG_VERSION=1.12.13 - GOLANG_DOWNLOAD_SHA256=da036454cb3353f9f507f0ceed4048feac611065e4e1818b434365eb32ac9bdc - ;; - 1.13) - INSTALL_GOLANG_VERSION=1.13.4 - GOLANG_DOWNLOAD_SHA256=692d17071736f74be04a72a06dab9cac1cd759377bd85316e52b2227604c004c - ;; 1.14) - INSTALL_GOLANG_VERSION=1.14.1 - GOLANG_DOWNLOAD_SHA256=2f49eb17ce8b48c680cdb166ffd7389702c0dec6effa090c324804a5cac8a7f8 + INSTALL_GOLANG_VERSION=1.14.15 + GOLANG_DOWNLOAD_SHA256=c64a57b374a81f7cf1408d2c410a28c6f142414f1ffa9d1062de1d653b0ae0d6 ;; 1.15) - INSTALL_GOLANG_VERSION=1.15.5 - GOLANG_DOWNLOAD_SHA256=9a58494e8da722c3aef248c9227b0e9c528c7318309827780f16220998180a0d + INSTALL_GOLANG_VERSION=1.15.10 + GOLANG_DOWNLOAD_SHA256=4aa1267517df32f2bf1cc3d55dfc27d0c6b2c2b0989449c96dd19273ccca051d + ;; + 1.16) + INSTALL_GOLANG_VERSION=1.16.9 + GOLANG_DOWNLOAD_SHA256=d2c095c95f63c2a3ef961000e0ecb9d81d5c68b6ece176e2a8a2db82dc02931c + ;; + 1.17) + INSTALL_GOLANG_VERSION=1.17.2 + GOLANG_DOWNLOAD_SHA256=f242a9db6a0ad1846de7b6d94d507915d14062660616a61ef7c808a76e4f1676 ;; *) echo "Unknown golang version $1"; exit 1; esac @@ -68,6 +68,14 @@ function print_chrome_args() { CHROME_VERSION=85.0.4183.83-1 CHROME_DRIVER_VERSION=85.0.4183.87 ;; + 87|87.0) + CHROME_VERSION=87.0.4280.88-1 + CHROME_DRIVER_VERSION=87.0.4280.88 + ;; + 89|89.0) + CHROME_VERSION=89.0.4389.90-1 + CHROME_DRIVER_VERSION=89.0.4389.23 + ;; *) echo "Unknown chrome version $1"; exit 1; esac printf -- "--build-arg CHROME_VERSION=%s " "$CHROME_VERSION" @@ -77,42 +85,19 @@ function print_chrome_args() { # see https://www.kernel.org/pub/software/scm/git function print_git_args() { case "$1" in - 2.18) - GIT_VERSION=2.18.0 - GIT_DOWNLOAD_SHA256=94faf2c0b02a7920b0b46f4961d8e9cad08e81418614102898a55f980fa3e7e4 - ;; - 2.21) - GIT_VERSION=2.21.0 - GIT_DOWNLOAD_SHA256=85eca51c7404da75e353eba587f87fea9481ba41e162206a6f70ad8118147bee - ;; - 2.22) - GIT_VERSION=2.22.0 - GIT_DOWNLOAD_SHA256=a4b7e4365bee43caa12a38d646d2c93743d755d1cea5eab448ffb40906c9da0b - ;; - 2.24) - GIT_VERSION=2.24.3 - GIT_DOWNLOAD_SHA256=ef6d1d1de1d7921a54d23d07479bd2766f050d6435cea5d3b5322aa4897cb3d7 - ;; - 2.25) - GIT_VERSION=2.25.4 - GIT_DOWNLOAD_SHA256=1b11d0ee481c6735e9f7d81a3576ef0989bde7843cb4adcdef578c0593d279ab - ;; - 2.26) - GIT_VERSION=2.26.2 - GIT_DOWNLOAD_SHA256=e1c17777528f55696815ef33587b1d20f5eec246669f3b839d15dbfffad9c121 - ;; - 2.27) - GIT_VERSION=2.27.0 - GIT_DOWNLOAD_SHA256=77ded85cbe42b1ffdc2578b460a1ef5d23bcbc6683eabcafbb0d394dffe2e787 - ;; - 2.28) - GIT_VERSION=2.28.0 - GIT_DOWNLOAD_SHA256=f914c60a874d466c1e18467c864a910dd4ea22281ba6d4d58077cb0c3f115170 - ;; 2.29) GIT_VERSION=2.29.0 GIT_DOWNLOAD_SHA256=fa08dc8424ef80c0f9bf307877f9e2e49f1a6049e873530d6747c2be770742ff ;; + 2.31) + GIT_VERSION=2.31.1 + GIT_DOWNLOAD_SHA256=46d37c229e9d786510e0c53b60065704ce92d5aedc16f2c5111e3ed35093bfa7 + ;; + 2.33) + GIT_VERSION=2.33.1 + GIT_DOWNLOAD_SHA256=02047f8dc8934d57ff5e02aadd8a2fe8e0bcf94a7158da375e48086cc46fce1d + ;; + *) echo "Unknown git version $1"; exit 1; esac @@ -144,10 +129,9 @@ function print_lfs_args() { function print_node_args() { case "$1" in - 8.x) NODE_INSTALL_VERSION=8.16.0 ;; - 10.x) NODE_INSTALL_VERSION=10.16.0 ;; - 12.x) NODE_INSTALL_VERSION=12.4.0 ;; - 12.18) NODE_INSTALL_VERSION=12.18.4 ;; + 12.22) NODE_INSTALL_VERSION=12.22.1 ;; + 14.15) NODE_INSTALL_VERSION=14.15.4 ;; + 14|14.16) NODE_INSTALL_VERSION=14.16.0 ;; *) echo "Unknown node version $1"; exit 1; esac printf -- "--build-arg NODE_INSTALL_VERSION=%s " "$NODE_INSTALL_VERSION" @@ -155,10 +139,10 @@ function print_node_args() { function print_yarn_args() { case "$1" in - 1.12) YARN_INSTALL_VERSION=1.12.3-1 ;; - 1.16) YARN_INSTALL_VERSION=1.16.0-1 ;; - 1.21) YARN_INSTALL_VERSION=1.21.1-1 ;; - 1.22) YARN_INSTALL_VERSION=1.22.5-1 ;; + 1.12) YARN_INSTALL_VERSION=1.12.3 ;; + 1.16) YARN_INSTALL_VERSION=1.16.0 ;; + 1.21) YARN_INSTALL_VERSION=1.21.1 ;; + 1.22) YARN_INSTALL_VERSION=1.22.10 ;; *) echo "Unknown yarn version $1"; exit 1; esac printf -- "--build-arg YARN_INSTALL_VERSION=%s " "$YARN_INSTALL_VERSION" @@ -168,27 +152,10 @@ function print_postgres_args() { printf -- "--build-arg POSTGRES_VERSION=%s " "$1" } -function print_ansible_args() { - printf -- "--build-arg ANSIBLE_VERSION=%s " "$1" -} - function print_docker_args() { printf -- "--build-arg DOCKER_VERSION=%s " "$1" } -function print_terraform_args() { - case "$1" in - 0.11) - TERRAFORM_VERSION=0.11.7 - TERRAFORM_DOWNLOAD_SHA256=6b8ce67647a59b2a3f70199c304abca0ddec0e49fd060944c26f666298e23418 - ;; - *) echo "Unknown terraform version $1"; exit 1; - esac - - printf -- "--build-arg TERRAFORM_VERSION=%s " "$TERRAFORM_VERSION" - printf -- "--build-arg TERRAFORM_DOWNLOAD_SHA256=%s " "$TERRAFORM_DOWNLOAD_SHA256" -} - function print_graphicsmagick_args() { case "$1" in 1.3.29) @@ -203,6 +170,10 @@ function print_graphicsmagick_args() { GRAPHISMAGICK_VERSION=1.3.34 GRAPHISMAGICK_DOWNLOAD_SHA256=4717f7a32d964c515d83706fd52d34e089c2ffa35f8fbf43c923ce19343cf2f4 ;; + 1.3.36) + GRAPHISMAGICK_VERSION=1.3.36 + GRAPHISMAGICK_DOWNLOAD_SHA256=1e6723c48c4abbb31197fadf8396b2d579d97e197123edc70a4f057f0533d563 + ;; *) echo "Unknown graphicsmagick version $1"; exit 1; esac @@ -223,29 +194,61 @@ function print_pgbouncer_args() { printf -- "--build-arg PGBOUNCER_DOWNLOAD_SHA256=%s " "$PGBOUNCER_DOWNLOAD_SHA256" } -function parse_arguments() { - read base - read base_version - - # Lock Ruby to Debian version to pin OpenSSL version - case "$base" in - ruby) - case "$base_version" in - 2.6.*) - base_version="$base_version-stretch" - ;; - *) - base_version="$base_version-buster" - ;; - esac +function print_bazelisk_args() { + case "$1" in + 1.9.0) + BAZELISK_VERSION=1.9.0 + BAZELISK_DOWNLOAD_SHA256=b8c7f2a1b07ad64a2f27f8f19a202f90d044de7b5b6ccc387a6fe5d4a8ec4937 + ;; + *) echo "Unknown bazelisk version $1"; exit 1; esac - printf -- "-f Dockerfile.custom " "$base" - printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$base" - printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$base_version" + printf -- "--build-arg BAZELISK_VERSION=%s " "$BAZELISK_VERSION" + printf -- "--build-arg BAZELISK_DOWNLOAD_SHA256=%s " "$BAZELISK_DOWNLOAD_SHA256" +} + +function print_ruby_args() { + case "$1" in + 2.6|2.6.*) + RUBY_VERSION="2.6.6" + RUBY_DOWNLOAD_SHA256="364b143def360bac1b74eb56ed60b1a0dca6439b00157ae11ff77d5cd2e92291" + ;; + + 2.7|2.7.patched) + RUBY_VERSION="2.7.4" + RUBY_DOWNLOAD_SHA256="3043099089608859fc8cce7f9fdccaa1f53a462457e3838ec3b25a7d609fbc5b" + ;; + + # Please update any clients still asking for these images to request `ruby-2.7` instead, + # which will provide the latest available patch level (see above.) + 2.7.*) + RUBY_VERSION="2.7.2" + RUBY_DOWNLOAD_SHA256="6e5706d0d4ee4e1e2f883db9d768586b4d06567debea353c796ec45e8321c3d4" + ;; + + 3.0|3.0.patched) + RUBY_VERSION="3.0.2" + RUBY_DOWNLOAD_SHA256="5085dee0ad9f06996a8acec7ebea4a8735e6fac22f22e2d98c3f2bc3bef7e6f1" + ;; + + *) echo "Unknown ruby version $1"; exit 1; + esac + + printf -- "--build-arg RUBY_VERSION=%s " "$RUBY_VERSION" + printf -- "--build-arg RUBY_DOWNLOAD_SHA256=%s " "$RUBY_DOWNLOAD_SHA256" +} + +function parse_arguments() { + printf -- "-f Dockerfile.custom " + + # defaults + CUSTOM_IMAGE_NAME=debian + CUSTOM_IMAGE_VERSION=buster + while read image; do read version case "$image" in + ruby) print_ruby_args $version ;; golang) print_golang_args $version ;; chrome) print_chrome_args $version ;; docker) print_docker_args $version ;; @@ -254,13 +257,15 @@ function parse_arguments() { node) print_node_args $version ;; yarn) print_yarn_args $version ;; postgresql) print_postgres_args $version ;; - ansible) print_ansible_args $version ;; - terraform) print_terraform_args $version ;; graphicsmagick) print_graphicsmagick_args $version ;; pgbouncer) print_pgbouncer_args $version ;; + bazelisk) print_bazelisk_args $version ;; *) exit 1;; esac done + + printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$CUSTOM_IMAGE_NAME" + printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$CUSTOM_IMAGE_VERSION" } function generate_command() { @@ -280,17 +285,11 @@ function build_custom_if_needed() { build_image_name=$1 full_image_name="$CI_REGISTRY_IMAGE:$build_image_name" - if [[ "$FORCE_BUILD" == "true" ]] || ! image_already_exists $full_image_name; then - docker_command=$(generate_command $@) - if [[ "$FORCE_BUILD" == "true" ]]; then - echo "Force building $build_image_name due to \$FORCE_BUILD=true with $docker_command" - else - echo "Building $build_image_name with $docker_command" - fi - eval $docker_command - else - echo "$build_image_name already exists, skipping build." - fi + # This re-uses and builds an existing image if needed + docker pull --quiet "$full_image" || true + docker_command=$(generate_command $@ --cache-from="$full_image_name" ) + echo "Building $build_image_name with $docker_command" + eval $docker_command } build_custom_if_needed $@ diff --git a/scripts/install-ansible b/scripts/install-ansible deleted file mode 100755 index dec9bc7..0000000 --- a/scripts/install-ansible +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -xeou pipefail - -ANSIBLE_VERSION=$1 - -apt-get update -apt-get install -y python-pip python-dev - -pip install --no-cache-dir --upgrade cffi -pip install --no-cache-dir "ansible==${ANSIBLE_VERSION}" - -apt-get autoremove -yq -apt-get clean -yqq -rm -rf /var/lib/apt/lists/* diff --git a/scripts/install-bazelisk b/scripts/install-bazelisk new file mode 100755 index 0000000..d3dcbdb --- /dev/null +++ b/scripts/install-bazelisk @@ -0,0 +1,14 @@ +#!/bin/bash + +set -xeou pipefail + +BAZELISK_VERSION=${1} +BAZELISK_DOWNLOAD_SHA256=${2} + +BAZELISK_DOWNLOAD_URL="https://github.com/bazelbuild/bazelisk/releases/download/v${BAZELISK_VERSION}/bazelisk-linux-amd64" + +curl -fsSL "$BAZELISK_DOWNLOAD_URL" -o bazelisk +echo "${BAZELISK_DOWNLOAD_SHA256} bazelisk" | sha256sum -c - + +chmod +x bazelisk +mv bazelisk /usr/local/bin/bazel diff --git a/scripts/install-essentials b/scripts/install-essentials index ea0d473..d855a32 100755 --- a/scripts/install-essentials +++ b/scripts/install-essentials @@ -7,29 +7,32 @@ export DEBIAN_FRONTEND=noninteractive apt-get update -EXIT_CODE=0 -grep "Debian GNU/Linux 9" /etc/issue || EXIT_CODE=$? +# We install `git-core` as some tooling expect `/usr/bin/git` +# other tools that rely on PATH ordering will pick a one in `/usr/local` +# if present -if [ $EXIT_CODE -eq 0 ] +if grep "Debian GNU/Linux 9" /etc/issue then apt-get install -y \ - curl wget build-essential apt-utils locales \ + curl wget build-essential apt-utils locales openssh-client \ libssl-dev libyaml-dev libreadline6-dev zlib1g-dev \ libncurses5-dev libffi-dev libgdbm3 libgdbm-dev \ - ca-certificates libyaml-dev checkinstall libxml2-dev \ + ca-certificates checkinstall libxml2-dev \ libxslt-dev libcurl4-openssl-dev libicu-dev \ logrotate python-docutils pkg-config cmake nodejs \ libkrb5-dev postgresql-client mysql-client unzip \ - libre2-dev gettext rsync + libsqlite3-dev libpq-dev libpng-dev libjpeg-dev libzstd-dev \ + libre2-dev libevent-dev gettext rsync git-core else apt-get install -y \ - curl wget build-essential apt-utils locales \ + curl wget build-essential apt-utils locales openssh-client \ libssl-dev libyaml-dev libreadline-dev zlib1g-dev \ - libncurses5-dev libffi-dev ca-certificates libyaml-dev libxml2-dev \ + libncurses5-dev libffi-dev ca-certificates libxml2-dev \ libxslt1-dev libcurl4-openssl-dev libicu-dev \ logrotate python-docutils pkg-config cmake \ libkrb5-dev postgresql-client unzip \ - libre2-dev gettext rsync + libsqlite3-dev libpq-dev libpng-dev libjpeg-dev libzstd-dev \ + libre2-dev libevent-dev gettext rsync git-core fi # Set UTF-8 diff --git a/scripts/install-node b/scripts/install-node index 724d191..a0f4092 100755 --- a/scripts/install-node +++ b/scripts/install-node @@ -2,16 +2,14 @@ set -xeuo pipefail IFS=$'\n\t' -NODE_INSTALL_VERSION=${1:-12.4.0} -YARN_INSTALL_VERSION=${2:-1.21.1-1} +NODE_INSTALL_VERSION=${1:-14.16.0} +YARN_INSTALL_VERSION=${2:-1.22.10} # Map MAJOR.MINOR.patch -> MAJOR.x NODE_MAJOR=`echo $NODE_INSTALL_VERSION | sed -r -e "s/([0-9]+)\.[0-9]+.*/\1\.x/g"` -# add official debian repos for node and yarn +# add official debian repos for node curl -sS -L https://deb.nodesource.com/setup_${NODE_MAJOR} | bash - -curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list apt-get update @@ -20,7 +18,8 @@ curl -s -O "https://deb.nodesource.com/node_$NODE_MAJOR/pool/main/n/nodejs/$NODE dpkg -i "$NODE_FILE_NAME" rm -f "$NODE_FILE_NAME" -apt-get install -y yarn=$YARN_INSTALL_VERSION +npm install --global yarn@${YARN_INSTALL_VERSION} +npm cache clean --force apt-get autoremove -yq apt-get clean -yqq diff --git a/scripts/install-pgbouncer b/scripts/install-pgbouncer index be00920..9c7c35f 100755 --- a/scripts/install-pgbouncer +++ b/scripts/install-pgbouncer @@ -1,19 +1,18 @@ #!/bin/bash +set -xeuo pipefail + PGBOUNCER_VERSION=${1} PGBOUNCER_DOWNLOAD_SHA256=${2} - PGBOUNCER_DOWNLOAD_URL="https://pgbouncer.github.io/downloads/files/$PGBOUNCER_VERSION/pgbouncer-$PGBOUNCER_VERSION.tar.gz" -curl -o pgbouncer.tar.gz -fsSL $PGBOUNCER_DOWNLOAD_URL -echo "${PGBOUNCER_DOWNLOAD_SHA256} pgbouncer.tar.gz" | sha256sum -c - - -tar xfz pgbouncer.tar.gz -rm -f pgbouncer.tar.gz -cd pgbouncer-${PGBOUNCER_VERSION} -./configure --prefix=/usr/local -make -cp pgbouncer /usr/local/bin - -cd .. -rm -rf pgbouncer-${PGBOUNCER_VERSION} +curl -o pgbouncer.tar.gz -fsSL $PGBOUNCER_DOWNLOAD_URL \ + && echo "${PGBOUNCER_DOWNLOAD_SHA256} pgbouncer.tar.gz" | sha256sum -c - \ + && tar xfz pgbouncer.tar.gz \ + && rm -f pgbouncer.tar.gz \ + && cd pgbouncer-${PGBOUNCER_VERSION} \ + && ./configure --prefix=/usr/local \ + && make \ + && cp pgbouncer /usr/local/bin \ + && cd .. \ + && rm -rf pgbouncer-${PGBOUNCER_VERSION} diff --git a/scripts/install-ruby b/scripts/install-ruby index 7aa1212..b08ff9d 100755 --- a/scripts/install-ruby +++ b/scripts/install-ruby @@ -1,14 +1,74 @@ #!/bin/bash -set -xeuo pipefail -IFS=$'\n\t' +set -xeou pipefail + +# Based on https://github.com/docker-library/ruby/blob/master/2.7/buster/Dockerfile + +RUBY_VERSION=${1} +RUBY_MAJOR=${1%.*} # strip last component +RUBY_DOWNLOAD_SHA256=${2} +RUBY_DOWNLOAD_URL="https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.gz" + +JEMALLOC_VERSION=5.2.1 +JEMALLOC_DOWNLOAD_SHA256="34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6" +JEMALLOC_DOWNLOAD_URL="https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" + +# Install needed packages +apt-get update +apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf + +# Download jemalloc +mkdir -p /usr/src/jemalloc +cd /usr/src/jemalloc +curl --retry 6 -L -so jemalloc.tar.bz2 ${JEMALLOC_DOWNLOAD_URL} +echo "${JEMALLOC_DOWNLOAD_SHA256} jemalloc.tar.bz2" | sha256sum -c - + +# Install jemalloc +tar -xjf jemalloc.tar.bz2 +rm jemalloc.tar.bz2 +cd jemalloc-${JEMALLOC_VERSION} +./autogen.sh --prefix=/usr --enable-prof +make -j "$(nproc)" install cd /tmp -wget http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.2.tar.gz -tar -xvzf ruby-2.2.2.tar.gz -cd ruby-2.2.2 -./configure --enable-shared --disable-install-doc --disable-install-rdoc --disable-install-capi -make install -j 3 +# Download Ruby +curl -fsSL "$RUBY_DOWNLOAD_URL" -o ruby.tar.gz +echo "${RUBY_DOWNLOAD_SHA256} ruby.tar.gz" | sha256sum -c - +# Skip installing Gem docs +mkdir -p /usr/local/etc +echo 'install: --no-document' >> /usr/local/etc/gemrc +echo 'update: --no-document' >> /usr/local/etc/gemrc + +# Unpack Ruby +mkdir -p /usr/src/ruby +tar -xzf ruby.tar.gz -C /usr/src/ruby --strip-components=1 +rm ruby.tar.gz +cd /usr/src/ruby + +# Apply patches +if [[ -d "/patches/ruby/$RUBY_VERSION" ]]; then + for i in "/patches/ruby/$RUBY_VERSION"/*.patch; do + echo "$i..." + patch -p1 -i "$i" + done +fi + +# Compile +cflags="-fno-omit-frame-pointer" ./configure --enable-shared --with-jemalloc --disable-install-doc --disable-install-rdoc --disable-install-capi +make install -j $(nproc) + +# Cleanup cd / -rm -rf /tmp/* +rm -rf /usr/src/ruby /usr/src/jemalloc +apt-get purge -y --auto-remove ruby + +# Verify +# verify we have no "ruby" packages installed +! dpkg -l | grep -i ruby +[ "$(command -v ruby)" = '/usr/local/bin/ruby' ] + +# rough smoke test +ruby --version +gem --version +bundle --version diff --git a/scripts/install-terraform b/scripts/install-terraform deleted file mode 100755 index 84c1e20..0000000 --- a/scripts/install-terraform +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -xeou pipefail - -TERRAFORM_VERSION=${1} -TERRAFORM_DOWNLOAD_SHA256=${2} - -TERRAFORM_DOWNLOAD_URL="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - -curl -fsSL "$TERRAFORM_DOWNLOAD_URL" -o terraform.zip -echo "${TERRAFORM_DOWNLOAD_SHA256} terraform.zip" | sha256sum -c - -unzip -d /usr/local/bin terraform.zip -rm terraform.zip diff --git a/scripts/install-www-gitlab-com b/scripts/install-www-gitlab-com index 9d57f79..14ff49e 100755 --- a/scripts/install-www-gitlab-com +++ b/scripts/install-www-gitlab-com @@ -5,45 +5,30 @@ IFS=$'\n\t' export DEBIAN_FRONTEND=noninteractive -# Install LaTeX and other packages +# echo "deb http://deb.debian.org/debian testing main" | tee -a /etc/apt/sources.list.d/testing.list +# Install packages apt-get update apt-get install -yq --no-install-recommends \ make gcc g++ locales \ - rsync git-core texlive-latex-recommended texlive-xetex \ - texlive-fonts-recommended lmodern ed file curl gnupg2 \ - unzip - -# Install pandoc -cd /tmp -curl -L -O https://github.com/jgm/pandoc/releases/download/2.3.1/pandoc-2.3.1-linux.tar.gz -tar xvf pandoc-2.3.1-linux.tar.gz -cp pandoc-2.3.1/bin/* /usr/local/bin -rm -rf /tmp/pandoc* - -# Install Google Cloud SDK for deploys via rsync -echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - -apt-get update -apt-get install -yq --no-install-recommends google-cloud-sdk python-crcmod python3-crcmod + rsync git-core \ + ed file curl gnupg2 \ + unzip \ + python3 python3-pip python3-crcmod python-minimal # Install Imagemagick for cropping the pictures on the team page apt-get install -yq --no-install-recommends imagemagick # Install node & yarn -NODE_INSTALL_VERSION=12.4.0 -YARN_INSTALL_VERSION=1.21.1-1 +NODE_INSTALL_VERSION=12.22.1 +YARN_INSTALL_VERSION=1.22.10 /scripts/install-node $NODE_INSTALL_VERSION $YARN_INSTALL_VERSION && node --version && yarn --version # Install yamllint -# We need the latest version -# https://packages.debian.org/testing/yamllint +# We need version 1.25.0+: https://github.com/adrienverge/yamllint/blob/master/CHANGELOG.rst YAMLLINT_VERSION=1.25.0 -echo "deb http://deb.debian.org/debian testing main" | tee -a /etc/apt/sources.list.d/testing.list -apt update -# Fix for immediate configuration error - https://gitlab.com/gitlab-org/gitlab-build-images/-/merge_requests/332 -apt -t testing install -o APT::Immediate-Configure=false -yq libcrypt1 -apt -t testing install -yq yamllint=${YAMLLINT_VERSION}-1 +# Temporarily pin pyyaml and pathspec to reduce compatibility with packages from Debian bullseye (testing) +pip3 install yamllint==${YAMLLINT_VERSION} pyyaml==5.4.1 pathspec==0.8.1 # Install gitlab-runner curl -O -J -L https://gitlab-ci-multi-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-ci-multi-runner-linux-amd64