diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9fd76bc..d2192ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ # we support merge request workflow only include: - - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' - local: '.gitlab/ci/*.yml' default: @@ -23,13 +22,5 @@ stages: variables: DOCKER_HOST: tcp://docker:2375 -cache-google-chrome: - image: debian:bullseye-slim - stage: automation - # Starts the job immediately - needs: [] - script: - - bash ./scripts/cache-google-chrome - # disable DinD - before_script: [] - services: [] +cache-chrome-chromium: + extends: .cache-google-chrome diff --git a/.gitlab/ci/definitions.yml b/.gitlab/ci/definitions.yml index c8c0a87..16fc573 100644 --- a/.gitlab/ci/definitions.yml +++ b/.gitlab/ci/definitions.yml @@ -30,3 +30,23 @@ variables: PUSH_CUSTOM_IMAGE: "true" BASE_BUILD_REGISTRY_IMAGE: $CI_REGISTRY_IMAGE + +.cache-google-chrome: + stage: automation + needs: [] + before_script: + - docker buildx create --use + script: + - | + docker buildx build \ + --build-arg CI_API_V4_URL=$CI_API_V4_URL \ + --build-arg CI_PROJECT_ID=$CI_PROJECT_ID \ + --build-arg CI_JOB_TOKEN=$CI_JOB_TOKEN \ + --platform linux/amd64,linux/arm64 \ + -f "Dockerfile.cache-chrome" \ + . + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_SERVER_HOST == "gitlab.com"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_SERVER_HOST == "gitlab.com"' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_SERVER_HOST == "gitlab.com"' + when: manual diff --git a/Dockerfile.cache-chrome b/Dockerfile.cache-chrome new file mode 100644 index 0000000..5e1d45c --- /dev/null +++ b/Dockerfile.cache-chrome @@ -0,0 +1,11 @@ +# Save amd64 chrome and arm64 .deb files to package registry +FROM debian:latest + +ARG TARGETARCH +ARG CI_API_V4_URL +ARG CI_PROJECT_ID +ARG CI_JOB_TOKEN + +ADD scripts/cache-google-chrome /cache-google-chrome + +RUN /cache-google-chrome diff --git a/scripts/cache-google-chrome b/scripts/cache-google-chrome index a148ba8..2dd7795 100755 --- a/scripts/cache-google-chrome +++ b/scripts/cache-google-chrome @@ -1,47 +1,94 @@ #!/bin/bash + # This script attempts to copy the latest version of the Google Chrome Debian # package into our own package registry. Google yanks old versions regularly, # making it hard to keep up with all the new versions. set -e -PKG=google-chrome-stable - export DEBIAN_FRONTEND=noninteractive -apt-get -y update -apt-get -y install apt-utils curl bash gnupg2 -curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add - -echo "deb [arch=$(dpkg --print-architecture)] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list +function cache-chrome() { + PKG=google-chrome-stable -echo "Updating apt to get Google Chrome packages..." + curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add - + echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >/etc/apt/sources.list.d/google.list -apt-get -y -q update + echo "Updating apt to get Google Chrome packages..." -echo "Checking for latest Chrome version in apt repository..." + apt-get -y -qq update -LATEST_VERSION=$(apt-cache show $PKG | grep Version | sort | tail -1 | sed -e "s/Version: //") -CHROME_DEB="google-chrome-stable_${LATEST_VERSION}_$(dpkg --print-architecture).deb" -CHROME_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PKG}/${LATEST_VERSION}/${CHROME_DEB}" + echo "Checking for latest Chrome version in apt repository..." -echo "Checking if cache has $CHROME_DEB" -FILE_CHECK=$(curl --silent --location --head --output /dev/null --write "%{http_code}\\n" "$CHROME_URL") + LATEST_VERSION=$(apt-cache show $PKG | grep Version | sort | tail -1 | sed -e "s/Version: //") + CHROME_DEB="google-chrome-stable_${LATEST_VERSION}_amd64.deb" + CHROME_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PKG}/${LATEST_VERSION}/${CHROME_DEB}" -if [ "$FILE_CHECK" -eq "200" ]; then - echo "Latest version $LATEST_VERSION is already cached!" -else - echo "Downloading latest Chrome version ($LATEST_VERSION) in apt repository..." - cd /tmp - apt-get download $PKG + echo "Checking if cache has $CHROME_DEB" + FILE_CHECK=$(curl --silent --location --head --output /dev/null --write "%{http_code}\\n" "$CHROME_URL") - if ! [ -f "$CHROME_DEB" ]; then - echo "Downloaded file didn't have expected name: $CHROME_DEB" - ls - exit 1 + if [ "$FILE_CHECK" -eq "200" ]; then + echo "Latest version $LATEST_VERSION is already cached!" + else + echo "Downloading latest Chrome version ($LATEST_VERSION) in apt repository..." + cd /tmp + apt-get download $PKG + + if ! [ -f "$CHROME_DEB" ]; then + echo "Downloaded file didn't have expected name: $CHROME_DEB" + ls + exit 1 + fi + + echo "Transferring $CHROME_DEB to GitLab packages" + curl --fail --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \ + --upload-file "./{$CHROME_DEB}" \ + "$CHROME_URL" fi +} - echo "Transferring $CHROME_DEB to GitLab packages" - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \ - --upload-file "./{$CHROME_DEB}" \ - "$CHROME_URL" +function cache-chromium() { + PKG=chromium + + echo "Checking for latest Chromium version in apt repository..." + + LATEST_VERSION=$(apt-cache show $PKG | grep Version | sort | tail -1 | sed -e "s/Version: //") + VERSION_NUMBER=$(echo $LATEST_VERSION | sed -e "s/~deb.*//") # remove debian version part to have chrome and chromium compatible version numbers + CHROMIUM_DEB="chromium_${VERSION_NUMBER}_arm64.deb" + CHROMIUM_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PKG}/${VERSION_NUMBER}/${CHROMIUM_DEB}" + + echo "Checking if cache has ${CHROMIUM_DEB}" + FILE_CHECK=$(curl --silent --location --head --output /dev/null --write "%{http_code}\\n" "$CHROMIUM_URL") + + if [ "$FILE_CHECK" -eq "200" ]; then + echo "Latest version ${LATEST_VERSION} is already cached!" + else + echo "Downloading latest Chromium version (${LATEST_VERSION}) from apt repository..." + cd /tmp + apt-get download "$PKG" + + DEB_FILE="chromium_${LATEST_VERSION}_arm64.deb" + + if ! [ -f "$DEB_FILE" ]; then + echo "Downloaded file didn't have expected name: ${DEB_FILE}" + ls + exit 1 + fi + + mv "$DEB_FILE" "$CHROMIUM_DEB" + + echo "Transferring ${CHROMIUM_DEB} to GitLab packages" + curl --fail --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \ + --upload-file "./{$CHROMIUM_DEB}" \ + "$CHROMIUM_URL" + fi +} + +apt-get -y -qq update +apt-get -y install apt-utils curl bash gnupg2 > /dev/null + +if [ "$TARGETARCH" == "amd64" ]; then + cache-chrome +else + cache-chromium fi diff --git a/scripts/install-chrome b/scripts/install-chrome index 32bec58..b4eb144 100755 --- a/scripts/install-chrome +++ b/scripts/install-chrome @@ -4,47 +4,26 @@ set -xeuo pipefail IFS=$'\n\t' function build_debian() { - ARCH="${TARGETARCH:-amd64}" + CHROME_VERSION=${1:-103.0.5060.134-1} + ARCH=${TARGETARCH:-amd64} - if [[ "${ARCH}" == "arm64" ]]; then - echo "The arm64 does not have prebuilt chrome. Using chromium instead." - apt-get update -q -y - apt-get install -y chromium chromium-driver - apt-get autoremove -yq - apt-get clean -yqq - rm -rf /var/lib/apt/lists/* - exit 0 + if [ "${ARCH}" == "amd64" ]; then + PKG=google-chrome-stable + else + PKG=chromium fi - CHROME_VERSION=${1:-99.0.4844.74-1} # We hard code the URL rather than using $CI_API_V4_URL $CI_PROJECT_ID, # because we would need to forward those variables - CHROME_DOWNLOAD_URL_BASE="https://gitlab.com/api/v4/projects/1075790/packages/generic/google-chrome-stable" - export DEBIAN_FRONTEND=noninteractive + CHROME_DOWNLOAD_URL_BASE="https://gitlab.com/api/v4/projects/1075790/packages/generic/${PKG}" + CHROME_DEB="${PKG}_${CHROME_VERSION}_${ARCH}.deb" + CHROME_URL="${CHROME_DOWNLOAD_URL_BASE}/${CHROME_VERSION}/${CHROME_DEB}" - curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add - - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >/etc/apt/sources.list.d/google.list - - apt-get update -q -y - - # Download from our package registry if we can't find the package in the apt repository - echo "Searching for $CHROME_VERSION in apt repository" - CHECK_VERSION=$(apt-cache show google-chrome-stable | grep Version | grep "$CHROME_VERSION") || true - - apt-cache policy google-chrome-stable - - if [[ -z $CHECK_VERSION ]]; then - CHROME_DEB="google-chrome-stable_${CHROME_VERSION}_${ARCH}.deb" - CHROME_URL="${CHROME_DOWNLOAD_URL_BASE}/${CHROME_VERSION}/${CHROME_DEB}" - echo "Downloading from our Package registry: $CHROME_URL" - curl --silent --show-error --fail -O $CHROME_URL - dpkg -i ./$CHROME_DEB || true - apt-get install -f -y - rm -f $CHROME_DEB - else - echo "Installing via apt-get" - apt-get install -y google-chrome-stable=$CHROME_VERSION - fi + echo "Downloading deb file from Package registry: $CHROME_URL" + curl --silent --show-error --fail -O $CHROME_URL + dpkg -i ./$CHROME_DEB || true + apt-get install -f -y + rm -f $CHROME_DEB rm -rf /var/lib/apt/lists/*