From 6b071e95121ad5c53104f2689b457a5d0b9ded57 Mon Sep 17 00:00:00 2001
From: Eidolon <furyhunter600@gmail.com>
Date: Thu, 16 May 2024 15:42:04 -0500
Subject: [PATCH] Use gitlab cache for vcpkg
Instead of using the install baked in the image, let's clone vcpkg
and set up a separate binary cache that can survive being cleaned.
This allows us to reuse built artifacts from the vcpkg manifest
baseline very quickly.
---
.gitlab/ci/jobs/macos-x86_64.yml | 75 +++++++++++++++++++++++++
.gitlab/ci/jobs/windows-x86.yml | 95 +++++++++++++++++++++++++++++---
2 files changed, 161 insertions(+), 9 deletions(-)
diff --git a/.gitlab/ci/jobs/macos-x86_64.yml b/.gitlab/ci/jobs/macos-x86_64.yml
index a8067d694c..3f0f3ca85c 100644
--- a/.gitlab/ci/jobs/macos-x86_64.yml
+++ b/.gitlab/ci/jobs/macos-x86_64.yml
@@ -3,6 +3,30 @@ osxcross x86_64:
stage: build
+ cache:
+ - key: ccache-$CI_PROJECT_PATH_SLUG-$CI_JOB_NAME_SLUG
+ fallback_keys:
+ - cache-$CI_PROJECT_PATH_SLUG-$CI_DEFAULT_BRANCH
+ - cache-$CI_PROJECT_PATH_SLUG-default
+ paths:
+ - ccache
+ - ccache_statslog
+
+ - key: apt-$CI_JOB_IMAGE
+ paths:
+ - apt-cache
+ unprotect: true
+
+ - key: vcpkg-root
+ paths:
+ - build/vcpkg-root
+ unprotect: true
+
+ - key: vcpkg-binary-cache-x64-osx
+ paths:
+ - build/vcpkg-binary-cache
+ unprotect: true
+
artifacts:
paths:
- "build.osxcross/bin/"
@@ -15,6 +39,27 @@ osxcross x86_64:
LD: x86_64-apple-darwin21.4-ld
script:
+ - |
+ # vcpkg
+ echo -e "\e[0Ksection_start:`date +%s`:vcpkg-root[collapsed=true]\r\e[0KUpdating vcpkg"
+
+ if [ -d "build/vcpkg-root" ]; then
+ pushd build/vcpkg-root
+ git fetch https://github.com/Microsoft/vcpkg master
+ git reset --hard FETCH_HEAD
+ popd
+ else
+ mkdir -p build
+ git clone https://github.com/Microsoft/vcpkg build/vcpkg-root
+ fi
+
+ export VCPKG_ROOT=$(pwd)/build/vcpkg-root
+ export VCPKG_BINARY_SOURCES="clear;files,$(pwd)/build/vcpkg-binary-cache,readwrite"
+
+ mkdir -p "build/vcpkg-binary-cache"
+
+ echo -e "\e[0Ksection_end:`date +%s`:vcpkg-root\r\e[0K"
+
- - |
# apt_development
echo -e "\e[0Ksection_start:`date +%s`:macports_development[collapsed=true]\r\e[0KInstalling development packages"
@@ -38,3 +83,33 @@ osxcross x86_64:
- |
# make
echo -e "\e[0Ksection_end:`date +%s`:make\r\e[0K"
+
+ after_script:
+ - - |
+ # apt_clean
+ echo -e "\e[0Ksection_start:`date +%s`:apt_clean[collapsed=true]\r\e[0KCleaning of unneeded APT packages"
+ - apt-get autoclean
+ - |
+ # apt_clean
+ echo -e "\e[0Ksection_end:`date +%s`:apt_clean\r\e[0K"
+
+ - - |
+ # vcpkg_clean
+ echo -e "\e[0Ksection_start:`date +%s`:vcpkg_clean[collapsed=true]\r\e[0KCleaning vcpkg-root"
+
+ if [ -d "build/vcpkg-root" ]; then
+ pushd "build/vcpkg-root"
+ git clean
+ popd
+ fi
+
+ echo -e "\e[0Ksection_end:`date +%s`:vcpkg_clean\r\e[0K"
+
+ - - |
+ # ccache_stats
+ echo -e "\e[0Ksection_start:`date +%s`:ccache_stats[collapsed=true]\r\e[0Kccache statistics:"
+ - ccache --show-stats
+ - ccache --show-log-stats || true
+ - |
+ # ccahe_stats
+ echo -e "\e[0Ksection_end:`date +%s`:ccache_stats\r\e[0K"
diff --git a/.gitlab/ci/jobs/windows-x86.yml b/.gitlab/ci/jobs/windows-x86.yml
index 0e5632012f..603549ce39 100644
--- a/.gitlab/ci/jobs/windows-x86.yml
+++ b/.gitlab/ci/jobs/windows-x86.yml
@@ -5,6 +5,30 @@ Windows x86:
when: on_success
+ cache:
+ - key: ccache-$CI_PROJECT_PATH_SLUG-$CI_JOB_NAME_SLUG
+ fallback_keys:
+ - cache-$CI_PROJECT_PATH_SLUG-$CI_DEFAULT_BRANCH
+ - cache-$CI_PROJECT_PATH_SLUG-default
+ paths:
+ - ccache
+ - ccache_statslog
+
+ - key: apt-$CI_JOB_IMAGE
+ paths:
+ - apt-cache
+ unprotect: true
+
+ - key: vcpkg-root
+ paths:
+ - build/vcpkg-root
+ unprotect: true
+
+ - key: vcpkg-binary-cache-x86-mingw-static
+ paths:
+ - build/vcpkg-binary-cache
+ unprotect: true
+
artifacts:
paths:
- "build/ninja-x86_mingw_static_vcpkg-debug/bin/"
@@ -18,6 +42,27 @@ Windows x86:
CXX: /usr/bin/i686-w64-mingw32-g++-posix
script:
+ - |
+ # vcpkg
+ echo -e "\e[0Ksection_start:`date +%s`:vcpkg-root[collapsed=true]\r\e[0KUpdating vcpkg"
+
+ if [ -d "build/vcpkg-root" ]; then
+ pushd build/vcpkg-root
+ git fetch https://github.com/Microsoft/vcpkg master
+ git reset --hard FETCH_HEAD
+ popd
+ else
+ mkdir -p build
+ git clone https://github.com/Microsoft/vcpkg build/vcpkg-root
+ fi
+
+ export VCPKG_ROOT=$(pwd)/build/vcpkg-root
+ export VCPKG_BINARY_SOURCES="clear;files,$(pwd)/build/vcpkg-binary-cache,readwrite"
+
+ mkdir -p "build/vcpkg-binary-cache"
+
+ echo -e "\e[0Ksection_end:`date +%s`:vcpkg-root\r\e[0K"
+
- - |
# apt_toolchain
echo -e "\e[0Ksection_start:`date +%s`:apt_toolchain[collapsed=true]\r\e[0KInstalling toolchain packages"
@@ -34,18 +79,20 @@ Windows x86:
# apt_development
echo -e "\e[0Ksection_end:`date +%s`:apt_development\r\e[0K"
- - - |
- # vcpkg_update
- echo -e "\e[0Ksection_start:`date +%s`:vcpkg_update[collapsed=true]\r\e[0KUpdating vcpkg"
- - git -C $VCPKG_ROOT pull
- - |
- # vcpkg_update
- echo -e "\e[0Ksection_end:`date +%s`:vcpkg_update\r\e[0K"
-
- - |
# cmake
echo -e "\e[0Ksection_start:`date +%s`:cmake[collapsed=false]\r\e[0KBuilding Makefiles"
- - cmake --preset ninja-x86_mingw_static_vcpkg-debug -DVCPKG_TARGET_TRIPLET=x86-mingw-static -DCMAKE_C_COMPILER=/usr/bin/i686-w64-mingw32-gcc-posix -DCMAKE_CXX_COMPILER=/usr/bin/i686-w64-mingw32-g++-posix -G "Unix Makefiles" -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/toolchains/mingw.cmake
+ - |
+ CCACHE=$(which ccache)
+
+ cmake \
+ --preset ninja-x86_mingw_static_vcpkg-debug \
+ -G "Unix Makefiles" \
+ -DCMAKE_C_COMPILER_LAUNCHER=$CCACHE \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=$CCACHE \
+ -DCMAKE_C_COMPILER=/usr/bin/i686-w64-mingw32-gcc-posix \
+ -DCMAKE_CXX_COMPILER=/usr/bin/i686-w64-mingw32-g++-posix \
+ -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/toolchains/mingw.cmake
- |
# cmake
@@ -58,3 +105,33 @@ Windows x86:
- |
# make
echo -e "\e[0Ksection_end:`date +%s`:make\r\e[0K"
+
+ after_script:
+ - - |
+ # apt_clean
+ echo -e "\e[0Ksection_start:`date +%s`:apt_clean[collapsed=true]\r\e[0KCleaning of unneeded APT packages"
+ - apt-get autoclean
+ - |
+ # apt_clean
+ echo -e "\e[0Ksection_end:`date +%s`:apt_clean\r\e[0K"
+
+ - - |
+ # vcpkg_clean
+ echo -e "\e[0Ksection_start:`date +%s`:vcpkg_clean[collapsed=true]\r\e[0KCleaning vcpkg-root"
+
+ if [ -d "build/vcpkg-root" ]; then
+ pushd "build/vcpkg-root"
+ git clean -f
+ popd
+ fi
+
+ echo -e "\e[0Ksection_end:`date +%s`:vcpkg_clean\r\e[0K"
+
+ - - |
+ # ccache_stats
+ echo -e "\e[0Ksection_start:`date +%s`:ccache_stats[collapsed=true]\r\e[0Kccache statistics:"
+ - ccache --show-stats
+ - ccache --show-log-stats || true
+ - |
+ # ccahe_stats
+ echo -e "\e[0Ksection_end:`date +%s`:ccache_stats\r\e[0K"
--
GitLab