From d9e13ab2ab670719735b9481ecadf996444e18d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20=22Pisco=22=20Fernandes?= Date: Thu, 4 Sep 2025 15:59:44 +0100 Subject: [PATCH] TUN-9803: Add windows builds to gitlab-ci --- .ci/ci-image.gitlab-ci.yml | 33 ++++ .ci/commons.gitlab-ci.yml | 31 ++++ .ci/image/.docker-images | 2 + .ci/image/Dockerfile | 25 +++ .ci/mac.gitlab-ci.yml | 66 ++++++++ .ci/release.gitlab-ci.yml | 39 +++++ {.teamcity => .ci/scripts}/mac/build.sh | 0 {.teamcity => .ci/scripts}/mac/install-go.sh | 0 {.teamcity => .ci/scripts}/package-windows.sh | 10 +- {.teamcity => .ci/scripts}/windows/builds.ps1 | 24 ++- .ci/scripts/windows/component-test.ps1 | 40 +++++ .ci/scripts/windows/go-wrapper.ps1 | 69 ++++++++ .ci/windows.gitlab-ci.yml | 80 +++++++++ .gitlab-ci.yml | 154 ++++-------------- .teamcity/windows/component-test.ps1 | 47 ------ Makefile | 14 +- cfsetup.yaml | 22 --- component-tests/test_token.py | 2 +- component-tests/util.py | 1 + 19 files changed, 443 insertions(+), 216 deletions(-) create mode 100644 .ci/ci-image.gitlab-ci.yml create mode 100644 .ci/commons.gitlab-ci.yml create mode 100644 .ci/image/.docker-images create mode 100644 .ci/image/Dockerfile create mode 100644 .ci/mac.gitlab-ci.yml create mode 100644 .ci/release.gitlab-ci.yml rename {.teamcity => .ci/scripts}/mac/build.sh (100%) rename {.teamcity => .ci/scripts}/mac/install-go.sh (100%) rename {.teamcity => .ci/scripts}/package-windows.sh (75%) rename {.teamcity => .ci/scripts}/windows/builds.ps1 (59%) create mode 100644 .ci/scripts/windows/component-test.ps1 create mode 100644 .ci/scripts/windows/go-wrapper.ps1 create mode 100644 .ci/windows.gitlab-ci.yml delete mode 100644 .teamcity/windows/component-test.ps1 diff --git a/.ci/ci-image.gitlab-ci.yml b/.ci/ci-image.gitlab-ci.yml new file mode 100644 index 00000000..ea5a8623 --- /dev/null +++ b/.ci/ci-image.gitlab-ci.yml @@ -0,0 +1,33 @@ +# Builds a custom CI Image when necessary + +include: + ##################################################### + ############## Build and Push CI Image ############## + ##################################################### + - component: $CI_SERVER_FQDN/cloudflare/ci/docker-image/build-push-image@~latest + inputs: + stage: pre-build + jobPrefix: ci-image + # runOnChangesTo: [".ci/image/**"] + # runOnMR: true + # runOnBranches: '^master$' + runOnBranches: "^.+$" + commentImageRefs: false + runner: vm-linux-x86-4cpu-8gb + EXTRA_DIB_ARGS: "--manifest=.ci/image/.docker-images" + + ##################################################### + ## Resolve the image reference for downstream jobs ## + ##################################################### + - component: $CI_SERVER_FQDN/cloudflare/ci/docker-image/get-image-ref@~latest + inputs: + stage: pre-build + jobPrefix: ci-image + # runOnMR: true + # runOnBranches: '^master$' + runOnBranches: "^.+$" + IMAGE_PATH: "$REGISTRY_HOST/stash/tun/cloudflared/ci-image/master" + VARIABLE_NAME: BUILD_IMAGE + needs: + - job: ci-image-build-push-image + optional: true diff --git a/.ci/commons.gitlab-ci.yml b/.ci/commons.gitlab-ci.yml new file mode 100644 index 00000000..7eea1e5e --- /dev/null +++ b/.ci/commons.gitlab-ci.yml @@ -0,0 +1,31 @@ +## A set of predefined rules to use on the different jobs +.default-rules: + # Rules to run the job only on the master branch + run-on-master: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: always + - when: never + # Rules to run the job only on branches that are not master. This is needed because for now + # we need to keep a similar behavior due to the integration with teamcity, which requires us + # to not trigger pipelines on tags and/or merge requests. + run-on-branch: + - if: $CI_COMMIT_TAG + when: never + - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + when: always + - when: never + +# This before_script is injected into every job that runs on master meaning that if there is no tag the step +# will succeed but only write "No tag present - Skipping" to the console. +.check-tag: + before_script: + - | + # Check if there is a Git tag pointing to HEAD + echo "Tag found: $(git tag --points-at HEAD | grep .)" + if git tag --points-at HEAD | grep .; then + echo "Tag found: $(git tag --points-at HEAD | grep .)" + export "VERSION=$(git tag --points-at HEAD | grep .)" + else + echo "No tag present — skipping." + exit 0 + fi \ No newline at end of file diff --git a/.ci/image/.docker-images b/.ci/image/.docker-images new file mode 100644 index 00000000..efa11f35 --- /dev/null +++ b/.ci/image/.docker-images @@ -0,0 +1,2 @@ +images: + - name: ci-image diff --git a/.ci/image/Dockerfile b/.ci/image/Dockerfile new file mode 100644 index 00000000..28513023 --- /dev/null +++ b/.ci/image/Dockerfile @@ -0,0 +1,25 @@ +ARG CLOUDFLARE_DOCKER_REGISTRY_HOST + +FROM ${CLOUDFLARE_DOCKER_REGISTRY_HOST:-registry.cfdata.org}/stash/cf/debian-images/bookworm/main:2025.7.0@sha256:6350da2f7e728dae2c1420f6dafc38e23cacc0b399d3d5b2f40fe48d9c8ff1ca + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install --no-install-recommends --allow-downgrades -y \ + build-essential \ + git \ + go-boring=1.24.4-1 \ + libffi-dev \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-venv \ + # libmsi and libgcab are libraries the wixl binary depends on. + libmsi-dev \ + libgcab-dev && \ + rm -rf /var/lib/apt/lists/* && \ + # Install wixl + curl -o /usr/local/bin/wixl -L https://pkg.cloudflare.com/binaries/wixl && \ + chmod a+x /usr/local/bin/wixl && \ + mkdir -p opt + +WORKDIR /opt diff --git a/.ci/mac.gitlab-ci.yml b/.ci/mac.gitlab-ci.yml new file mode 100644 index 00000000..1fc433de --- /dev/null +++ b/.ci/mac.gitlab-ci.yml @@ -0,0 +1,66 @@ +include: + - local: .ci/commons.gitlab-ci.yml + +############################### +### Defaults for Mac Builds ### +############################### +.mac-build-defaults: &mac-build-defaults + rules: + - !reference [.default-rules, run-on-branch] + tags: + - "macstadium-${RUNNER_ARCH}" + parallel: + matrix: + - RUNNER_ARCH: [arm, intel] + cache: {} + +###################################### +### Build Cloudflared Mac Binaries ### +###################################### +build-cloudflared-macos: &build-mac + <<: *mac-build-defaults + stage: build + artifacts: + paths: + - artifacts/* + script: + - '[ "${RUNNER_ARCH}" = "arm" ] && export TARGET_ARCH=arm64' + - '[ "${RUNNER_ARCH}" = "intel" ] && export TARGET_ARCH=amd64' + - ARCH=$(uname -m) + - echo ARCH=$ARCH - TARGET_ARCH=$TARGET_ARCH + - ./.ci/scripts/mac/install-go.sh + - BUILD_SCRIPT=.ci/scripts/mac/build.sh + - if [[ ! -x ${BUILD_SCRIPT} ]] ; then exit ; fi + - set -euo pipefail + - echo "Executing ${BUILD_SCRIPT}" + - exec ${BUILD_SCRIPT} + +############################################### +### Build and Sign Cloudflared Mac Binaries ### +############################################### +build-and-sign-cloudflared-macos: + <<: *build-mac + rules: + - !reference [.default-rules, run-on-master] + secrets: + APPLE_DEV_CA_CERT: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/apple_dev_ca_cert_v2/data@kv + file: false + CFD_CODE_SIGN_CERT: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_cert_v2/data@kv + file: false + CFD_CODE_SIGN_KEY: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_key_v2/data@kv + file: false + CFD_CODE_SIGN_PASS: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_pass_v2/data@kv + file: false + CFD_INSTALLER_CERT: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_cert_v2/data@kv + file: false + CFD_INSTALLER_KEY: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_key_v2/data@kv + file: false + CFD_INSTALLER_PASS: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_pass_v2/data@kv + file: false diff --git a/.ci/release.gitlab-ci.yml b/.ci/release.gitlab-ci.yml new file mode 100644 index 00000000..a41247cb --- /dev/null +++ b/.ci/release.gitlab-ci.yml @@ -0,0 +1,39 @@ +include: + - local: .ci/commons.gitlab-ci.yml + +########################################### +### Push Cloudflared Binaries to Github ### +########################################### +release-cloudflared-to-github: + stage: release + image: $BUILD_IMAGE + extends: .check-tag + needs: + - ci-image-get-image-ref + - package-windows + - build-and-sign-cloudflared-macos + rules: + - !reference [.default-rules, run-on-master] + cache: + paths: + - .cache/pip + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + KV_NAMESPACE: 380e19aa04314648949b6ad841417ebe + KV_ACCOUNT: 5ab4e9dfbd435d24068829fda0077963 + secrets: + KV_API_TOKEN: + vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_kv_api_token/data@kv + file: false + API_KEY: + vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_github_api_key/data@kv + file: false + script: + - python3 --version ; pip --version # For debugging + - python3 -m venv venv + - source venv/bin/activate + - pip install pynacl==1.4.0 pygithub==1.55 + - echo $VERSION + - echo $TAG_EXISTS + - echo "Running release because tag exists." + - make gitlab-release diff --git a/.teamcity/mac/build.sh b/.ci/scripts/mac/build.sh similarity index 100% rename from .teamcity/mac/build.sh rename to .ci/scripts/mac/build.sh diff --git a/.teamcity/mac/install-go.sh b/.ci/scripts/mac/install-go.sh similarity index 100% rename from .teamcity/mac/install-go.sh rename to .ci/scripts/mac/install-go.sh diff --git a/.teamcity/package-windows.sh b/.ci/scripts/package-windows.sh similarity index 75% rename from .teamcity/package-windows.sh rename to .ci/scripts/package-windows.sh index 6715af92..d0020f03 100755 --- a/.teamcity/package-windows.sh +++ b/.ci/scripts/package-windows.sh @@ -1,19 +1,23 @@ +#!/bin/bash +python3 -m venv env +. env/bin/activate +pip install pynacl==1.4.0 pygithub==1.55 + VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*") echo $VERSION export TARGET_OS=windows # This controls the directory the built artifacts go into -export BUILT_ARTIFACT_DIR=built_artifacts/ +export BUILT_ARTIFACT_DIR=artifacts/ export FINAL_ARTIFACT_DIR=artifacts/ mkdir -p $BUILT_ARTIFACT_DIR mkdir -p $FINAL_ARTIFACT_DIR windowsArchs=("amd64" "386") for arch in ${windowsArchs[@]}; do export TARGET_ARCH=$arch - # Copy exe into final directory + # Copy .exe from artifacts directory cp $BUILT_ARTIFACT_DIR/cloudflared-windows-$arch.exe ./cloudflared.exe make cloudflared-msi # Copy msi into final directory mv cloudflared-$VERSION-$arch.msi $FINAL_ARTIFACT_DIR/cloudflared-windows-$arch.msi - cp $BUILT_ARTIFACT_DIR/cloudflared-windows-$arch.exe $FINAL_ARTIFACT_DIR/cloudflared-windows-$arch.exe done diff --git a/.teamcity/windows/builds.ps1 b/.ci/scripts/windows/builds.ps1 similarity index 59% rename from .teamcity/windows/builds.ps1 rename to .ci/scripts/windows/builds.ps1 index ee43493f..e4a42ea2 100644 --- a/.teamcity/windows/builds.ps1 +++ b/.ci/scripts/windows/builds.ps1 @@ -2,27 +2,23 @@ Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" -# Relative path to working directory -$CloudflaredDirectory = "go\src\github.com\cloudflare\cloudflared" +$env:TARGET_OS = "windows" +$env:LOCAL_OS = "windows" -cd $CloudflaredDirectory +New-Item -Path ".\artifacts" -ItemType Directory Write-Output "Building for amd64" -$env:TARGET_OS = "windows" -$env:CGO_ENABLED = 1 $env:TARGET_ARCH = "amd64" -$env:Path = "$Env:Temp\go\bin;$($env:Path)" - -go env -go version - +$env:LOCAL_ARCH = "amd64" +$env:CGO_ENABLED = 1 & make cloudflared if ($LASTEXITCODE -ne 0) { throw "Failed to build cloudflared for amd64" } -copy .\cloudflared.exe .\cloudflared-windows-amd64.exe +copy .\cloudflared.exe .\artifacts\cloudflared-windows-amd64.exe Write-Output "Building for 386" -$env:CGO_ENABLED = 0 $env:TARGET_ARCH = "386" -make cloudflared +$env:LOCAL_ARCH = "386" +$env:CGO_ENABLED = 0 +& make cloudflared if ($LASTEXITCODE -ne 0) { throw "Failed to build cloudflared for 386" } -copy .\cloudflared.exe .\cloudflared-windows-386.exe \ No newline at end of file +copy .\cloudflared.exe .\artifacts\cloudflared-windows-386.exe diff --git a/.ci/scripts/windows/component-test.ps1 b/.ci/scripts/windows/component-test.ps1 new file mode 100644 index 00000000..abf024ff --- /dev/null +++ b/.ci/scripts/windows/component-test.ps1 @@ -0,0 +1,40 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$env:TARGET_OS = "windows" +$env:LOCAL_OS = "windows" +$env:TARGET_ARCH = "amd64" +$env:LOCAL_ARCH = "amd64" +$env:CGO_ENABLED = 1 + +python --version +python -m pip --version + + +Write-Host "Building cloudflared" +& make cloudflared +if ($LASTEXITCODE -ne 0) { throw "Failed to build cloudflared" } + + +Write-Host "Running unit tests" +# Not testing with race detector because of https://github.com/golang/go/issues/61058 +# We already test it on other platforms +go test -failfast -v -mod=vendor ./... +if ($LASTEXITCODE -ne 0) { throw "Failed unit tests" } + + +# On Gitlab runners we need to add all of this addresses to the NO_PROXY list in order for the tests to run. +$env:NO_PROXY = "pypi.org,files.pythonhosted.org,api.cloudflare.com,argotunneltest.com,argotunnel.com,trycloudflare.com,${env:NO_PROXY}" +Write-Host "No Proxy: ${env:NO_PROXY}" +Write-Host "Running component tests" +try { + python -m pip --disable-pip-version-check install --upgrade -r component-tests/requirements.txt --use-pep517 + python component-tests/setup.py --type create + python -m pytest component-tests -o log_cli=true --log-cli-level=INFO + if ($LASTEXITCODE -ne 0) { + throw "Failed component tests" + } +} finally { + python component-tests/setup.py --type cleanup +} diff --git a/.ci/scripts/windows/go-wrapper.ps1 b/.ci/scripts/windows/go-wrapper.ps1 new file mode 100644 index 00000000..217553a3 --- /dev/null +++ b/.ci/scripts/windows/go-wrapper.ps1 @@ -0,0 +1,69 @@ +Param( + [string]$GoVersion, + [string]$ScriptToExecute +) + +# This script its a wrapper that downloads a specific version +# of go, adds it to the PATH and executes a script with that go +# version in the path. + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +# Get the path to the system's temporary directory. +$tempPath = [System.IO.Path]::GetTempPath() + +# Create a unique name for the new temporary folder. +$folderName = "go_" + (Get-Random) + +# Join the temp path and the new folder name to create the full path. +$fullPath = Join-Path -Path $tempPath -ChildPath $folderName + +# Store the current value of PATH environment variable. +$oldPath = $env:Path + +# Use a try...finally block to ensure the temporrary folder and PATH are cleaned up. +try { + # Create the temporary folder. + Write-Host "Creating temporary folder at: $fullPath" + $newTempFolder = New-Item -ItemType Directory -Path $fullPath -Force + + # Download go + $url = "https://go.dev/dl/$GoVersion.windows-amd64.zip" + $destinationFile = Join-Path -Path $newTempFolder.FullName -ChildPath "go$GoVersion.windows-amd64.zip" + Write-Host "Downloading go from: $url" + Invoke-WebRequest -Uri $url -OutFile $destinationFile + Write-Host "File downloaded to: $destinationFile" + + # Unzip the downloaded file. + Write-Host "Unzipping the file..." + Expand-Archive -Path $destinationFile -DestinationPath $newTempFolder.FullName -Force + Write-Host "File unzipped successfully." + + # Define the go/bin path wich is inside the temporary folder + $goBinPath = Join-Path -Path $fullPath -ChildPath "go\bin" + + # Add the go/bin path to the PATH environment variable. + $env:Path = "$goBinPath;$($env:Path)" + Write-Host "Added $goBinPath to the environment PATH." + + go env + go version + + & $ScriptToExecute +} finally { + # Cleanup: Remove the path from the environment variable and then the temporary folder. + Write-Host "Starting cleanup..." + + $env:Path = $oldPath + Write-Host "Reverted changes in the environment PATH." + + # Remove the temporary folder and its contents. + if (Test-Path -Path $fullPath) { + Remove-Item -Path $fullPath -Recurse -Force + Write-Host "Temporary folder and its contents have been removed." + } else { + Write-Host "Temporary folder does not exist, no cleanup needed." + } +} diff --git a/.ci/windows.gitlab-ci.yml b/.ci/windows.gitlab-ci.yml new file mode 100644 index 00000000..256c7ff4 --- /dev/null +++ b/.ci/windows.gitlab-ci.yml @@ -0,0 +1,80 @@ +include: + - local: .ci/commons.gitlab-ci.yml + +################################### +### Defaults for Windows Builds ### +################################### +.windows-build-defaults: &windows-build-defaults + rules: + - !reference [.default-rules, run-on-branch] + tags: + - windows-x86 + cache: {} + +########################################## +### Build Cloudflared Windows Binaries ### +########################################## +build-cloudflared-windows: + <<: *windows-build-defaults + stage: build + script: + - powershell -ExecutionPolicy Bypass -File ".\.ci\scripts\windows\go-wrapper.ps1" "${GO_VERSION}" ".\.ci\scripts\windows\builds.ps1" + artifacts: + paths: + - artifacts/* + +###################################################### +### Load Environment Variables for Component Tests ### +###################################################### +load-windows-env-variables: + rules: + - !reference [.default-rules, run-on-branch] + stage: pre-build + script: + - echo "COMPONENT_TESTS_CONFIG=component-test-config.yaml" >> windows.env + - echo "COMPONENT_TESTS_CONFIG_CONTENT=Y2xvdWRmbGFyZWRfYmluYXJ5OiBjbG91ZGZsYXJlZC5leGUKY3JlZGVudGlhbHNfZmlsZTogY3JlZC5qc29uCm9yaWdpbmNlcnQ6IGNlcnQucGVtCnpvbmVfZG9tYWluOiBhcmdvdHVubmVsdGVzdC5jb20Kem9uZV90YWc6IDQ4Nzk2ZjFlNzBiYjc2NjljMjliYjUxYmEyODJiZjY1" >> windows.env + - echo "DNS_API_TOKEN=$DNS_API_TOKEN" >> windows.env + # We have to encode the `COMPONENT_TESTS_ORIGINCERT` secret, because it content is a file, otherwise we can't export it using gitlab + - echo "COMPONENT_TESTS_ORIGINCERT=$(echo "$COMPONENT_TESTS_ORIGINCERT" | base64 -w0)" >> windows.env + secrets: + DNS_API_TOKEN: + vault: gitlab/cloudflare/tun/cloudflared/_dev/_terraform_atlantis/component_tests_token/data@kv + file: false + COMPONENT_TESTS_ORIGINCERT: + vault: gitlab/cloudflare/tun/cloudflared/_dev/component_tests_cert_pem/data@kv + file: false + artifacts: + access: 'none' + reports: + dotenv: windows.env + cache: {} + +################################### +### Run Windows Component Tests ### +################################### +component-tests-cloudflared-windows: + <<: *windows-build-defaults + stage: test + needs: ["load-windows-env-variables"] + script: + # We have to decode the secret we encoded on the `load-windows-env-variables` job + - $env:COMPONENT_TESTS_ORIGINCERT = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($env:COMPONENT_TESTS_ORIGINCERT)) + - powershell -ExecutionPolicy Bypass -File ".\.ci\scripts\windows\go-wrapper.ps1" "${GO_VERSION}" ".\.ci\scripts\windows\component-test.ps1" + +################################ +### Package Windows Binaries ### +################################ +package-windows: + rules: + - !reference [.default-rules, run-on-master] + stage: package + needs: + - ci-image-get-image-ref + - build-cloudflared-windows + image: $BUILD_IMAGE + script: + - .ci/scripts/package-windows.sh + cache: {} + artifacts: + paths: + - artifacts/* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9059de5e..a4ea2864 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ variables: GOPATH: "$CI_PROJECT_DIR/.go" GOMODCACHE: "$GOPATH/pkg/mod" GO_BIN_DIR: "$GOPATH/bin" + GO_VERSION: "go1.24.4" cache: # Cache Go modules and the binaries. @@ -15,49 +16,42 @@ cache: - ${GOPATH}/pkg/mod/ # For Go modules - ${GO_BIN_DIR}/ -stages: [build, release] - default: id_tokens: VAULT_ID_TOKEN: aud: https://vault.cfdata.org -# This before_script is injected into every job that runs on master meaning that if there is no tag the step -# will succeed but only write "No tag present - Skipping" to the console. -.check_tag: - before_script: - - | - # Check if there is a Git tag pointing to HEAD - echo "Tag found: $(git tag --points-at HEAD | grep .)" - if git tag --points-at HEAD | grep .; then - echo "Tag found: $(git tag --points-at HEAD | grep .)" - export "VERSION=$(git tag --points-at HEAD | grep .)" - else - echo "No tag present — skipping." - exit 0 - fi +stages: [pre-build, build, test, package, release] -## A set of predefined rules to use on the different jobs -.default_rules: - # Rules to run the job only on the master branch - run_on_master: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: always - - when: never - # Rules to run the job only on branches that are not master. This is needed because for now - # we need to keep a similar behavior due to the integration with teamcity, which requires us - # to not trigger pipelines on tags and/or merge requests. - run_on_branch: - - if: $CI_COMMIT_TAG - when: never - - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH - when: always - - when: never +include: + ##################################################### + ########## Import Commons Configurations ############ + ##################################################### + - local: .ci/commons.gitlab-ci.yml + ##################################################### + ############# Build or Fetch CI Image ############### + ##################################################### + - local: .ci/ci-image.gitlab-ci.yml + + ##################################################### + ################## Windows Builds ################### + ##################################################### + - local: .ci/windows.gitlab-ci.yml + + ##################################################### + ################### macOS Builds #################### + ##################################################### + - local: .ci/mac.gitlab-ci.yml + + ##################################################### + ################# Release Packages ################## + ##################################################### + - local: .ci/release.gitlab-ci.yml # Template for Go setup, including caching and installation .go_setup: - image: docker-registry.cfdata.org/stash/devtools/ci-builders/golang-1.24/master:3090-3e32590@sha256:fc81df4f8322f022d93712ee40bb1e5752fdbe9868d1e5a23fd851ad6fbecb91 + image: docker-registry.cfdata.org/stash/devtools/ci-builders/golang-1.24/master:3219-cc8b513@sha256:4fe3ff47ba07f9b23429f49fbd063cc1a34156dd11b8113e325ad3762f94a1db before_script: - mkdir -p ${GOPATH} ${GOMODCACHE} ${GO_BIN_DIR} - export PATH=$PATH:${GO_BIN_DIR} @@ -71,102 +65,10 @@ default: echo "govulncheck found in cache, skipping installation."; fi -# ----------------------------------------------- -# Stage 1: Build on every PR -# ----------------------------------------------- -build_cloudflared_macos: &build - stage: build - rules: - - !reference [.default_rules, run_on_branch] - tags: - - "macstadium-${RUNNER_ARCH}" - parallel: - matrix: - - RUNNER_ARCH: [arm, intel] - artifacts: - paths: - - artifacts/* - script: - - '[ "${RUNNER_ARCH}" = "arm" ] && export TARGET_ARCH=arm64' - - '[ "${RUNNER_ARCH}" = "intel" ] && export TARGET_ARCH=amd64' - - ARCH=$(uname -m) - - echo ARCH=$ARCH - TARGET_ARCH=$TARGET_ARCH - - ./.teamcity/mac/install-go.sh - - BUILD_SCRIPT=.teamcity/mac/build.sh - - if [[ ! -x ${BUILD_SCRIPT} ]] ; then exit ; fi - - set -euo pipefail - - echo "Executing ${BUILD_SCRIPT}" - - exec ${BUILD_SCRIPT} - vulncheck: stage: build extends: .go_setup rules: - - !reference [.default_rules, run_on_branch] + - !reference [.default-rules, run-on-branch] script: - make vulncheck - -# ----------------------------------------------- -# Stage 1: Build and sign only on releases -# ----------------------------------------------- -build_and_sign_cloudflared_macos: - <<: *build - rules: - - !reference [.default_rules, run_on_master] - secrets: - APPLE_DEV_CA_CERT: - vault: gitlab/cloudflare/tun/cloudflared/_branch/master/apple_dev_ca_cert_v2/data@kv - file: false - CFD_CODE_SIGN_CERT: - vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_cert_v2/data@kv - file: false - CFD_CODE_SIGN_KEY: - vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_key_v2/data@kv - file: false - CFD_CODE_SIGN_PASS: - vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_pass_v2/data@kv - file: false - CFD_INSTALLER_CERT: - vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_cert_v2/data@kv - file: false - CFD_INSTALLER_KEY: - vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_key_v2/data@kv - file: false - CFD_INSTALLER_PASS: - vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_pass_v2/data@kv - file: false - -# ----------------------------------------------- -# Stage 2: Release to Github after building and signing -# ----------------------------------------------- -release_cloudflared_macos_to_github: - stage: release - image: docker-registry.cfdata.org/stash/tun/docker-images/cloudflared-ci/main:6-8616fe631b76-amd64@sha256:96f4fd05e66cec03e0864c1bcf09324c130d4728eef45ee994716da499183614 - extends: .check_tag - dependencies: - - build_and_sign_cloudflared_macos - rules: - - !reference [.default_rules, run_on_master] - cache: - paths: - - .cache/pip - variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - KV_NAMESPACE: 380e19aa04314648949b6ad841417ebe - KV_ACCOUNT: 5ab4e9dfbd435d24068829fda0077963 - secrets: - KV_API_TOKEN: - vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_kv_api_token/data@kv - file: false - API_KEY: - vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_github_api_key/data@kv - file: false - script: - - python3 --version ; pip --version # For debugging - - python3 -m venv venv - - source venv/bin/activate - - pip install pynacl==1.4.0 pygithub==1.55 - - echo $VERSION - - echo $TAG_EXISTS - - echo "Running release because tag exists." - - make macos-release diff --git a/.teamcity/windows/component-test.ps1 b/.teamcity/windows/component-test.ps1 deleted file mode 100644 index f8e716d0..00000000 --- a/.teamcity/windows/component-test.ps1 +++ /dev/null @@ -1,47 +0,0 @@ -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" -$ProgressPreference = "SilentlyContinue" - -$WorkingDirectory = Get-Location -$CloudflaredDirectory = "$WorkingDirectory\go\src\github.com\cloudflare\cloudflared" - -go env -go version - -$env:TARGET_OS = "windows" -$env:CGO_ENABLED = 1 -$env:TARGET_ARCH = "amd64" -$env:Path = "$Env:Temp\go\bin;$($env:Path)" - -python --version -python -m pip --version - -cd $CloudflaredDirectory - -go env -go version - -Write-Output "Building cloudflared" - -& make cloudflared -if ($LASTEXITCODE -ne 0) { throw "Failed to build cloudflared" } - -echo $LASTEXITCODE - -Write-Output "Running unit tests" - -# Not testing with race detector because of https://github.com/golang/go/issues/61058 -# We already test it on other platforms -go test -failfast -v -mod=vendor ./... -if ($LASTEXITCODE -ne 0) { throw "Failed unit tests" } - -Write-Output "Running component tests" - -python -m pip --disable-pip-version-check install --upgrade -r component-tests/requirements.txt --use-pep517 -python component-tests/setup.py --type create -python -m pytest component-tests -o log_cli=true --log-cli-level=INFO -if ($LASTEXITCODE -ne 0) { - python component-tests/setup.py --type cleanup - throw "Failed component tests" -} -python component-tests/setup.py --type cleanup diff --git a/Makefile b/Makefile index e9b411dc..fa694c94 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,13 @@ else DEB_PACKAGE_NAME := $(BINARY_NAME) endif -DATE := $(shell date -u -r RELEASE_NOTES '+%Y-%m-%d-%H%M UTC') +# Use git in windows since we don't have access to the `date` tool +ifeq ($(TARGET_OS), windows) + DATE := $(shell git log -1 --format="%ad" --date=format-local:'%Y-%m-%dT%H:%M UTC' -- RELEASE_NOTES) +else + DATE := $(shell date -u -r RELEASE_NOTES '+%Y-%m-%d-%H:%M UTC') +endif + VERSION_FLAGS := -X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)" ifdef PACKAGE_MANAGER VERSION_FLAGS := $(VERSION_FLAGS) -X "github.com/cloudflare/cloudflared/cmd/cloudflared/updater.BuiltForPackageManager=$(PACKAGE_MANAGER)" @@ -64,6 +70,8 @@ else ifeq ($(LOCAL_ARCH),x86_64) TARGET_ARCH ?= amd64 else ifeq ($(LOCAL_ARCH),amd64) TARGET_ARCH ?= amd64 +else ifeq ($(LOCAL_ARCH),386) + TARGET_ARCH ?= 386 else ifeq ($(LOCAL_ARCH),i686) TARGET_ARCH ?= amd64 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 5),armv8) @@ -230,8 +238,8 @@ github-release: python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) python3 github_message.py --release-version $(VERSION) -.PHONY: macos-release -macos-release: +.PHONY: gitlab-release +gitlab-release: python3 github_release.py --path $(PWD)/artifacts/ --release-version $(VERSION) .PHONY: r2-linux-release diff --git a/cfsetup.yaml b/cfsetup.yaml index dd01f650..aba6925e 100644 --- a/cfsetup.yaml +++ b/cfsetup.yaml @@ -123,28 +123,6 @@ bookworm: &bookworm - export GOOS=linux - export GOARCH=arm64 - make cloudflared-deb - package-windows: - build_dir: *build_dir - builddeps: - - *pinned_go - - build-essential - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - - wget - # libmsi and libgcab are libraries the wixl binary depends on. - - libmsi-dev - - libgcab-dev - - python3-venv - pre-cache: - - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin - - chmod a+x /usr/local/bin/wixl - post-cache: - - python3 -m venv env - - . env/bin/activate - - pip install pynacl==1.4.0 pygithub==1.55 - - .teamcity/package-windows.sh test: build_dir: *build_dir builddeps: &build_deps_tests diff --git a/component-tests/test_token.py b/component-tests/test_token.py index ed99cb05..36b37239 100644 --- a/component-tests/test_token.py +++ b/component-tests/test_token.py @@ -1,7 +1,7 @@ import base64 import json -from setup import get_config_from_file, persist_origin_cert +from setup import get_config_from_file from util import start_cloudflared diff --git a/component-tests/util.py b/component-tests/util.py index ee63c24a..c7fd59c4 100644 --- a/component-tests/util.py +++ b/component-tests/util.py @@ -111,6 +111,7 @@ def inner_wait_tunnel_ready(tunnel_url=None, require_min_connections=1): metrics_url = f'http://localhost:{METRICS_PORT}/ready' with requests.Session() as s: + LOGGER.debug("Waiting for tunnel to be ready...") resp = send_request(s, metrics_url, True) ready_connections = resp.json()["readyConnections"]