curben
/
cloudflared-mirror
mirror of https://github.com/cloudflare/cloudflared.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

TUN-9800: Add pipelines for linux packaging

This commit is contained in:
João "Pisco" Fernandes 2025-10-31 14:43:50 +00:00
parent 70658b863b
commit a8fdbb83d0
11 changed files with 229 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.ci/commons.gitlab-ci.yml
View File

@ -3,14 +3,14 @@
# Rules to run the job only on the master branch
run-on-master:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: always
when: on_success
- when: never
# Rules to run the job only on merge requests
run-on-mr:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
when: always
when: on_success
- when: never
# Rules to run the job on merge_requests and master branch
run-always:
@ -18,7 +18,7 @@
when: never
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH != null && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: always
when: on_success
- when: never
# This before_script is injected into every job that runs on master meaning that if there is no tag the step

8
.ci/image/Dockerfile
View File

@ -16,7 +16,13 @@ RUN apt-get update && \
python3-venv \
# libmsi and libgcab are libraries the wixl binary depends on.
libmsi-dev \
libgcab-dev && \
libgcab-dev \
# deb and rpm build tools
rubygem-fpm \
rpm \
# create deb and rpm repository files
reprepro \
createrepo-c && \
rm -rf /var/lib/apt/lists/* && \
# Install wixl
curl -o /usr/local/bin/wixl -L https://pkg.cloudflare.com/binaries/wixl && \

31
.ci/linux.gitlab-ci.yml
View File

@ -8,6 +8,18 @@
imageVersion: "3371-f5539bd6f83d@sha256:a2a68f580070f9411d0d3155959ed63b700ef319b5fcc62db340e92227bbc628"
CGO_ENABLED: 1
.default-packaging-job: &packaging-job-defaults
stage: build
needs:
- ci-image-get-image-ref
rules:
- !reference [.default-rules, run-on-master]
image: $BUILD_IMAGE
cache: {}
artifacts:
paths:
- artifacts/*
include:
###################
### Linux Build ###
@ -89,3 +101,22 @@ component-tests-linux-fips:
variables:
<<: *component-tests-variables
COMPONENT_TESTS_FIPS: 1
################################
####### Linux Packaging ########
################################
linux-packaging:
<<: *packaging-job-defaults
parallel:
matrix:
- ARCH: ["386", "amd64", "arm", "armhf", "arm64"]
script:
- ./.ci/scripts/linux/build-packages.sh ${ARCH}
################################
##### Linux FIPS Packaging #####
################################
linux-packaging-fips:
<<: *packaging-job-defaults
script:
- ./.ci/scripts/linux/build-packages-fips.sh

120
.ci/release.gitlab-ci.yml
View File

@ -1,17 +1,28 @@
include:
- local: .ci/commons.gitlab-ci.yml
###########################################
### Push Cloudflared Binaries to Github ###
###########################################
release-cloudflared-to-github:
######################################
### Build and Push DockerHub Image ###
######################################
- component: $CI_SERVER_FQDN/cloudflare/ci/docker-image/build-push-image@~latest
inputs:
stage: release
jobPrefix: docker-hub
runOnMR: false
runOnBranches: '^master$'
runOnChangesTo: ['RELEASE_NOTES']
needs:
- generate-version-file
- release-cloudflared-to-r2
commentImageRefs: false
runner: vm-linux-x86-4cpu-8gb
DOCKER_USER_BRANCH: svcgithubdockerhubcloudflar045
DOCKER_PASSWORD_BRANCH: gitlab/cloudflare/tun/cloudflared/_dev/dockerhub/svc_password/data
EXTRA_DIB_ARGS: --overwrite
.default-release-job: &release-job-defaults
stage: release
image: $BUILD_IMAGE
extends: .check-tag
needs:
- ci-image-get-image-ref
- package-windows
- build-and-sign-cloudflared-macos
rules:
- !reference [.default-rules, run-on-master]
cache:
@ -19,8 +30,16 @@ release-cloudflared-to-github:
- .cache/pip
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
# KV Vars
KV_NAMESPACE: 380e19aa04314648949b6ad841417ebe
KV_ACCOUNT: 5ab4e9dfbd435d24068829fda0077963
KV_ACCOUNT: &cf-account 5ab4e9dfbd435d24068829fda0077963
# R2 Vars
R2_BUCKET: cloudflared-pkgs
R2_ACCOUNT_ID: *cf-account
# APT and RPM Repository Vars
GPG_PUBLIC_KEY_URL: "https://pkg.cloudflare.com/cloudflare-ascii-pubkey.gpg"
PKG_URL: "https://pkg.cloudflare.com/cloudflared"
BINARY_NAME: cloudflared
secrets:
KV_API_TOKEN:
vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_kv_api_token/data@kv
@ -28,12 +47,77 @@ release-cloudflared-to-github:
API_KEY:
vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_github_api_key/data@kv
file: false
R2_CLIENT_ID:
vault: gitlab/cloudflare/tun/cloudflared/_dev/_terraform_atlantis/r2_api_token/client_id@kv
file: false
R2_CLIENT_SECRET:
vault: gitlab/cloudflare/tun/cloudflared/_dev/_terraform_atlantis/r2_api_token/client_secret@kv
file: false
LINUX_SIGNING_PUBLIC_KEY:
vault: gitlab/cloudflare/tun/cloudflared/_dev/gpg_v1/public_key@kv
file: false
LINUX_SIGNING_PRIVATE_KEY:
vault: gitlab/cloudflare/tun/cloudflared/_dev/gpg_v1/private_key@kv
file: false
LINUX_SIGNING_PUBLIC_KEY_2:
vault: gitlab/cloudflare/tun/cloudflared/_dev/gpg_v2/public_key@kv
file: false
LINUX_SIGNING_PRIVATE_KEY_2:
vault: gitlab/cloudflare/tun/cloudflared/_dev/gpg_v2/private_key@kv
file: false
###########################################
### Push Cloudflared Binaries to Github ###
###########################################
release-cloudflared-to-github:
<<: *release-job-defaults
extends: .check-tag
needs:
- build-and-sign-cloudflared-macos
- ci-image-get-image-ref
- linux-packaging
- linux-packaging-fips
- package-windows
script:
- python3 --version ; pip --version # For debugging
- python3 -m venv venv
- source venv/bin/activate
- pip install pynacl==1.4.0 pygithub==1.55
- echo $VERSION
- echo $TAG_EXISTS
- echo "Running release because tag exists."
- make gitlab-release
- ./.ci/scripts/release-target.sh github-release
#########################################
### Upload Cloudflared Binaries to R2 ###
#########################################
release-cloudflared-to-r2:
<<: *release-job-defaults
extends: .check-tag
needs:
- ci-image-get-image-ref
- linux-packaging # We only release non-FIPS binaries to R2
- release-cloudflared-to-github
script:
- ./.ci/scripts/release-target.sh r2-linux-release
#################################################
### Upload Cloudflared Nightly Binaries to R2 ###
#################################################
release-cloudflared-nightly-to-r2:
<<: *release-job-defaults
variables:
R2_BUCKET: cloudflared-pkgs-next
GPG_PUBLIC_KEY_URL: "https://next.pkg.cloudflare.com/cloudflare-ascii-pubkey.gpg"
PKG_URL: "https://next.pkg.cloudflare.com/cloudflared"
needs:
- ci-image-get-image-ref
- linux-packaging # We only release non-FIPS binaries to R2
script:
- ./.ci/scripts/release-target.sh r2-linux-release
#############################
### Generate Version File ###
#############################
generate-version-file:
<<: *release-job-defaults
needs:
- ci-image-get-image-ref
script:
- make generate-docker-version
artifacts:
paths:
- versions

0
build-packages-fips.sh → .ci/scripts/linux/build-packages-fips.sh
View File

59
.ci/scripts/linux/build-packages.sh Executable file
View File

@ -0,0 +1,59 @@
#!/bin/bash
# Check if architecture argument is provided
if [ $# -eq 0 ]; then
echo "Error: Architecture argument is required"
echo "Usage: $0 <architecture>"
exit 1
fi
# Parameters
arch=$1
# Get Version
VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
echo $VERSION
# Disable FIPS module in go-boring
export GOEXPERIMENT=noboringcrypto
export CGO_ENABLED=0
# This controls the directory the built artifacts go into
export ARTIFACT_DIR=artifacts/
mkdir -p $ARTIFACT_DIR
export TARGET_OS=linux
unset TARGET_ARM
export TARGET_ARCH=$arch
## Support for arm platforms without hardware FPU enabled
if [[ $arch == arm ]] ; then
export TARGET_ARCH=arm
export TARGET_ARM=5
fi
## Support for armhf builds
if [[ $arch == armhf ]] ; then
export TARGET_ARCH=arm
export TARGET_ARM=7
fi
make cloudflared-deb
mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb
# rpm packages invert the - and _ and use x86_64 instead of amd64.
RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g')
RPMARCH=$arch
if [ $arch == "amd64" ];then
RPMARCH="x86_64"
fi
if [ $arch == "arm64" ]; then
RPMARCH="aarch64"
fi
make cloudflared-rpm
mv cloudflared-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-linux-$RPMARCH.rpm
# finally move the linux binary as well.
mv ./cloudflared $ARTIFACT_DIR/cloudflared-linux-$arch

18
.ci/scripts/release-target.sh Executable file
View File

@ -0,0 +1,18 @@
#!/bin/bash
set -e -o pipefail
# Check if a make target is provided as an argument
if [ $# -eq 0 ]; then
echo "Error: Make target argument is required"
echo "Usage: $0 <make-target>"
exit 1
fi
MAKE_TARGET=$1
python3 -m venv venv
source venv/bin/activate
# Our release scripts are written in python, so we should install their dependecies here.
pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9
make $MAKE_TARGET

10
Makefile
View File

@ -221,10 +221,6 @@ cloudflared-deb: cloudflared cloudflared.1
cloudflared-rpm: cloudflared cloudflared.1
$(call build_package,rpm)
.PHONY: cloudflared-pkg
cloudflared-pkg: cloudflared cloudflared.1
$(call build_package,osxpkg)
.PHONY: cloudflared-msi
cloudflared-msi:
wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs
@ -235,12 +231,8 @@ github-release-dryrun:
.PHONY: github-release
github-release:
python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
python3 github_message.py --release-version $(VERSION)
.PHONY: gitlab-release
gitlab-release:
python3 github_release.py --path $(PWD)/artifacts/ --release-version $(VERSION)
python3 github_message.py --release-version $(VERSION)
.PHONY: r2-linux-release
r2-linux-release:

48
build-packages.sh
View File

@ -1,48 +0,0 @@
#!/bin/bash
VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
echo $VERSION
# Disable FIPS module in go-boring
export GOEXPERIMENT=noboringcrypto
export CGO_ENABLED=0
# This controls the directory the built artifacts go into
export ARTIFACT_DIR=artifacts/
mkdir -p $ARTIFACT_DIR
linuxArchs=("386" "amd64" "arm" "armhf" "arm64")
export TARGET_OS=linux
for arch in ${linuxArchs[@]}; do
unset TARGET_ARM
export TARGET_ARCH=$arch
## Support for arm platforms without hardware FPU enabled
if [[ $arch == arm ]] ; then
export TARGET_ARCH=arm
export TARGET_ARM=5
fi
## Support for armhf builds
if [[ $arch == armhf ]] ; then
export TARGET_ARCH=arm
export TARGET_ARM=7
fi
make cloudflared-deb
mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb
# rpm packages invert the - and _ and use x86_64 instead of amd64.
RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g')
RPMARCH=$arch
if [ $arch == "amd64" ];then
RPMARCH="x86_64"
fi
if [ $arch == "arm64" ]; then
RPMARCH="aarch64"
fi
make cloudflared-rpm
mv cloudflared-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-linux-$RPMARCH.rpm
# finally move the linux binary as well.
mv ./cloudflared $ARTIFACT_DIR/cloudflared-linux-$arch
done

201
cfsetup.yaml
View File

@ -4,85 +4,6 @@ build_dir: &build_dir /cfsetup_build
default-flavor: bookworm
bookworm: &bookworm
build-linux:
build_dir: *build_dir
builddeps: &build_deps
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
- rpm
- libffi-dev
- golangci-lint=1.64.8-2
pre-cache: &build_pre_cache
- export GOCACHE=/cfsetup_build/.cache/go-build
- go install golang.org/x/tools/cmd/goimports@v0.30.0
post-cache:
# Linting
- make lint
- make fmt-check
# Build binary for component test
- GOOS=linux GOARCH=amd64 make cloudflared
build-linux-fips:
build_dir: *build_dir
builddeps: *build_deps
pre-cache: *build_pre_cache
post-cache:
- export FIPS=true
# Build binary for component test
- GOOS=linux GOARCH=amd64 make cloudflared
cover:
build_dir: *build_dir
builddeps: *build_deps
pre-cache: *build_pre_cache
post-cache:
- make cover
# except FIPS and macos
build-linux-release:
build_dir: *build_dir
builddeps: &build_deps_release
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
- rpm
- libffi-dev
- python3-dev
- python3-pip
- python3-setuptools
- wget
- python3-venv
post-cache:
- python3 -m venv env
- . /cfsetup_build/env/bin/activate
- pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9
# build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
- ./build-packages.sh
# handle FIPS separately so that we built with gofips compiler
build-linux-fips-release:
build_dir: *build_dir
builddeps: *build_deps_release
post-cache:
# same logic as above, but for FIPS packages only
- ./build-packages-fips.sh
generate-versions-file:
build_dir: *build_dir
builddeps:
- *pinned_go
- build-essential
post-cache:
- make generate-docker-version
build-deb:
build_dir: *build_dir
builddeps: &build_deb_deps
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
post-cache:
- export GOOS=linux
- export GOARCH=amd64
- make cloudflared-deb
build-fips-internal-deb:
build_dir: *build_dir
builddeps: &build_fips_deb_deps
@ -118,126 +39,14 @@ bookworm: &bookworm
- make cloudflared-deb
build-deb-arm64:
build_dir: *build_dir
builddeps: *build_deb_deps
builddeps:
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
post-cache:
- export GOOS=linux
- export GOARCH=arm64
- make cloudflared-deb
test:
build_dir: *build_dir
builddeps: &build_deps_tests
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
- rpm
- libffi-dev
- gotest-to-teamcity
pre-cache: *build_pre_cache
post-cache:
- export GOOS=linux
- export GOARCH=amd64
- export PATH="$HOME/go/bin:$PATH"
- make test | gotest-to-teamcity
test-fips:
build_dir: *build_dir
builddeps: *build_deps_tests
pre-cache: *build_pre_cache
post-cache:
- export GOOS=linux
- export GOARCH=amd64
- export FIPS=true
- export PATH="$HOME/go/bin:$PATH"
- make test | gotest-to-teamcity
component-test:
build_dir: *build_dir
builddeps: &build_deps_component_test
- *pinned_go
- python3
- python3-pip
- python3-setuptools
# procps installs the ps command which is needed in test_sysv_service
# because the init script uses ps pid to determine if the agent is
# running
- procps
- python3-venv
pre-cache-copy-paths:
- component-tests/requirements.txt
post-cache: &component_test_post_cache
- python3 -m venv env
- . env/bin/activate
- pip install --upgrade -r component-tests/requirements.txt
# Creates and routes a Named Tunnel for this build. Also constructs
# config file from env vars.
- python3 component-tests/setup.py --type create
- pytest component-tests -o log_cli=true --log-cli-level=INFO
# The Named Tunnel is deleted and its route unprovisioned here.
- python3 component-tests/setup.py --type cleanup
component-test-fips:
build_dir: *build_dir
builddeps: *build_deps_component_test
pre-cache-copy-paths:
- component-tests/requirements.txt
post-cache: *component_test_post_cache
github-release-dryrun:
build_dir: *build_dir
builddeps:
- *pinned_go
- build-essential
- python3-dev
- libffi-dev
- python3-setuptools
- python3-pip
- python3-venv
post-cache:
- python3 -m venv env
- . env/bin/activate
- pip install pynacl==1.4.0 pygithub==1.55
- make github-release-dryrun
github-release:
build_dir: *build_dir
builddeps:
- *pinned_go
- build-essential
- python3-dev
- libffi-dev
- python3-setuptools
- python3-pip
- python3-venv
post-cache:
- python3 -m venv env
- . env/bin/activate
- pip install pynacl==1.4.0 pygithub==1.55
- make github-release
r2-linux-release:
build_dir: *build_dir
builddeps: &r2-linux-release-deps
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
- rpm
- wget
- python3-dev
- libffi-dev
- python3-setuptools
- python3-pip
- reprepro
- createrepo-c
- python3-venv
post-cache:
- python3 -m venv env
- . env/bin/activate
- pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9
- make r2-linux-release
r2-next-linux-release:
build_dir: *build_dir
builddeps: *r2-linux-release-deps
post-cache:
- python3 -m venv env
- . env/bin/activate
- pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9
- make r2-next-linux-release
trixie: *bookworm

10
release_pkgs.py
View File

@ -260,7 +260,7 @@ def upload_from_directories(pkg_uploader, directory, release, binary):
"""
1. looks into a built_artifacts folder for cloudflared debs
1. looks into a artifacts folder for cloudflared debs
2. creates Packages.gz, InRelease (signed) files
3. uploads them to Cloudflare R2
@ -294,7 +294,7 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, primary_gpg_key_id
for release in releases:
for arch in archs:
print(f"creating deb pkgs for {release} and {arch}...")
pkg_creator.create_deb_pkgs(release, f"./built_artifacts/cloudflared-linux-{arch}.deb")
pkg_creator.create_deb_pkgs(release, f"./artifacts/cloudflared-linux-{arch}.deb")
print("uploading latest to r2...")
upload_from_directories(pkg_uploader, "dists", None, binary_name)
@ -381,10 +381,6 @@ def parse_args():
downloaders can use to verify signing"
)
parser.add_argument(
"--gpg-public-key-url-2", default=os.environ.get("GPG_PUBLIC_KEY_URL_2"), help="Secondary GPG public key url for rollover"
)
parser.add_argument(
"--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders"
)
@ -456,7 +452,7 @@ if __name__ == "__main__":
create_rpm_packaging(
pkg_creator,
pkg_uploader,
"./built_artifacts",
"./artifacts",
args.release_tag,
args.binary,
secondary_gpg_key_name,