From a8fdbb83d03c182efe7042241bbe99974a257a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20=22Pisco=22=20Fernandes?= Date: Fri, 31 Oct 2025 14:43:50 +0000 Subject: [PATCH] TUN-9800: Add pipelines for linux packaging --- .ci/commons.gitlab-ci.yml | 6 +- .ci/image/Dockerfile | 8 +- .ci/linux.gitlab-ci.yml | 31 +++ .ci/release.gitlab-ci.yml | 120 +++++++++-- .../scripts/linux/build-packages-fips.sh | 0 .ci/scripts/linux/build-packages.sh | 59 +++++ .ci/scripts/release-target.sh | 18 ++ Makefile | 10 +- build-packages.sh | 48 ----- cfsetup.yaml | 201 +----------------- release_pkgs.py | 10 +- 11 files changed, 229 insertions(+), 282 deletions(-) rename build-packages-fips.sh => .ci/scripts/linux/build-packages-fips.sh (100%) create mode 100755 .ci/scripts/linux/build-packages.sh create mode 100755 .ci/scripts/release-target.sh delete mode 100755 build-packages.sh diff --git a/.ci/commons.gitlab-ci.yml b/.ci/commons.gitlab-ci.yml index 990c5709..43b43f22 100644 --- a/.ci/commons.gitlab-ci.yml +++ b/.ci/commons.gitlab-ci.yml @@ -3,14 +3,14 @@ # Rules to run the job only on the master branch run-on-master: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: always + when: on_success - when: never # Rules to run the job only on merge requests run-on-mr: - if: $CI_COMMIT_TAG when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: always + when: on_success - when: never # Rules to run the job on merge_requests and master branch run-always: @@ -18,7 +18,7 @@ when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_COMMIT_BRANCH != null && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: always + when: on_success - when: never # This before_script is injected into every job that runs on master meaning that if there is no tag the step diff --git a/.ci/image/Dockerfile b/.ci/image/Dockerfile index d2766e09..9d700fff 100644 --- a/.ci/image/Dockerfile +++ b/.ci/image/Dockerfile @@ -16,7 +16,13 @@ RUN apt-get update && \ python3-venv \ # libmsi and libgcab are libraries the wixl binary depends on. libmsi-dev \ - libgcab-dev && \ + libgcab-dev \ + # deb and rpm build tools + rubygem-fpm \ + rpm \ + # create deb and rpm repository files + reprepro \ + createrepo-c && \ rm -rf /var/lib/apt/lists/* && \ # Install wixl curl -o /usr/local/bin/wixl -L https://pkg.cloudflare.com/binaries/wixl && \ diff --git a/.ci/linux.gitlab-ci.yml b/.ci/linux.gitlab-ci.yml index 96181af7..a751a24e 100644 --- a/.ci/linux.gitlab-ci.yml +++ b/.ci/linux.gitlab-ci.yml @@ -8,6 +8,18 @@ imageVersion: "3371-f5539bd6f83d@sha256:a2a68f580070f9411d0d3155959ed63b700ef319b5fcc62db340e92227bbc628" CGO_ENABLED: 1 +.default-packaging-job: &packaging-job-defaults + stage: build + needs: + - ci-image-get-image-ref + rules: + - !reference [.default-rules, run-on-master] + image: $BUILD_IMAGE + cache: {} + artifacts: + paths: + - artifacts/* + include: ################### ### Linux Build ### @@ -89,3 +101,22 @@ component-tests-linux-fips: variables: <<: *component-tests-variables COMPONENT_TESTS_FIPS: 1 + +################################ +####### Linux Packaging ######## +################################ +linux-packaging: + <<: *packaging-job-defaults + parallel: + matrix: + - ARCH: ["386", "amd64", "arm", "armhf", "arm64"] + script: + - ./.ci/scripts/linux/build-packages.sh ${ARCH} + +################################ +##### Linux FIPS Packaging ##### +################################ +linux-packaging-fips: + <<: *packaging-job-defaults + script: + - ./.ci/scripts/linux/build-packages-fips.sh diff --git a/.ci/release.gitlab-ci.yml b/.ci/release.gitlab-ci.yml index a41247cb..5c7c53b2 100644 --- a/.ci/release.gitlab-ci.yml +++ b/.ci/release.gitlab-ci.yml @@ -1,17 +1,28 @@ include: - local: .ci/commons.gitlab-ci.yml -########################################### -### Push Cloudflared Binaries to Github ### -########################################### -release-cloudflared-to-github: + ###################################### + ### Build and Push DockerHub Image ### + ###################################### + - component: $CI_SERVER_FQDN/cloudflare/ci/docker-image/build-push-image@~latest + inputs: + stage: release + jobPrefix: docker-hub + runOnMR: false + runOnBranches: '^master$' + runOnChangesTo: ['RELEASE_NOTES'] + needs: + - generate-version-file + - release-cloudflared-to-r2 + commentImageRefs: false + runner: vm-linux-x86-4cpu-8gb + DOCKER_USER_BRANCH: svcgithubdockerhubcloudflar045 + DOCKER_PASSWORD_BRANCH: gitlab/cloudflare/tun/cloudflared/_dev/dockerhub/svc_password/data + EXTRA_DIB_ARGS: --overwrite + +.default-release-job: &release-job-defaults stage: release image: $BUILD_IMAGE - extends: .check-tag - needs: - - ci-image-get-image-ref - - package-windows - - build-and-sign-cloudflared-macos rules: - !reference [.default-rules, run-on-master] cache: @@ -19,8 +30,16 @@ release-cloudflared-to-github: - .cache/pip variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + # KV Vars KV_NAMESPACE: 380e19aa04314648949b6ad841417ebe - KV_ACCOUNT: 5ab4e9dfbd435d24068829fda0077963 + KV_ACCOUNT: &cf-account 5ab4e9dfbd435d24068829fda0077963 + # R2 Vars + R2_BUCKET: cloudflared-pkgs + R2_ACCOUNT_ID: *cf-account + # APT and RPM Repository Vars + GPG_PUBLIC_KEY_URL: "https://pkg.cloudflare.com/cloudflare-ascii-pubkey.gpg" + PKG_URL: "https://pkg.cloudflare.com/cloudflared" + BINARY_NAME: cloudflared secrets: KV_API_TOKEN: vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_kv_api_token/data@kv @@ -28,12 +47,77 @@ release-cloudflared-to-github: API_KEY: vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_github_api_key/data@kv file: false + R2_CLIENT_ID: + vault: gitlab/cloudflare/tun/cloudflared/_dev/_terraform_atlantis/r2_api_token/client_id@kv + file: false + R2_CLIENT_SECRET: + vault: gitlab/cloudflare/tun/cloudflared/_dev/_terraform_atlantis/r2_api_token/client_secret@kv + file: false + LINUX_SIGNING_PUBLIC_KEY: + vault: gitlab/cloudflare/tun/cloudflared/_dev/gpg_v1/public_key@kv + file: false + LINUX_SIGNING_PRIVATE_KEY: + vault: gitlab/cloudflare/tun/cloudflared/_dev/gpg_v1/private_key@kv + file: false + LINUX_SIGNING_PUBLIC_KEY_2: + vault: gitlab/cloudflare/tun/cloudflared/_dev/gpg_v2/public_key@kv + file: false + LINUX_SIGNING_PRIVATE_KEY_2: + vault: gitlab/cloudflare/tun/cloudflared/_dev/gpg_v2/private_key@kv + file: false + +########################################### +### Push Cloudflared Binaries to Github ### +########################################### +release-cloudflared-to-github: + <<: *release-job-defaults + extends: .check-tag + needs: + - build-and-sign-cloudflared-macos + - ci-image-get-image-ref + - linux-packaging + - linux-packaging-fips + - package-windows script: - - python3 --version ; pip --version # For debugging - - python3 -m venv venv - - source venv/bin/activate - - pip install pynacl==1.4.0 pygithub==1.55 - - echo $VERSION - - echo $TAG_EXISTS - - echo "Running release because tag exists." - - make gitlab-release + - ./.ci/scripts/release-target.sh github-release + +######################################### +### Upload Cloudflared Binaries to R2 ### +######################################### +release-cloudflared-to-r2: + <<: *release-job-defaults + extends: .check-tag + needs: + - ci-image-get-image-ref + - linux-packaging # We only release non-FIPS binaries to R2 + - release-cloudflared-to-github + script: + - ./.ci/scripts/release-target.sh r2-linux-release + +################################################# +### Upload Cloudflared Nightly Binaries to R2 ### +################################################# +release-cloudflared-nightly-to-r2: + <<: *release-job-defaults + variables: + R2_BUCKET: cloudflared-pkgs-next + GPG_PUBLIC_KEY_URL: "https://next.pkg.cloudflare.com/cloudflare-ascii-pubkey.gpg" + PKG_URL: "https://next.pkg.cloudflare.com/cloudflared" + needs: + - ci-image-get-image-ref + - linux-packaging # We only release non-FIPS binaries to R2 + script: + - ./.ci/scripts/release-target.sh r2-linux-release + +############################# +### Generate Version File ### +############################# +generate-version-file: + <<: *release-job-defaults + needs: + - ci-image-get-image-ref + script: + - make generate-docker-version + artifacts: + paths: + - versions diff --git a/build-packages-fips.sh b/.ci/scripts/linux/build-packages-fips.sh similarity index 100% rename from build-packages-fips.sh rename to .ci/scripts/linux/build-packages-fips.sh diff --git a/.ci/scripts/linux/build-packages.sh b/.ci/scripts/linux/build-packages.sh new file mode 100755 index 00000000..a6ca2037 --- /dev/null +++ b/.ci/scripts/linux/build-packages.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Check if architecture argument is provided +if [ $# -eq 0 ]; then + echo "Error: Architecture argument is required" + echo "Usage: $0 " + exit 1 +fi + +# Parameters +arch=$1 + +# Get Version +VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*") +echo $VERSION + +# Disable FIPS module in go-boring +export GOEXPERIMENT=noboringcrypto +export CGO_ENABLED=0 + +# This controls the directory the built artifacts go into +export ARTIFACT_DIR=artifacts/ +mkdir -p $ARTIFACT_DIR + +export TARGET_OS=linux + +unset TARGET_ARM +export TARGET_ARCH=$arch + +## Support for arm platforms without hardware FPU enabled +if [[ $arch == arm ]] ; then + export TARGET_ARCH=arm + export TARGET_ARM=5 +fi + +## Support for armhf builds +if [[ $arch == armhf ]] ; then + export TARGET_ARCH=arm + export TARGET_ARM=7 +fi + +make cloudflared-deb +mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb + +# rpm packages invert the - and _ and use x86_64 instead of amd64. +RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g') +RPMARCH=$arch +if [ $arch == "amd64" ];then + RPMARCH="x86_64" +fi +if [ $arch == "arm64" ]; then + RPMARCH="aarch64" +fi +make cloudflared-rpm +mv cloudflared-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-linux-$RPMARCH.rpm + +# finally move the linux binary as well. +mv ./cloudflared $ARTIFACT_DIR/cloudflared-linux-$arch + diff --git a/.ci/scripts/release-target.sh b/.ci/scripts/release-target.sh new file mode 100755 index 00000000..8c998310 --- /dev/null +++ b/.ci/scripts/release-target.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e -o pipefail + +# Check if a make target is provided as an argument +if [ $# -eq 0 ]; then + echo "Error: Make target argument is required" + echo "Usage: $0 " + exit 1 +fi + +MAKE_TARGET=$1 + +python3 -m venv venv +source venv/bin/activate + +# Our release scripts are written in python, so we should install their dependecies here. +pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9 +make $MAKE_TARGET diff --git a/Makefile b/Makefile index 7eaccec7..8490480e 100644 --- a/Makefile +++ b/Makefile @@ -221,10 +221,6 @@ cloudflared-deb: cloudflared cloudflared.1 cloudflared-rpm: cloudflared cloudflared.1 $(call build_package,rpm) -.PHONY: cloudflared-pkg -cloudflared-pkg: cloudflared cloudflared.1 - $(call build_package,osxpkg) - .PHONY: cloudflared-msi cloudflared-msi: wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs @@ -235,12 +231,8 @@ github-release-dryrun: .PHONY: github-release github-release: - python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) - python3 github_message.py --release-version $(VERSION) - -.PHONY: gitlab-release -gitlab-release: python3 github_release.py --path $(PWD)/artifacts/ --release-version $(VERSION) + python3 github_message.py --release-version $(VERSION) .PHONY: r2-linux-release r2-linux-release: diff --git a/build-packages.sh b/build-packages.sh deleted file mode 100755 index df5dc7bb..00000000 --- a/build-packages.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*") -echo $VERSION - -# Disable FIPS module in go-boring -export GOEXPERIMENT=noboringcrypto -export CGO_ENABLED=0 - -# This controls the directory the built artifacts go into -export ARTIFACT_DIR=artifacts/ -mkdir -p $ARTIFACT_DIR - -linuxArchs=("386" "amd64" "arm" "armhf" "arm64") -export TARGET_OS=linux -for arch in ${linuxArchs[@]}; do - unset TARGET_ARM - export TARGET_ARCH=$arch - - ## Support for arm platforms without hardware FPU enabled - if [[ $arch == arm ]] ; then - export TARGET_ARCH=arm - export TARGET_ARM=5 - fi - - ## Support for armhf builds - if [[ $arch == armhf ]] ; then - export TARGET_ARCH=arm - export TARGET_ARM=7 - fi - - make cloudflared-deb - mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb - - # rpm packages invert the - and _ and use x86_64 instead of amd64. - RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g') - RPMARCH=$arch - if [ $arch == "amd64" ];then - RPMARCH="x86_64" - fi - if [ $arch == "arm64" ]; then - RPMARCH="aarch64" - fi - make cloudflared-rpm - mv cloudflared-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-linux-$RPMARCH.rpm - - # finally move the linux binary as well. - mv ./cloudflared $ARTIFACT_DIR/cloudflared-linux-$arch -done diff --git a/cfsetup.yaml b/cfsetup.yaml index b007312c..a9be5d11 100644 --- a/cfsetup.yaml +++ b/cfsetup.yaml @@ -4,85 +4,6 @@ build_dir: &build_dir /cfsetup_build default-flavor: bookworm bookworm: &bookworm - build-linux: - build_dir: *build_dir - builddeps: &build_deps - - *pinned_go - - build-essential - - fakeroot - - rubygem-fpm - - rpm - - libffi-dev - - golangci-lint=1.64.8-2 - pre-cache: &build_pre_cache - - export GOCACHE=/cfsetup_build/.cache/go-build - - go install golang.org/x/tools/cmd/goimports@v0.30.0 - post-cache: - # Linting - - make lint - - make fmt-check - # Build binary for component test - - GOOS=linux GOARCH=amd64 make cloudflared - build-linux-fips: - build_dir: *build_dir - builddeps: *build_deps - pre-cache: *build_pre_cache - post-cache: - - export FIPS=true - # Build binary for component test - - GOOS=linux GOARCH=amd64 make cloudflared - cover: - build_dir: *build_dir - builddeps: *build_deps - pre-cache: *build_pre_cache - post-cache: - - make cover - # except FIPS and macos - build-linux-release: - build_dir: *build_dir - builddeps: &build_deps_release - - *pinned_go - - build-essential - - fakeroot - - rubygem-fpm - - rpm - - libffi-dev - - python3-dev - - python3-pip - - python3-setuptools - - wget - - python3-venv - post-cache: - - python3 -m venv env - - . /cfsetup_build/env/bin/activate - - pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9 - # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts - - ./build-packages.sh - # handle FIPS separately so that we built with gofips compiler - build-linux-fips-release: - build_dir: *build_dir - builddeps: *build_deps_release - post-cache: - # same logic as above, but for FIPS packages only - - ./build-packages-fips.sh - generate-versions-file: - build_dir: *build_dir - builddeps: - - *pinned_go - - build-essential - post-cache: - - make generate-docker-version - build-deb: - build_dir: *build_dir - builddeps: &build_deb_deps - - *pinned_go - - build-essential - - fakeroot - - rubygem-fpm - post-cache: - - export GOOS=linux - - export GOARCH=amd64 - - make cloudflared-deb build-fips-internal-deb: build_dir: *build_dir builddeps: &build_fips_deb_deps @@ -118,126 +39,14 @@ bookworm: &bookworm - make cloudflared-deb build-deb-arm64: build_dir: *build_dir - builddeps: *build_deb_deps + builddeps: + - *pinned_go + - build-essential + - fakeroot + - rubygem-fpm post-cache: - export GOOS=linux - export GOARCH=arm64 - make cloudflared-deb - test: - build_dir: *build_dir - builddeps: &build_deps_tests - - *pinned_go - - build-essential - - fakeroot - - rubygem-fpm - - rpm - - libffi-dev - - gotest-to-teamcity - pre-cache: *build_pre_cache - post-cache: - - export GOOS=linux - - export GOARCH=amd64 - - export PATH="$HOME/go/bin:$PATH" - - make test | gotest-to-teamcity - test-fips: - build_dir: *build_dir - builddeps: *build_deps_tests - pre-cache: *build_pre_cache - post-cache: - - export GOOS=linux - - export GOARCH=amd64 - - export FIPS=true - - export PATH="$HOME/go/bin:$PATH" - - make test | gotest-to-teamcity - component-test: - build_dir: *build_dir - builddeps: &build_deps_component_test - - *pinned_go - - python3 - - python3-pip - - python3-setuptools - # procps installs the ps command which is needed in test_sysv_service - # because the init script uses ps pid to determine if the agent is - # running - - procps - - python3-venv - pre-cache-copy-paths: - - component-tests/requirements.txt - post-cache: &component_test_post_cache - - python3 -m venv env - - . env/bin/activate - - pip install --upgrade -r component-tests/requirements.txt - # Creates and routes a Named Tunnel for this build. Also constructs - # config file from env vars. - - python3 component-tests/setup.py --type create - - pytest component-tests -o log_cli=true --log-cli-level=INFO - # The Named Tunnel is deleted and its route unprovisioned here. - - python3 component-tests/setup.py --type cleanup - component-test-fips: - build_dir: *build_dir - builddeps: *build_deps_component_test - pre-cache-copy-paths: - - component-tests/requirements.txt - post-cache: *component_test_post_cache - github-release-dryrun: - build_dir: *build_dir - builddeps: - - *pinned_go - - build-essential - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - - python3-venv - post-cache: - - python3 -m venv env - - . env/bin/activate - - pip install pynacl==1.4.0 pygithub==1.55 - - make github-release-dryrun - github-release: - build_dir: *build_dir - builddeps: - - *pinned_go - - build-essential - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - - python3-venv - post-cache: - - python3 -m venv env - - . env/bin/activate - - pip install pynacl==1.4.0 pygithub==1.55 - - make github-release - r2-linux-release: - build_dir: *build_dir - builddeps: &r2-linux-release-deps - - *pinned_go - - build-essential - - fakeroot - - rubygem-fpm - - rpm - - wget - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - - reprepro - - createrepo-c - - python3-venv - post-cache: - - python3 -m venv env - - . env/bin/activate - - pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9 - - make r2-linux-release - - r2-next-linux-release: - build_dir: *build_dir - builddeps: *r2-linux-release-deps - post-cache: - - python3 -m venv env - - . env/bin/activate - - pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9 - - make r2-next-linux-release trixie: *bookworm diff --git a/release_pkgs.py b/release_pkgs.py index 173e842b..df16e092 100644 --- a/release_pkgs.py +++ b/release_pkgs.py @@ -260,7 +260,7 @@ def upload_from_directories(pkg_uploader, directory, release, binary): """ - 1. looks into a built_artifacts folder for cloudflared debs + 1. looks into a artifacts folder for cloudflared debs 2. creates Packages.gz, InRelease (signed) files 3. uploads them to Cloudflare R2 @@ -294,7 +294,7 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, primary_gpg_key_id for release in releases: for arch in archs: print(f"creating deb pkgs for {release} and {arch}...") - pkg_creator.create_deb_pkgs(release, f"./built_artifacts/cloudflared-linux-{arch}.deb") + pkg_creator.create_deb_pkgs(release, f"./artifacts/cloudflared-linux-{arch}.deb") print("uploading latest to r2...") upload_from_directories(pkg_uploader, "dists", None, binary_name) @@ -381,10 +381,6 @@ def parse_args(): downloaders can use to verify signing" ) - parser.add_argument( - "--gpg-public-key-url-2", default=os.environ.get("GPG_PUBLIC_KEY_URL_2"), help="Secondary GPG public key url for rollover" - ) - parser.add_argument( "--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders" ) @@ -456,7 +452,7 @@ if __name__ == "__main__": create_rpm_packaging( pkg_creator, pkg_uploader, - "./built_artifacts", + "./artifacts", args.release_tag, args.binary, secondary_gpg_key_name,