From 47733ba25e9dc84b01ad2b98f97baa550e566615 Mon Sep 17 00:00:00 2001 From: lneto Date: Fri, 5 Jul 2024 19:39:22 +0100 Subject: [PATCH] TUN-8523: refactor makefile and cfsetup - remove unused targets in Makefile - order deps in cfsetup.yaml - only build cloudflared not all linux targets - rename stages to be more explicit - adjust build deps of build-linux-release - adjust build deps of build-linux-fips-release - rename github_release_pkgs_pre_cache to build_release_pre_cache - only build release release artifacts within build-linux-release - only build release release artifacts within build-linux-fips-release - remove github-release-macos - remove github-release-windows - adjust builddeps of test and test-fips - create builddeps anchor for component-test and use it in component-test-fips - remove wixl from build-linux-* - rename release-pkgs-linux to r2-linux-release - add github-release: artifacts uplooad and set release message - clean build directory before build - add step to package windows binaries - refactor windows script One of TeamCity changes is moving the artifacts to the built artifacts, hence, there is no need to cp files from artifacts to built_artifacts - create anchor for release builds - create anchor for tests stages - remove reprepro and createrepo as they are only called by release_pkgs.py --- .teamcity/mac/build.sh | 2 +- .teamcity/package-windows.sh | 5 +- Makefile | 31 ++------ cfsetup.yaml | 132 +++++++++++++++-------------------- 4 files changed, 65 insertions(+), 105 deletions(-) diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index d1eaa568..865b296e 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -188,6 +188,6 @@ for arch in ${archs[@]}; do fi done -# cleanup build the build directory because this script is not ran within containers, +# cleanup build directory because this script is not ran within containers, # which might lead to future issues in subsequent runs. rm -rf "${TARGET_DIRECTORY}" diff --git a/.teamcity/package-windows.sh b/.teamcity/package-windows.sh index 9d92a21f..0139a51b 100755 --- a/.teamcity/package-windows.sh +++ b/.teamcity/package-windows.sh @@ -9,9 +9,8 @@ windowsArchs=("amd64" "386") for arch in ${windowsArchs[@]}; do export TARGET_ARCH=$arch # Copy exe into final directory - cp ./artifacts/cloudflared-windows-$arch.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe - cp ./artifacts/cloudflared-windows-$arch.exe ./cloudflared.exe + cp $ARTIFACT_DIR/cloudflared-windows-$arch.exe ./cloudflared.exe make cloudflared-msi # Copy msi into final directory mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi -done \ No newline at end of file +done diff --git a/Makefile b/Makefile index ce7d8f40..1d044ac9 100644 --- a/Makefile +++ b/Makefile @@ -218,37 +218,14 @@ cloudflared-pkg: cloudflared cloudflared.1 cloudflared-msi: wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs -.PHONY: cloudflared-darwin-amd64.tgz -cloudflared-darwin-amd64.tgz: cloudflared - tar czf cloudflared-darwin-amd64.tgz cloudflared - rm cloudflared - .PHONY: github-release -github-release: cloudflared - python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION) - -.PHONY: github-release-built-pkgs -github-release-built-pkgs: +github-release: python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) - -.PHONY: release-pkgs-linux -release-pkgs-linux: - python3 ./release_pkgs.py - -.PHONY: github-message -github-message: python3 github_message.py --release-version $(VERSION) -.PHONY: github-mac-upload -github-mac-upload: - python3 github_release.py --path artifacts --release-version $(VERSION) - -.PHONY: github-windows-upload -github-windows-upload: - python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.exe --release-version $(VERSION) --name cloudflared-windows-amd64.exe - python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.msi --release-version $(VERSION) --name cloudflared-windows-amd64.msi - python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe - python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi +.PHONY: r2-linux-release +r2-linux-release: + python3 ./release_pkgs.py .PHONY: capnp capnp: diff --git a/cfsetup.yaml b/cfsetup.yaml index 12afa0e9..953a80a1 100644 --- a/cfsetup.yaml +++ b/cfsetup.yaml @@ -3,34 +3,27 @@ pinned_go: &pinned_go go-boring=1.22.2-1 build_dir: &build_dir /cfsetup_build default-flavor: bullseye buster: &buster - build: + build-linux: build_dir: *build_dir builddeps: &build_deps - *pinned_go - build-essential - - gotest-to-teamcity - fakeroot - rubygem-fpm - rpm - libffi-dev - - reprepro - - createrepo pre-cache: &build_pre_cache - export GOCACHE=/cfsetup_build/.cache/go-build - go install golang.org/x/tools/cmd/goimports@latest post-cache: - # TODO: TUN-8126 this is temporary to make sure packages can be built before release - - ./build-packages.sh # Build binary for component test - GOOS=linux GOARCH=amd64 make cloudflared - build-fips: + build-linux-fips: build_dir: *build_dir builddeps: *build_deps pre-cache: *build_pre_cache post-cache: - export FIPS=true - # TODO: TUN-8126 this is temporary to make sure packages can be built before release - - ./build-packages-fips.sh # Build binary for component test - GOOS=linux GOARCH=amd64 make cloudflared cover: @@ -39,28 +32,21 @@ buster: &buster pre-cache: *build_pre_cache post-cache: - make cover - # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64) - github-release-pkgs: + # except FIPS and macos + build-linux-release: build_dir: *build_dir - builddeps: + builddeps: &build_deps_release - *pinned_go - build-essential - fakeroot - rubygem-fpm - rpm - - wget - # libmsi and libgcab are libraries the wixl binary depends on. - - libmsi-dev - - libgcab-dev - - python3-dev - libffi-dev - - python3-setuptools + - python3-dev - python3-pip - - reprepro - - createrepo - pre-cache: &github_release_pkgs_pre_cache - - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin - - chmod a+x /usr/local/bin/wixl + - python3-setuptools + - wget + pre-cache: &build_release_pre_cache - pip3 install pynacl==1.4.0 - pip3 install pygithub==1.55 - pip3 install boto3==1.22.9 @@ -68,32 +54,14 @@ buster: &buster post-cache: # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts - ./build-packages.sh - # release the packages built and moved to /cfsetup/built_artifacts - - make github-release-built-pkgs - # publish packages to linux repos - - make release-pkgs-linux # handle FIPS separately so that we built with gofips compiler - github-fips-release-pkgs: + build-linux-fips-release: build_dir: *build_dir - builddeps: - - *pinned_go - - build-essential - - fakeroot - - rubygem-fpm - - rpm - - wget - # libmsi and libgcab are libraries the wixl binary depends on. - - libmsi-dev - - libgcab-dev - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - pre-cache: *github_release_pkgs_pre_cache + builddeps: *build_deps_release + pre-cache: *build_release_pre_cache post-cache: # same logic as above, but for FIPS packages only - ./build-packages-fips.sh - - make github-release-built-pkgs generate-versions-file: build_dir: *build_dir builddeps: @@ -152,21 +120,7 @@ buster: &buster - export GOOS=linux - export GOARCH=arm64 - make cloudflared-deb - github-release-macos-amd64: - build_dir: *build_dir - builddeps: &build_pygithub - - *pinned_go - - build-essential - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - pre-cache: &install_pygithub - - pip3 install pynacl==1.4.0 - - pip3 install pygithub==1.55 - post-cache: - - make github-mac-upload - github-release-windows: + package-windows: build_dir: *build_dir builddeps: - *pinned_go @@ -186,10 +140,16 @@ buster: &buster - pip3 install pygithub==1.55 post-cache: - .teamcity/package-windows.sh - - make github-windows-upload test: build_dir: *build_dir - builddeps: *build_deps + builddeps: &build_deps_tests + - *pinned_go + - build-essential + - fakeroot + - rubygem-fpm + - rpm + - libffi-dev + - gotest-to-teamcity pre-cache: *build_pre_cache post-cache: - export GOOS=linux @@ -199,7 +159,7 @@ buster: &buster - make test | gotest-to-teamcity test-fips: build_dir: *build_dir - builddeps: *build_deps + builddeps: *build_deps_tests pre-cache: *build_pre_cache post-cache: - export GOOS=linux @@ -210,7 +170,7 @@ buster: &buster - make test | gotest-to-teamcity component-test: build_dir: *build_dir - builddeps: + builddeps: &build_deps_component_test - *pinned_go - python3.7 - python3-pip @@ -230,24 +190,48 @@ buster: &buster - python3 component-tests/setup.py --type cleanup component-test-fips: build_dir: *build_dir - builddeps: - - *pinned_go - - python3.7 - - python3-pip - - python3-setuptools - # procps installs the ps command which is needed in test_sysv_service because the init script - # uses ps pid to determine if the agent is running - - procps + builddeps: *build_deps_component_test pre-cache-copy-paths: - component-tests/requirements.txt pre-cache: *component_test_pre_cache post-cache: *component_test_post_cache - github-message-release: + github-release: build_dir: *build_dir - builddeps: *build_pygithub - pre-cache: *install_pygithub + builddeps: + - *pinned_go + - build-essential + - python3-dev + - libffi-dev + - python3-setuptools + - python3-pip + pre-cache: + - pip3 install pynacl==1.4.0 + - pip3 install pygithub==1.55 post-cache: + - make github-release - make github-message + r2-linux-release: + build_dir: *build_dir + builddeps: + - *pinned_go + - build-essential + - fakeroot + - rubygem-fpm + - rpm + - wget + - python3-dev + - libffi-dev + - python3-setuptools + - python3-pip + - reprepro + - createrepo + pre-cache: + - pip3 install pynacl==1.4.0 + - pip3 install pygithub==1.55 + - pip3 install boto3==1.22.9 + - pip3 install python-gnupg==0.4.9 + post-cache: + - make r2-linux-release bullseye: *buster bookworm: *buster