TUN-8523: refactor makefile and cfsetup

- remove unused targets in Makefile
- order deps in cfsetup.yaml
- only build cloudflared not all linux targets
- rename stages to be more explicit
- adjust build deps of build-linux-release
- adjust build deps of build-linux-fips-release
- rename github_release_pkgs_pre_cache to build_release_pre_cache
- only build release release artifacts within build-linux-release
- only build release release artifacts within build-linux-fips-release
- remove github-release-macos
- remove github-release-windows
- adjust builddeps of test and test-fips
- create builddeps anchor for component-test and use it in component-test-fips
- remove wixl from build-linux-*
- rename release-pkgs-linux to r2-linux-release
- add github-release: artifacts uplooad and set release message
- clean build directory before build
- add step to package windows binaries
- refactor windows script
One of TeamCity changes is moving the artifacts to the built artifacts, hence, there is no need to cp files from artifacts to built_artifacts
- create anchor for release builds
- create anchor for tests stages
- remove reprepro and createrepo as they are only called by release_pkgs.py

.teamcity/mac/build.sh

@@ -188,6 +188,6 @@ for arch in ${archs[@]}; do
   fi
 done
 
-# cleanup build the build directory because this script is not ran within containers,
+# cleanup build directory because this script is not ran within containers,
 # which might lead to future issues in subsequent runs.
 rm -rf "${TARGET_DIRECTORY}"

.teamcity/package-windows.sh

@@ -9,9 +9,8 @@ windowsArchs=("amd64" "386")
 for arch in ${windowsArchs[@]}; do
     export TARGET_ARCH=$arch
     # Copy exe into final directory
-    cp ./artifacts/cloudflared-windows-$arch.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
+    cp $ARTIFACT_DIR/cloudflared-windows-$arch.exe ./cloudflared.exe
-    cp ./artifacts/cloudflared-windows-$arch.exe ./cloudflared.exe
     make cloudflared-msi
     # Copy msi into final directory
     mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi
-done
+done

Makefile

@@ -218,37 +218,14 @@ cloudflared-pkg: cloudflared cloudflared.1
 cloudflared-msi:
 	wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs
 
-.PHONY: cloudflared-darwin-amd64.tgz
-cloudflared-darwin-amd64.tgz: cloudflared
-	tar czf cloudflared-darwin-amd64.tgz cloudflared
-	rm cloudflared
-
 .PHONY: github-release
-github-release: cloudflared
+github-release:
-	python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION)
-
-.PHONY: github-release-built-pkgs
-github-release-built-pkgs:
 	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
-
-.PHONY: release-pkgs-linux
-release-pkgs-linux:
-	python3 ./release_pkgs.py
-
-.PHONY: github-message
-github-message:
 	python3 github_message.py --release-version $(VERSION)
 
-.PHONY: github-mac-upload
+.PHONY: r2-linux-release
-github-mac-upload:
+r2-linux-release:
-	python3 github_release.py --path artifacts --release-version $(VERSION) 
+	python3 ./release_pkgs.py
-
-.PHONY: github-windows-upload
-github-windows-upload:
-	python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.exe --release-version $(VERSION) --name cloudflared-windows-amd64.exe
-	python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.msi --release-version $(VERSION) --name cloudflared-windows-amd64.msi
-	python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe
-	python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi
 
 .PHONY: capnp
 capnp:

cfsetup.yaml

@@ -3,34 +3,27 @@ pinned_go: &pinned_go go-boring=1.22.2-1
 build_dir: &build_dir /cfsetup_build
 default-flavor: bullseye
 buster: &buster
-  build:
+  build-linux:
     build_dir: *build_dir
     builddeps: &build_deps
       - *pinned_go
       - build-essential
-      - gotest-to-teamcity
       - fakeroot
       - rubygem-fpm
       - rpm
       - libffi-dev
-      - reprepro
-      - createrepo
     pre-cache: &build_pre_cache
       - export GOCACHE=/cfsetup_build/.cache/go-build
       - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
-      # TODO: TUN-8126 this is temporary to make sure packages can be built before release
-      - ./build-packages.sh
       # Build binary for component test
       - GOOS=linux GOARCH=amd64 make cloudflared
-  build-fips:
+  build-linux-fips:
     build_dir: *build_dir
     builddeps: *build_deps
     pre-cache: *build_pre_cache
     post-cache:
       - export FIPS=true
-      # TODO: TUN-8126 this is temporary to make sure packages can be built before release
-      - ./build-packages-fips.sh
       # Build binary for component test
       - GOOS=linux GOARCH=amd64 make cloudflared
   cover:
@@ -39,28 +32,21 @@ buster: &buster
     pre-cache: *build_pre_cache
     post-cache:
      - make cover
-  # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64)
+  # except FIPS and macos 
-  github-release-pkgs:
+  build-linux-release:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps_release
       - *pinned_go
       - build-essential
       - fakeroot
       - rubygem-fpm
       - rpm
-      - wget
-      # libmsi and libgcab are libraries the wixl binary depends on.
-      - libmsi-dev
-      - libgcab-dev
-      - python3-dev
       - libffi-dev
-      - python3-setuptools
+      - python3-dev
       - python3-pip
-      - reprepro
+      - python3-setuptools
-      - createrepo
+      - wget
-    pre-cache: &github_release_pkgs_pre_cache
+    pre-cache: &build_release_pre_cache
-      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
-      - chmod a+x /usr/local/bin/wixl
       - pip3 install pynacl==1.4.0
       - pip3 install pygithub==1.55
       - pip3 install boto3==1.22.9
@@ -68,32 +54,14 @@ buster: &buster
     post-cache:
       # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
       - ./build-packages.sh
-      # release the packages built and moved to /cfsetup/built_artifacts
-      - make github-release-built-pkgs
-      # publish packages to linux repos
-      - make release-pkgs-linux
   # handle FIPS separately so that we built with gofips compiler
-  github-fips-release-pkgs:
+  build-linux-fips-release:
     build_dir: *build_dir
-    builddeps:
+    builddeps: *build_deps_release
-      - *pinned_go
+    pre-cache: *build_release_pre_cache
-      - build-essential
-      - fakeroot
-      - rubygem-fpm
-      - rpm
-      - wget
-      # libmsi and libgcab are libraries the wixl binary depends on.
-      - libmsi-dev
-      - libgcab-dev
-      - python3-dev
-      - libffi-dev
-      - python3-setuptools
-      - python3-pip
-    pre-cache: *github_release_pkgs_pre_cache
     post-cache:
       # same logic as above, but for FIPS packages only
       - ./build-packages-fips.sh
-      - make github-release-built-pkgs
   generate-versions-file:
     build_dir: *build_dir
     builddeps:
@@ -152,21 +120,7 @@ buster: &buster
       - export GOOS=linux
       - export GOARCH=arm64
       - make cloudflared-deb
-  github-release-macos-amd64:
+  package-windows:
-    build_dir: *build_dir
-    builddeps: &build_pygithub
-      - *pinned_go
-      - build-essential
-      - python3-dev
-      - libffi-dev
-      - python3-setuptools
-      - python3-pip
-    pre-cache: &install_pygithub
-      - pip3 install pynacl==1.4.0
-      - pip3 install pygithub==1.55
-    post-cache:
-      - make github-mac-upload
-  github-release-windows:
     build_dir: *build_dir
     builddeps:
       - *pinned_go
@@ -186,10 +140,16 @@ buster: &buster
       - pip3 install pygithub==1.55
     post-cache:
       - .teamcity/package-windows.sh
-      - make github-windows-upload
   test:
     build_dir: *build_dir
-    builddeps: *build_deps
+    builddeps: &build_deps_tests
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - libffi-dev
+      - gotest-to-teamcity
     pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
@@ -199,7 +159,7 @@ buster: &buster
       - make test | gotest-to-teamcity
   test-fips:
     build_dir: *build_dir
-    builddeps: *build_deps
+    builddeps: *build_deps_tests
     pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
@@ -210,7 +170,7 @@ buster: &buster
       - make test | gotest-to-teamcity
   component-test:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps_component_test
       - *pinned_go
       - python3.7
       - python3-pip
@@ -230,24 +190,48 @@ buster: &buster
       - python3 component-tests/setup.py --type cleanup
   component-test-fips:
     build_dir: *build_dir
-    builddeps:
+    builddeps: *build_deps_component_test
-      - *pinned_go
-      - python3.7
-      - python3-pip
-      - python3-setuptools
-      # procps installs the ps command which is needed in test_sysv_service because the init script
-      # uses ps pid to determine if the agent is running
-      - procps
     pre-cache-copy-paths:
       - component-tests/requirements.txt
     pre-cache: *component_test_pre_cache
     post-cache: *component_test_post_cache
-  github-message-release:
+  github-release:
     build_dir: *build_dir
-    builddeps: *build_pygithub
+    builddeps: 
-    pre-cache: *install_pygithub
+      - *pinned_go
+      - build-essential
+      - python3-dev
+      - libffi-dev
+      - python3-setuptools
+      - python3-pip
+    pre-cache:
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
     post-cache:
+      - make github-release
       - make github-message
+  r2-linux-release:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - wget
+      - python3-dev
+      - libffi-dev
+      - python3-setuptools
+      - python3-pip
+      - reprepro
+      - createrepo
+    pre-cache:
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
+      - pip3 install boto3==1.22.9
+      - pip3 install python-gnupg==0.4.9
+    post-cache:
+      - make r2-linux-release
 
 bullseye: *buster
 bookworm: *buster