From 1f38deca1e947c1c405e1a1d86176bd9be91f536 Mon Sep 17 00:00:00 2001 From: GoncaloGarcia Date: Tue, 2 Jul 2024 12:45:17 +0100 Subject: [PATCH 1/9] TUN-8504: Use pre-installed python version instead of downloading it on Windows builds Recently python.org started blocking our requests. We've asked the Devtools team to upgrade the default python installation to 3.10 so that we can use it in our tests --- .teamcity/windows/component-test.ps1 | 49 ++++------------------------ 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/.teamcity/windows/component-test.ps1 b/.teamcity/windows/component-test.ps1 index e4d8727d..fe70738e 100644 --- a/.teamcity/windows/component-test.ps1 +++ b/.teamcity/windows/component-test.ps1 @@ -5,41 +5,6 @@ $ProgressPreference = "SilentlyContinue" $WorkingDirectory = Get-Location $CloudflaredDirectory = "$WorkingDirectory\go\src\github.com\cloudflare\cloudflared" -Write-Output "Installing python..." - -$PythonVersion = "3.10.11" -$PythonZipFile = "$env:Temp\python-$PythonVersion-embed-amd64.zip" -$PipInstallFile = "$env:Temp\get-pip.py" -$PythonZipUrl = "https://www.python.org/ftp/python/$PythonVersion/python-$PythonVersion-embed-amd64.zip" -$PythonPath = "$WorkingDirectory\Python" -$PythonBinPath = "$PythonPath\python.exe" - -# Download Python zip file -Invoke-WebRequest -Uri $PythonZipUrl -OutFile $PythonZipFile - -# Download Python pip file -Invoke-WebRequest -Uri "https://bootstrap.pypa.io/get-pip.py" -OutFile $PipInstallFile - -# Extract Python files -Expand-Archive $PythonZipFile -DestinationPath $PythonPath -Force - -# Add Python to PATH -$env:Path = "$PythonPath\Scripts;$PythonPath;$($env:Path)" - -Write-Output "Installed to $PythonPath" - -# Install pip -& $PythonBinPath $PipInstallFile - -# Add package paths in pythonXX._pth to unblock python -m pip -$PythonImportPathFile = "$PythonPath\python310._pth" -$ComponentTestsDir = "$CloudflaredDirectory\component-tests\" -@($ComponentTestsDir, "Lib\site-packages", $(Get-Content $PythonImportPathFile)) | Set-Content $PythonImportPathFile - -# Test Python installation -& $PythonBinPath --version -& $PythonBinPath -m pip --version - go env go version @@ -48,8 +13,8 @@ $env:CGO_ENABLED = 1 $env:TARGET_ARCH = "amd64" $env:Path = "$Env:Temp\go\bin;$($env:Path)" -& $PythonBinPath --version -& $PythonBinPath -m pip --version +python --version +python -m pip --version cd $CloudflaredDirectory @@ -72,11 +37,11 @@ if ($LASTEXITCODE -ne 0) { throw "Failed unit tests" } Write-Output "Running component tests" -& $PythonBinPath -m pip install --upgrade -r component-tests/requirements.txt -& $PythonBinPath component-tests/setup.py --type create -& $PythonBinPath -m pytest component-tests -o log_cli=true --log-cli-level=INFO +python -m pip --disable-pip-version-check install --upgrade -r component-tests/requirements.txt +python component-tests/setup.py --type create +python -m pytest component-tests -o log_cli=true --log-cli-level=INFO if ($LASTEXITCODE -ne 0) { - & $PythonBinPath component-tests/setup.py --type cleanup + python component-tests/setup.py --type cleanup throw "Failed component tests" } -& $PythonBinPath component-tests/setup.py --type cleanup \ No newline at end of file +python component-tests/setup.py --type cleanup \ No newline at end of file From d875839e5e229c54b3e12d6b56193345fda19b34 Mon Sep 17 00:00:00 2001 From: Devin Carr Date: Tue, 25 Jun 2024 14:15:50 -0700 Subject: [PATCH 2/9] TUN-8487: Add user-agent for quick-tunnel requests --- cmd/cloudflared/tunnel/quick_tunnel.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/cloudflared/tunnel/quick_tunnel.go b/cmd/cloudflared/tunnel/quick_tunnel.go index dc8e8707..64013e58 100644 --- a/cmd/cloudflared/tunnel/quick_tunnel.go +++ b/cmd/cloudflared/tunnel/quick_tunnel.go @@ -35,7 +35,13 @@ func RunQuickTunnel(sc *subcommandContext) error { Timeout: httpTimeout, } - resp, err := client.Post(fmt.Sprintf("%s/tunnel", sc.c.String("quick-service")), "application/json", nil) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/tunnel", sc.c.String("quick-service")), nil) + if err != nil { + return errors.Wrap(err, "failed to build quick tunnel request") + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("User-Agent", buildInfo.UserAgent()) + resp, err := client.Do(req) if err != nil { return errors.Wrap(err, "failed to request quick Tunnel") } From 6174c4588b18e61efc1d9981ec1aa37dfd7cb5e4 Mon Sep 17 00:00:00 2001 From: Devin Carr Date: Wed, 26 Jun 2024 13:34:49 -0700 Subject: [PATCH 3/9] TUN-8489: Add default noop logger for capnprpc --- tunnelrpc/quic/cloudflared_client.go | 2 +- tunnelrpc/quic/cloudflared_server.go | 4 +--- tunnelrpc/quic/session_client.go | 2 +- tunnelrpc/quic/session_server.go | 4 +--- tunnelrpc/registration_client.go | 2 +- tunnelrpc/registration_server.go | 5 +---- tunnelrpc/utils.go | 20 ++++++++++++++++++++ 7 files changed, 26 insertions(+), 13 deletions(-) diff --git a/tunnelrpc/quic/cloudflared_client.go b/tunnelrpc/quic/cloudflared_client.go index 7cefd388..85a96b5f 100644 --- a/tunnelrpc/quic/cloudflared_client.go +++ b/tunnelrpc/quic/cloudflared_client.go @@ -32,7 +32,7 @@ func NewCloudflaredClient(ctx context.Context, stream io.ReadWriteCloser, reques return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(rpcStreamProtocolSignature), n) } transport := tunnelrpc.SafeTransport(stream) - conn := rpc.NewConn(transport) + conn := tunnelrpc.NewClientConn(transport) client := pogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn) return &CloudflaredClient{ client: client, diff --git a/tunnelrpc/quic/cloudflared_server.go b/tunnelrpc/quic/cloudflared_server.go index cb2c8b1a..2bba6f1c 100644 --- a/tunnelrpc/quic/cloudflared_server.go +++ b/tunnelrpc/quic/cloudflared_server.go @@ -6,8 +6,6 @@ import ( "io" "time" - "zombiezen.com/go/capnproto2/rpc" - "github.com/cloudflare/cloudflared/tunnelrpc" "github.com/cloudflare/cloudflared/tunnelrpc/pogs" ) @@ -58,7 +56,7 @@ func (s *CloudflaredServer) handleRPC(ctx context.Context, stream io.ReadWriteCl defer transport.Close() main := pogs.CloudflaredServer_ServerToClient(s.sessionManager, s.configManager) - rpcConn := rpc.NewConn(transport, rpc.MainInterface(main.Client)) + rpcConn := tunnelrpc.NewServerConn(transport, main.Client) defer rpcConn.Close() // We ignore the errors here because if cloudflared fails to handle a request, we will just move on. diff --git a/tunnelrpc/quic/session_client.go b/tunnelrpc/quic/session_client.go index 05c49910..75e36906 100644 --- a/tunnelrpc/quic/session_client.go +++ b/tunnelrpc/quic/session_client.go @@ -31,7 +31,7 @@ func NewSessionClient(ctx context.Context, stream io.ReadWriteCloser, requestTim return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(rpcStreamProtocolSignature), n) } transport := tunnelrpc.SafeTransport(stream) - conn := rpc.NewConn(transport) + conn := tunnelrpc.NewClientConn(transport) return &SessionClient{ client: pogs.NewSessionManager_PogsClient(conn.Bootstrap(ctx), conn), transport: transport, diff --git a/tunnelrpc/quic/session_server.go b/tunnelrpc/quic/session_server.go index f5269e3c..1d724742 100644 --- a/tunnelrpc/quic/session_server.go +++ b/tunnelrpc/quic/session_server.go @@ -6,8 +6,6 @@ import ( "io" "time" - "zombiezen.com/go/capnproto2/rpc" - "github.com/cloudflare/cloudflared/tunnelrpc" "github.com/cloudflare/cloudflared/tunnelrpc/pogs" ) @@ -48,7 +46,7 @@ func (s *SessionManagerServer) Serve(ctx context.Context, stream io.ReadWriteClo defer transport.Close() main := pogs.SessionManager_ServerToClient(s.sessionManager) - rpcConn := rpc.NewConn(transport, rpc.MainInterface(main.Client)) + rpcConn := tunnelrpc.NewServerConn(transport, main.Client) defer rpcConn.Close() select { diff --git a/tunnelrpc/registration_client.go b/tunnelrpc/registration_client.go index e27ad610..f41819f3 100644 --- a/tunnelrpc/registration_client.go +++ b/tunnelrpc/registration_client.go @@ -35,7 +35,7 @@ type registrationClient struct { func NewRegistrationClient(ctx context.Context, stream io.ReadWriteCloser, requestTimeout time.Duration) RegistrationClient { transport := SafeTransport(stream) - conn := rpc.NewConn(transport) + conn := NewClientConn(transport) client := pogs.NewRegistrationServer_PogsClient(conn.Bootstrap(ctx), conn) return ®istrationClient{ client: client, diff --git a/tunnelrpc/registration_server.go b/tunnelrpc/registration_server.go index 84044e84..a40e3fb3 100644 --- a/tunnelrpc/registration_server.go +++ b/tunnelrpc/registration_server.go @@ -4,8 +4,6 @@ import ( "context" "io" - "zombiezen.com/go/capnproto2/rpc" - "github.com/cloudflare/cloudflared/tunnelrpc/pogs" ) @@ -28,8 +26,7 @@ func (s *RegistrationServer) Serve(ctx context.Context, stream io.ReadWriteClose defer transport.Close() main := pogs.RegistrationServer_ServerToClient(s.registrationServer) - rpcConn := rpc.NewConn(transport, rpc.MainInterface(main.Client)) - defer rpcConn.Close() + rpcConn := NewServerConn(transport, main.Client) select { case <-rpcConn.Done(): diff --git a/tunnelrpc/utils.go b/tunnelrpc/utils.go index dd99dcea..27ac15c7 100644 --- a/tunnelrpc/utils.go +++ b/tunnelrpc/utils.go @@ -1,10 +1,12 @@ package tunnelrpc import ( + "context" "io" "time" "github.com/pkg/errors" + capnp "zombiezen.com/go/capnproto2" "zombiezen.com/go/capnproto2/rpc" ) @@ -67,3 +69,21 @@ func isTemporaryError(e error) bool { t, ok := e.(temp) return ok && t.Temporary() } + +// NoopCapnpLogger provides a logger to discard all capnp rpc internal logging messages as +// they are by default provided to stdout if no logger interface is provided. These logging +// messages in cloudflared have typically not provided a high amount of pratical value +// as the messages are extremely verbose and don't provide a good insight into the message +// contents or rpc method names. +type noopCapnpLogger struct{} + +func (noopCapnpLogger) Infof(ctx context.Context, format string, args ...interface{}) {} +func (noopCapnpLogger) Errorf(ctx context.Context, format string, args ...interface{}) {} + +func NewClientConn(transport rpc.Transport) *rpc.Conn { + return rpc.NewConn(transport, rpc.ConnLog(noopCapnpLogger{})) +} + +func NewServerConn(transport rpc.Transport, client capnp.Client) *rpc.Conn { + return rpc.NewConn(transport, rpc.MainInterface(client), rpc.ConnLog(noopCapnpLogger{})) +} From 75752b681b5a1b5337ddd01837b4bef74cc70436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveirinha?= Date: Tue, 12 Dec 2023 14:05:15 +0000 Subject: [PATCH 4/9] TUN-8057: cloudflared uses new PQ curve ID --- supervisor/pqtunnels.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervisor/pqtunnels.go b/supervisor/pqtunnels.go index f8dce98d..70a3fd69 100644 --- a/supervisor/pqtunnels.go +++ b/supervisor/pqtunnels.go @@ -12,7 +12,7 @@ import ( // to https://pqtunnels.cloudflareresearch.com. const ( - PQKex = tls.CurveID(0xfe31) // X25519Kyber768Draft00 + PQKex = tls.CurveID(0x6399) // X25519Kyber768Draft00 PQKexName = "X25519Kyber768Draft00" ) From c95959e845f0825abe4f681f70ea4d0eb59a696c Mon Sep 17 00:00:00 2001 From: lneto Date: Fri, 5 Jul 2024 19:39:22 +0100 Subject: [PATCH 5/9] TUN-8520: add macos arm64 build - refactor build script for macos to include arm64 build - refactor Makefile to upload all the artifacts instead of issuing one by one - update cfsetup due to 2. - place build files in specific folders - cleanup build directory before/after creating build artifacts --- .teamcity/mac/build.sh | 85 +++++++++++++++++++++++------------------- Makefile | 3 +- 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index d2c80be4..d1eaa568 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -12,8 +12,6 @@ export GO111MODULE=on # build 'cloudflared-darwin-amd64.tgz' mkdir -p artifacts -FILENAME="$(pwd)/artifacts/cloudflared-darwin-amd64.tgz" -PKGNAME="$(pwd)/artifacts/cloudflared-amd64.pkg" TARGET_DIRECTORY=".build" BINARY_NAME="cloudflared" VERSION=$(git describe --tags --always --dirty="-dev") @@ -28,7 +26,6 @@ export PATH="$PATH:/usr/local/bin" mkdir -p ../src/github.com/cloudflare/ cp -r . ../src/github.com/cloudflare/cloudflared cd ../src/github.com/cloudflare/cloudflared -GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared # Add code signing private key to the key chain if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then @@ -138,47 +135,59 @@ else fi fi -# sign the cloudflared binary -if [[ ! -z "$CODE_SIGN_NAME" ]]; then - codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME} +# cleanup the build directory because the previous execution might have failed without cleaning up. +rm -rf "${TARGET_DIRECTORY}" +archs=("amd64" "arm64") +export TARGET_OS=darwin +for arch in ${archs[@]}; do - # notarize the binary - # TODO: TUN-5789 -fi + FILENAME="$(pwd)/artifacts/cloudflared-darwin-$arch.tgz" + PKGNAME="$(pwd)/artifacts/cloudflared-$arch.pkg" + TARGET_ARCH=$arch GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared -# creating build directory -rm -rf $TARGET_DIRECTORY -mkdir "${TARGET_DIRECTORY}" -mkdir "${TARGET_DIRECTORY}/contents" -cp -r ".mac_resources/scripts" "${TARGET_DIRECTORY}/scripts" + # sign the cloudflared binary + if [[ ! -z "$CODE_SIGN_NAME" ]]; then + codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME} + + # notarize the binary + # TODO: TUN-5789 + fi -# copy cloudflared into the build directory -cp ${BINARY_NAME} "${TARGET_DIRECTORY}/contents/${PRODUCT}" + ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${arch}-build" + # creating build directory + rm -rf $ARCH_TARGET_DIRECTORY + mkdir "${ARCH_TARGET_DIRECTORY}" + mkdir "${ARCH_TARGET_DIRECTORY}/contents" + cp -r ".mac_resources/scripts" "${ARCH_TARGET_DIRECTORY}/scripts" -# compress cloudflared into a tar and gzipped file -tar czf "$FILENAME" "${BINARY_NAME}" + # copy cloudflared into the build directory + cp ${BINARY_NAME} "${ARCH_TARGET_DIRECTORY}/contents/${PRODUCT}" -# build the installer package -if [[ ! -z "$PKG_SIGN_NAME" ]]; then - pkgbuild --identifier com.cloudflare.${PRODUCT} \ - --version ${VERSION} \ - --scripts ${TARGET_DIRECTORY}/scripts \ - --root ${TARGET_DIRECTORY}/contents \ - --install-location /usr/local/bin \ - --sign "${PKG_SIGN_NAME}" \ - ${PKGNAME} + # compress cloudflared into a tar and gzipped file + tar czf "$FILENAME" "${BINARY_NAME}" - # notarize the package - # TODO: TUN-5789 -else + # build the installer package + if [[ ! -z "$PKG_SIGN_NAME" ]]; then pkgbuild --identifier com.cloudflare.${PRODUCT} \ - --version ${VERSION} \ - --scripts ${TARGET_DIRECTORY}/scripts \ - --root ${TARGET_DIRECTORY}/contents \ - --install-location /usr/local/bin \ - ${PKGNAME} -fi + --version ${VERSION} \ + --scripts ${ARCH_TARGET_DIRECTORY}/scripts \ + --root ${ARCH_TARGET_DIRECTORY}/contents \ + --install-location /usr/local/bin \ + --sign "${PKG_SIGN_NAME}" \ + ${PKGNAME} + # notarize the package + # TODO: TUN-5789 + else + pkgbuild --identifier com.cloudflare.${PRODUCT} \ + --version ${VERSION} \ + --scripts ${ARCH_TARGET_DIRECTORY}/scripts \ + --root ${ARCH_TARGET_DIRECTORY}/contents \ + --install-location /usr/local/bin \ + ${PKGNAME} + fi +done -# cleaning up the build directory -rm -rf $TARGET_DIRECTORY +# cleanup build the build directory because this script is not ran within containers, +# which might lead to future issues in subsequent runs. +rm -rf "${TARGET_DIRECTORY}" diff --git a/Makefile b/Makefile index 1961f03d..ce7d8f40 100644 --- a/Makefile +++ b/Makefile @@ -241,8 +241,7 @@ github-message: .PHONY: github-mac-upload github-mac-upload: - python3 github_release.py --path artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz - python3 github_release.py --path artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg + python3 github_release.py --path artifacts --release-version $(VERSION) .PHONY: github-windows-upload github-windows-upload: From 47733ba25e9dc84b01ad2b98f97baa550e566615 Mon Sep 17 00:00:00 2001 From: lneto Date: Fri, 5 Jul 2024 19:39:22 +0100 Subject: [PATCH 6/9] TUN-8523: refactor makefile and cfsetup - remove unused targets in Makefile - order deps in cfsetup.yaml - only build cloudflared not all linux targets - rename stages to be more explicit - adjust build deps of build-linux-release - adjust build deps of build-linux-fips-release - rename github_release_pkgs_pre_cache to build_release_pre_cache - only build release release artifacts within build-linux-release - only build release release artifacts within build-linux-fips-release - remove github-release-macos - remove github-release-windows - adjust builddeps of test and test-fips - create builddeps anchor for component-test and use it in component-test-fips - remove wixl from build-linux-* - rename release-pkgs-linux to r2-linux-release - add github-release: artifacts uplooad and set release message - clean build directory before build - add step to package windows binaries - refactor windows script One of TeamCity changes is moving the artifacts to the built artifacts, hence, there is no need to cp files from artifacts to built_artifacts - create anchor for release builds - create anchor for tests stages - remove reprepro and createrepo as they are only called by release_pkgs.py --- .teamcity/mac/build.sh | 2 +- .teamcity/package-windows.sh | 5 +- Makefile | 31 ++------ cfsetup.yaml | 132 +++++++++++++++-------------------- 4 files changed, 65 insertions(+), 105 deletions(-) diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index d1eaa568..865b296e 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -188,6 +188,6 @@ for arch in ${archs[@]}; do fi done -# cleanup build the build directory because this script is not ran within containers, +# cleanup build directory because this script is not ran within containers, # which might lead to future issues in subsequent runs. rm -rf "${TARGET_DIRECTORY}" diff --git a/.teamcity/package-windows.sh b/.teamcity/package-windows.sh index 9d92a21f..0139a51b 100755 --- a/.teamcity/package-windows.sh +++ b/.teamcity/package-windows.sh @@ -9,9 +9,8 @@ windowsArchs=("amd64" "386") for arch in ${windowsArchs[@]}; do export TARGET_ARCH=$arch # Copy exe into final directory - cp ./artifacts/cloudflared-windows-$arch.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe - cp ./artifacts/cloudflared-windows-$arch.exe ./cloudflared.exe + cp $ARTIFACT_DIR/cloudflared-windows-$arch.exe ./cloudflared.exe make cloudflared-msi # Copy msi into final directory mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi -done \ No newline at end of file +done diff --git a/Makefile b/Makefile index ce7d8f40..1d044ac9 100644 --- a/Makefile +++ b/Makefile @@ -218,37 +218,14 @@ cloudflared-pkg: cloudflared cloudflared.1 cloudflared-msi: wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs -.PHONY: cloudflared-darwin-amd64.tgz -cloudflared-darwin-amd64.tgz: cloudflared - tar czf cloudflared-darwin-amd64.tgz cloudflared - rm cloudflared - .PHONY: github-release -github-release: cloudflared - python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION) - -.PHONY: github-release-built-pkgs -github-release-built-pkgs: +github-release: python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) - -.PHONY: release-pkgs-linux -release-pkgs-linux: - python3 ./release_pkgs.py - -.PHONY: github-message -github-message: python3 github_message.py --release-version $(VERSION) -.PHONY: github-mac-upload -github-mac-upload: - python3 github_release.py --path artifacts --release-version $(VERSION) - -.PHONY: github-windows-upload -github-windows-upload: - python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.exe --release-version $(VERSION) --name cloudflared-windows-amd64.exe - python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.msi --release-version $(VERSION) --name cloudflared-windows-amd64.msi - python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe - python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi +.PHONY: r2-linux-release +r2-linux-release: + python3 ./release_pkgs.py .PHONY: capnp capnp: diff --git a/cfsetup.yaml b/cfsetup.yaml index 12afa0e9..953a80a1 100644 --- a/cfsetup.yaml +++ b/cfsetup.yaml @@ -3,34 +3,27 @@ pinned_go: &pinned_go go-boring=1.22.2-1 build_dir: &build_dir /cfsetup_build default-flavor: bullseye buster: &buster - build: + build-linux: build_dir: *build_dir builddeps: &build_deps - *pinned_go - build-essential - - gotest-to-teamcity - fakeroot - rubygem-fpm - rpm - libffi-dev - - reprepro - - createrepo pre-cache: &build_pre_cache - export GOCACHE=/cfsetup_build/.cache/go-build - go install golang.org/x/tools/cmd/goimports@latest post-cache: - # TODO: TUN-8126 this is temporary to make sure packages can be built before release - - ./build-packages.sh # Build binary for component test - GOOS=linux GOARCH=amd64 make cloudflared - build-fips: + build-linux-fips: build_dir: *build_dir builddeps: *build_deps pre-cache: *build_pre_cache post-cache: - export FIPS=true - # TODO: TUN-8126 this is temporary to make sure packages can be built before release - - ./build-packages-fips.sh # Build binary for component test - GOOS=linux GOARCH=amd64 make cloudflared cover: @@ -39,28 +32,21 @@ buster: &buster pre-cache: *build_pre_cache post-cache: - make cover - # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64) - github-release-pkgs: + # except FIPS and macos + build-linux-release: build_dir: *build_dir - builddeps: + builddeps: &build_deps_release - *pinned_go - build-essential - fakeroot - rubygem-fpm - rpm - - wget - # libmsi and libgcab are libraries the wixl binary depends on. - - libmsi-dev - - libgcab-dev - - python3-dev - libffi-dev - - python3-setuptools + - python3-dev - python3-pip - - reprepro - - createrepo - pre-cache: &github_release_pkgs_pre_cache - - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin - - chmod a+x /usr/local/bin/wixl + - python3-setuptools + - wget + pre-cache: &build_release_pre_cache - pip3 install pynacl==1.4.0 - pip3 install pygithub==1.55 - pip3 install boto3==1.22.9 @@ -68,32 +54,14 @@ buster: &buster post-cache: # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts - ./build-packages.sh - # release the packages built and moved to /cfsetup/built_artifacts - - make github-release-built-pkgs - # publish packages to linux repos - - make release-pkgs-linux # handle FIPS separately so that we built with gofips compiler - github-fips-release-pkgs: + build-linux-fips-release: build_dir: *build_dir - builddeps: - - *pinned_go - - build-essential - - fakeroot - - rubygem-fpm - - rpm - - wget - # libmsi and libgcab are libraries the wixl binary depends on. - - libmsi-dev - - libgcab-dev - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - pre-cache: *github_release_pkgs_pre_cache + builddeps: *build_deps_release + pre-cache: *build_release_pre_cache post-cache: # same logic as above, but for FIPS packages only - ./build-packages-fips.sh - - make github-release-built-pkgs generate-versions-file: build_dir: *build_dir builddeps: @@ -152,21 +120,7 @@ buster: &buster - export GOOS=linux - export GOARCH=arm64 - make cloudflared-deb - github-release-macos-amd64: - build_dir: *build_dir - builddeps: &build_pygithub - - *pinned_go - - build-essential - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - pre-cache: &install_pygithub - - pip3 install pynacl==1.4.0 - - pip3 install pygithub==1.55 - post-cache: - - make github-mac-upload - github-release-windows: + package-windows: build_dir: *build_dir builddeps: - *pinned_go @@ -186,10 +140,16 @@ buster: &buster - pip3 install pygithub==1.55 post-cache: - .teamcity/package-windows.sh - - make github-windows-upload test: build_dir: *build_dir - builddeps: *build_deps + builddeps: &build_deps_tests + - *pinned_go + - build-essential + - fakeroot + - rubygem-fpm + - rpm + - libffi-dev + - gotest-to-teamcity pre-cache: *build_pre_cache post-cache: - export GOOS=linux @@ -199,7 +159,7 @@ buster: &buster - make test | gotest-to-teamcity test-fips: build_dir: *build_dir - builddeps: *build_deps + builddeps: *build_deps_tests pre-cache: *build_pre_cache post-cache: - export GOOS=linux @@ -210,7 +170,7 @@ buster: &buster - make test | gotest-to-teamcity component-test: build_dir: *build_dir - builddeps: + builddeps: &build_deps_component_test - *pinned_go - python3.7 - python3-pip @@ -230,24 +190,48 @@ buster: &buster - python3 component-tests/setup.py --type cleanup component-test-fips: build_dir: *build_dir - builddeps: - - *pinned_go - - python3.7 - - python3-pip - - python3-setuptools - # procps installs the ps command which is needed in test_sysv_service because the init script - # uses ps pid to determine if the agent is running - - procps + builddeps: *build_deps_component_test pre-cache-copy-paths: - component-tests/requirements.txt pre-cache: *component_test_pre_cache post-cache: *component_test_post_cache - github-message-release: + github-release: build_dir: *build_dir - builddeps: *build_pygithub - pre-cache: *install_pygithub + builddeps: + - *pinned_go + - build-essential + - python3-dev + - libffi-dev + - python3-setuptools + - python3-pip + pre-cache: + - pip3 install pynacl==1.4.0 + - pip3 install pygithub==1.55 post-cache: + - make github-release - make github-message + r2-linux-release: + build_dir: *build_dir + builddeps: + - *pinned_go + - build-essential + - fakeroot + - rubygem-fpm + - rpm + - wget + - python3-dev + - libffi-dev + - python3-setuptools + - python3-pip + - reprepro + - createrepo + pre-cache: + - pip3 install pynacl==1.4.0 + - pip3 install pygithub==1.55 + - pip3 install boto3==1.22.9 + - pip3 install python-gnupg==0.4.9 + post-cache: + - make r2-linux-release bullseye: *buster bookworm: *buster From 13b2e423eda8d8c21737ba6442c486f47ba2c920 Mon Sep 17 00:00:00 2001 From: lneto Date: Mon, 15 Jul 2024 14:24:16 +0100 Subject: [PATCH 7/9] Release 2024.7.0 --- RELEASE_NOTES | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 9b1c9727..2ce3edc0 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,11 @@ +2024.7.0 +- 2024-07-05 TUN-8520: add macos arm64 build +- 2024-07-05 TUN-8523: refactor makefile and cfsetup +- 2024-07-02 TUN-8504: Use pre-installed python version instead of downloading it on Windows builds +- 2024-06-26 TUN-8489: Add default noop logger for capnprpc +- 2024-06-25 TUN-8487: Add user-agent for quick-tunnel requests +- 2023-12-12 TUN-8057: cloudflared uses new PQ curve ID + 2024.6.1 - 2024-06-12 TUN-8461: Don't log Failed to send session payload if the error is EOF - 2024-06-07 TUN-8456: Update quic-go to 0.45 and collect mtu and congestion control metrics From 26ae1ca3c8ade8184847c0968613f5e57ce9421f Mon Sep 17 00:00:00 2001 From: lneto Date: Tue, 16 Jul 2024 12:10:31 +0100 Subject: [PATCH 8/9] TUN-8543: use -p flag to create intermediate directories --- .teamcity/mac/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index 865b296e..588ef42f 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -156,8 +156,8 @@ for arch in ${archs[@]}; do ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${arch}-build" # creating build directory rm -rf $ARCH_TARGET_DIRECTORY - mkdir "${ARCH_TARGET_DIRECTORY}" - mkdir "${ARCH_TARGET_DIRECTORY}/contents" + mkdir -p "${ARCH_TARGET_DIRECTORY}" + mkdir -p "${ARCH_TARGET_DIRECTORY}/contents" cp -r ".mac_resources/scripts" "${ARCH_TARGET_DIRECTORY}/scripts" # copy cloudflared into the build directory From db239e7319243c9cd80d82a570bd4cb661730c72 Mon Sep 17 00:00:00 2001 From: lneto Date: Tue, 16 Jul 2024 16:24:52 +0100 Subject: [PATCH 9/9] Release 2024.7.1 --- RELEASE_NOTES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 2ce3edc0..36972ceb 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,6 @@ +2024.7.1 +- 2024-07-16 TUN-8543: use -p flag to create intermediate directories + 2024.7.0 - 2024-07-05 TUN-8520: add macos arm64 build - 2024-07-05 TUN-8523: refactor makefile and cfsetup