DEVTOOLS-16383: Create GitlabCI pipeline to release Mac builds

Adds a new Gitlab CI pipeline that releases cloudflared Mac builds and replaces the Teamcity adhoc job.
This will build, sign and create a new Github release or add the artifacts to an existing release if the other jobs finish first.

.gitlab-ci.yml

@@ -0,0 +1,121 @@
+stages: [build, release]
+
+default:
+  id_tokens:
+    VAULT_ID_TOKEN:
+      aud: https://vault.cfdata.org
+
+# This before_script is injected into every job that runs on master meaning that if there is no tag the step
+# will succeed but only write "No tag present - Skipping" to the console.
+.check_tag:
+  before_script:
+    - |
+      # Check if there is a Git tag pointing to HEAD
+      echo "Tag found: $(git tag --points-at HEAD | grep .)"
+      if git tag --points-at HEAD | grep .; then
+        echo "Tag found: $(git tag --points-at HEAD | grep .)"
+        export "VERSION=$(git tag --points-at HEAD | grep .)"
+      else
+        echo "No tag present — skipping."
+        exit 0
+      fi
+
+# -----------------------------------------------
+# Stage 1: Build on every PR
+# -----------------------------------------------
+build_cloudflared_macos: &build
+  stage: build
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_BRANCH != "master"
+      when: always
+    - when: never
+  tags:
+    - "macstadium-${RUNNER_ARCH}"
+  parallel:
+    matrix:
+      - RUNNER_ARCH: [arm, intel]
+  artifacts:
+    paths:
+      - artifacts/*
+  script:
+    - '[ "${RUNNER_ARCH}" = "arm" ] && export TARGET_ARCH=arm64'
+    - '[ "${RUNNER_ARCH}" = "intel" ] && export TARGET_ARCH=amd64'
+    - ARCH=$(uname -m)
+    - echo ARCH=$ARCH - TARGET_ARCH=$TARGET_ARCH
+    - ./.teamcity/mac/install-cloudflare-go.sh
+    - export PATH="/tmp/go/bin:$PATH"
+    - BUILD_SCRIPT=.teamcity/mac/build.sh
+    - if [[ ! -x ${BUILD_SCRIPT} ]] ; then exit ; fi
+    - set -euo pipefail
+    - echo "Executing ${BUILD_SCRIPT}"
+    - exec ${BUILD_SCRIPT}
+
+# -----------------------------------------------
+# Stage 1: Build and sign only on releases
+# -----------------------------------------------
+build_and_sign_cloudflared_macos:
+  <<: *build
+  extends: .check_tag
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      when: always
+    - when: never
+  secrets:
+    APPLE_DEV_CA_CERT:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/apple_dev_ca_cert/data@kv
+      file: false
+    CFD_CODE_SIGN_CERT:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_cert_v2/data@kv
+      file: false
+    CFD_CODE_SIGN_KEY:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_key_v2/data@kv
+      file: false
+    CFD_CODE_SIGN_PASS:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_pass_v2/data@kv
+      file: false
+    CFD_INSTALLER_CERT:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_cert_v2/data@kv
+      file: false
+    CFD_INSTALLER_KEY:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_key_v2/data@kv
+      file: false
+    CFD_INSTALLER_PASS:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_pass_v2/data@kv
+      file: false
+
+# -----------------------------------------------
+# Stage 2: Release to Github after building and signing
+# -----------------------------------------------
+release_cloudflared_macos_to_github:
+  stage: release
+  image: docker-registry.cfdata.org/stash/tun/docker-images/cloudflared-ci/main:6-8616fe631b76-amd64@sha256:96f4fd05e66cec03e0864c1bcf09324c130d4728eef45ee994716da499183614
+  extends: .check_tag
+  dependencies:
+    - build_and_sign_cloudflared_macos
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      when: always
+    - when: never
+  cache:
+    paths:
+      - .cache/pip
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+    KV_NAMESPACE: 380e19aa04314648949b6ad841417ebe
+    KV_ACCOUNT: 5ab4e9dfbd435d24068829fda0077963
+  secrets:
+    KV_API_TOKEN:
+      vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_kv_api_token/data@kv
+      file: false
+    API_KEY:
+      vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_github_api_key/data@kv
+      file: false
+  script:
+    - python3 --version ; pip --version  # For debugging
+    - python3 -m venv venv
+    - source venv/bin/activate
+    - pip install pynacl==1.4.0 pygithub==1.55
+    - echo $VERSION
+    - echo $TAG_EXISTS
+    - echo "Running release because tag exists."
+    - make macos-release

.teamcity/mac/build.sh

@@ -49,7 +49,7 @@ import_certificate() {
       echo -n -e ${CERTIFICATE_ENV_VAR} | base64 -D > ${CERTIFICATE_FILE_NAME}
       # we set || true  here and for every `security import invoke` because the  "duplicate SecKeychainItemImport" error
       # will cause set -e to exit 1. It is okay we do this because we deliberately handle this error in the lines below.
-      local out=$(security import ${CERTIFICATE_FILE_NAME} -A 2>&1) || true
+      local out=$(security import ${CERTIFICATE_FILE_NAME} -T /usr/bin/pkgbuild -A 2>&1) || true
       local exitcode=$?
       # delete the certificate from disk
       rm -rf ${CERTIFICATE_FILE_NAME}
@@ -68,6 +68,28 @@ import_certificate() {
     fi
 }
 
+create_cloudflared_build_keychain() {
+  # Reusing the private key password as the keychain key
+  local PRIVATE_KEY_PASS=$1
+
+  # Create keychain only if it doesn't already exist
+  if [ ! -f "$HOME/Library/Keychains/cloudflared_build_keychain.keychain-db" ]; then
+    security create-keychain -p "$PRIVATE_KEY_PASS" cloudflared_build_keychain
+  else
+    echo "Keychain already exists: cloudflared_build_keychain"
+  fi
+
+  # Append temp keychain to the user domain
+  security list-keychains -d user -s cloudflared_build_keychain $(security list-keychains -d user | sed s/\"//g)
+
+  # Remove relock timeout
+  security set-keychain-settings cloudflared_build_keychain
+
+  # Unlock keychain so it doesn't require password
+  security unlock-keychain -p "$PRIVATE_KEY_PASS" cloudflared_build_keychain
+
+}
+
 # Imports private keys to the Apple KeyChain
 import_private_keys() {
     local PRIVATE_KEY_NAME=$1
@@ -83,7 +105,7 @@ import_private_keys() {
         echo -n -e ${PRIVATE_KEY_ENV_VAR} | base64 -D > ${PRIVATE_KEY_FILE_NAME}
         # we set || true  here and for every `security import invoke` because the  "duplicate SecKeychainItemImport" error
         # will cause set -e to exit 1. It is okay we do this because we deliberately handle this error in the lines below.
-        local out=$(security import ${PRIVATE_KEY_FILE_NAME} -A -P "${PRIVATE_KEY_PASS}" 2>&1) || true
+        local out=$(security import ${PRIVATE_KEY_FILE_NAME} -k cloudflared_build_keychain -P "$PRIVATE_KEY_PASS" -T /usr/bin/pkgbuild -A -P "${PRIVATE_KEY_PASS}" 2>&1) || true
         local exitcode=$?
         rm -rf ${PRIVATE_KEY_FILE_NAME}
         if [ -n "$out" ]; then
@@ -100,6 +122,9 @@ import_private_keys() {
     fi
 }
 
+# Create temp keychain only for this build
+create_cloudflared_build_keychain "${CFD_CODE_SIGN_PASS}"
+
 # Add Apple Root Developer certificate to the key chain
 import_certificate "Apple Developer CA" "${APPLE_DEV_CA_CERT}" "${APPLE_CA_CERT}"
 
@@ -119,8 +144,8 @@ import_certificate "Developer ID Installer" "${CFD_INSTALLER_CERT}" "${INSTALLER
 if [[ ! -z "$CFD_CODE_SIGN_NAME" ]]; then
   CODE_SIGN_NAME="${CFD_CODE_SIGN_NAME}"
 else
-  if [[ -n "$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)" ]]; then
+  if [[ -n "$(security find-certificate -c "Developer ID Application" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)" ]]; then
-    CODE_SIGN_NAME=$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)
+    CODE_SIGN_NAME=$(security find-certificate -c "Developer ID Application" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)
   else
     CODE_SIGN_NAME=""
   fi
@@ -130,8 +155,8 @@ fi
 if [[ ! -z "$CFD_INSTALLER_NAME" ]]; then
   PKG_SIGN_NAME="${CFD_INSTALLER_NAME}"
 else
-  if [[ -n "$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)" ]]; then
+  if [[ -n "$(security find-certificate -c "Developer ID Installer" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)" ]]; then
-    PKG_SIGN_NAME=$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)
+    PKG_SIGN_NAME=$(security find-certificate -c "Developer ID Installer" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)
   else
     PKG_SIGN_NAME=""
   fi
@@ -142,9 +167,16 @@ rm -rf "${TARGET_DIRECTORY}"
 export TARGET_OS="darwin"
 GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared
 
+
+# This allows apple tools to use the certificates in the keychain without requiring password input.
+# This command always needs to run after the certificates have been loaded into the keychain
+if [[ ! -z "$CFD_CODE_SIGN_PASS" ]]; then
+  security set-key-partition-list -S apple-tool:,apple: -s -k "${CFD_CODE_SIGN_PASS}" cloudflared_build_keychain
+fi
+
 # sign the cloudflared binary
 if [[ ! -z "$CODE_SIGN_NAME" ]]; then
-  codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
+  codesign --keychain $HOME/Library/Keychains/cloudflared_build_keychain.keychain-db -s "${CODE_SIGN_NAME}" -fv --options runtime --timestamp ${BINARY_NAME}
 
  # notarize the binary
  # TODO: TUN-5789
@@ -165,11 +197,13 @@ tar czf "$FILENAME" "${BINARY_NAME}"
 
 # build the installer package
 if [[ ! -z "$PKG_SIGN_NAME" ]]; then
+
   pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \
       --scripts ${ARCH_TARGET_DIRECTORY}/scripts \
       --root ${ARCH_TARGET_DIRECTORY}/contents \
       --install-location /usr/local/bin \
+      --keychain cloudflared_build_keychain \
       --sign "${PKG_SIGN_NAME}" \
       ${PKGNAME}
 
@@ -187,3 +221,8 @@ fi
 # cleanup build directory because this script is not ran within containers,
 # which might lead to future issues in subsequent runs.
 rm -rf "${TARGET_DIRECTORY}"
+
+# cleanup the keychain
+security default-keychain -d user -s login.keychain-db
+security list-keychains -d user -s login.keychain-db
+security delete-keychain cloudflared_build_keychain

Makefile

@@ -237,6 +237,10 @@ github-release:
 	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
 	python3 github_message.py --release-version $(VERSION)
 
+.PHONY: macos-release
+macos-release:
+	python3 github_release.py --path $(PWD)/artifacts/ --release-version $(VERSION)
+
 .PHONY: r2-linux-release
 r2-linux-release:
 	python3 ./release_pkgs.py

github_release.py

@@ -61,7 +61,7 @@ def assert_tag_exists(repo, version):
         raise Exception("Tag {} not found".format(version))
 
 
-def get_or_create_release(repo, version, dry_run=False):
+def get_or_create_release(repo, version, dry_run=False, is_draft=False):
     """
     Get a Github Release matching the version tag or create a new one.
     If a conflict occurs on creation, attempt to fetch the Release on last time
@@ -81,8 +81,11 @@ def get_or_create_release(repo, version, dry_run=False):
         return
 
     try:
-        logging.info("Creating release %s", version)
+        if is_draft:
-        return repo.create_git_release(version, version, "")
+            logging.info("Drafting release %s", version)
+        else:
+            logging.info("Creating release %s", version)
+        return repo.create_git_release(version, version, "", is_draft)
     except GithubException as e:
         errors = e.data.get("errors", [])
         if e.status == 422 and any(
@@ -129,6 +132,10 @@ def parse_args():
         "--dry-run", action="store_true", help="Do not create release or upload asset"
     )
 
+    parser.add_argument(
+        "--draft", action="store_true", help="Create a draft release"
+    )
+
     args = parser.parse_args()
     is_valid = True
     if not args.release_version:
@@ -292,7 +299,7 @@ def main():
                 for filename in onlyfiles:
                     binary_path = os.path.join(args.path, filename)
                     assert_asset_version(binary_path, args.release_version)
-                release = get_or_create_release(repo, args.release_version, args.dry_run)
+                release = get_or_create_release(repo, args.release_version, args.dry_run, args.draft)
                 for filename in onlyfiles:
                     binary_path = os.path.join(args.path, filename)
                     upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id,