diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..0569e8ea --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,121 @@ +stages: [build, release] + +default: + id_tokens: + VAULT_ID_TOKEN: + aud: https://vault.cfdata.org + +# This before_script is injected into every job that runs on master meaning that if there is no tag the step +# will succeed but only write "No tag present - Skipping" to the console. +.check_tag: + before_script: + - | + # Check if there is a Git tag pointing to HEAD + echo "Tag found: $(git tag --points-at HEAD | grep .)" + if git tag --points-at HEAD | grep .; then + echo "Tag found: $(git tag --points-at HEAD | grep .)" + export "VERSION=$(git tag --points-at HEAD | grep .)" + else + echo "No tag present — skipping." + exit 0 + fi + +# ----------------------------------------------- +# Stage 1: Build on every PR +# ----------------------------------------------- +build_cloudflared_macos: &build + stage: build + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_BRANCH != "master" + when: always + - when: never + tags: + - "macstadium-${RUNNER_ARCH}" + parallel: + matrix: + - RUNNER_ARCH: [arm, intel] + artifacts: + paths: + - artifacts/* + script: + - '[ "${RUNNER_ARCH}" = "arm" ] && export TARGET_ARCH=arm64' + - '[ "${RUNNER_ARCH}" = "intel" ] && export TARGET_ARCH=amd64' + - ARCH=$(uname -m) + - echo ARCH=$ARCH - TARGET_ARCH=$TARGET_ARCH + - ./.teamcity/mac/install-cloudflare-go.sh + - export PATH="/tmp/go/bin:$PATH" + - BUILD_SCRIPT=.teamcity/mac/build.sh + - if [[ ! -x ${BUILD_SCRIPT} ]] ; then exit ; fi + - set -euo pipefail + - echo "Executing ${BUILD_SCRIPT}" + - exec ${BUILD_SCRIPT} + +# ----------------------------------------------- +# Stage 1: Build and sign only on releases +# ----------------------------------------------- +build_and_sign_cloudflared_macos: + <<: *build + extends: .check_tag + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - when: never + secrets: + APPLE_DEV_CA_CERT: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/apple_dev_ca_cert/data@kv + file: false + CFD_CODE_SIGN_CERT: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_cert_v2/data@kv + file: false + CFD_CODE_SIGN_KEY: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_key_v2/data@kv + file: false + CFD_CODE_SIGN_PASS: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_pass_v2/data@kv + file: false + CFD_INSTALLER_CERT: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_cert_v2/data@kv + file: false + CFD_INSTALLER_KEY: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_key_v2/data@kv + file: false + CFD_INSTALLER_PASS: + vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_pass_v2/data@kv + file: false + +# ----------------------------------------------- +# Stage 2: Release to Github after building and signing +# ----------------------------------------------- +release_cloudflared_macos_to_github: + stage: release + image: docker-registry.cfdata.org/stash/tun/docker-images/cloudflared-ci/main:6-8616fe631b76-amd64@sha256:96f4fd05e66cec03e0864c1bcf09324c130d4728eef45ee994716da499183614 + extends: .check_tag + dependencies: + - build_and_sign_cloudflared_macos + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - when: never + cache: + paths: + - .cache/pip + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + KV_NAMESPACE: 380e19aa04314648949b6ad841417ebe + KV_ACCOUNT: 5ab4e9dfbd435d24068829fda0077963 + secrets: + KV_API_TOKEN: + vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_kv_api_token/data@kv + file: false + API_KEY: + vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_github_api_key/data@kv + file: false + script: + - python3 --version ; pip --version # For debugging + - python3 -m venv venv + - source venv/bin/activate + - pip install pynacl==1.4.0 pygithub==1.55 + - echo $VERSION + - echo $TAG_EXISTS + - echo "Running release because tag exists." + - make macos-release diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index ff50a252..765c1de5 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -49,7 +49,7 @@ import_certificate() { echo -n -e ${CERTIFICATE_ENV_VAR} | base64 -D > ${CERTIFICATE_FILE_NAME} # we set || true here and for every `security import invoke` because the "duplicate SecKeychainItemImport" error # will cause set -e to exit 1. It is okay we do this because we deliberately handle this error in the lines below. - local out=$(security import ${CERTIFICATE_FILE_NAME} -A 2>&1) || true + local out=$(security import ${CERTIFICATE_FILE_NAME} -T /usr/bin/pkgbuild -A 2>&1) || true local exitcode=$? # delete the certificate from disk rm -rf ${CERTIFICATE_FILE_NAME} @@ -68,6 +68,28 @@ import_certificate() { fi } +create_cloudflared_build_keychain() { + # Reusing the private key password as the keychain key + local PRIVATE_KEY_PASS=$1 + + # Create keychain only if it doesn't already exist + if [ ! -f "$HOME/Library/Keychains/cloudflared_build_keychain.keychain-db" ]; then + security create-keychain -p "$PRIVATE_KEY_PASS" cloudflared_build_keychain + else + echo "Keychain already exists: cloudflared_build_keychain" + fi + + # Append temp keychain to the user domain + security list-keychains -d user -s cloudflared_build_keychain $(security list-keychains -d user | sed s/\"//g) + + # Remove relock timeout + security set-keychain-settings cloudflared_build_keychain + + # Unlock keychain so it doesn't require password + security unlock-keychain -p "$PRIVATE_KEY_PASS" cloudflared_build_keychain + +} + # Imports private keys to the Apple KeyChain import_private_keys() { local PRIVATE_KEY_NAME=$1 @@ -83,7 +105,7 @@ import_private_keys() { echo -n -e ${PRIVATE_KEY_ENV_VAR} | base64 -D > ${PRIVATE_KEY_FILE_NAME} # we set || true here and for every `security import invoke` because the "duplicate SecKeychainItemImport" error # will cause set -e to exit 1. It is okay we do this because we deliberately handle this error in the lines below. - local out=$(security import ${PRIVATE_KEY_FILE_NAME} -A -P "${PRIVATE_KEY_PASS}" 2>&1) || true + local out=$(security import ${PRIVATE_KEY_FILE_NAME} -k cloudflared_build_keychain -P "$PRIVATE_KEY_PASS" -T /usr/bin/pkgbuild -A -P "${PRIVATE_KEY_PASS}" 2>&1) || true local exitcode=$? rm -rf ${PRIVATE_KEY_FILE_NAME} if [ -n "$out" ]; then @@ -100,6 +122,9 @@ import_private_keys() { fi } +# Create temp keychain only for this build +create_cloudflared_build_keychain "${CFD_CODE_SIGN_PASS}" + # Add Apple Root Developer certificate to the key chain import_certificate "Apple Developer CA" "${APPLE_DEV_CA_CERT}" "${APPLE_CA_CERT}" @@ -119,8 +144,8 @@ import_certificate "Developer ID Installer" "${CFD_INSTALLER_CERT}" "${INSTALLER if [[ ! -z "$CFD_CODE_SIGN_NAME" ]]; then CODE_SIGN_NAME="${CFD_CODE_SIGN_NAME}" else - if [[ -n "$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)" ]]; then - CODE_SIGN_NAME=$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1) + if [[ -n "$(security find-certificate -c "Developer ID Application" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)" ]]; then + CODE_SIGN_NAME=$(security find-certificate -c "Developer ID Application" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1) else CODE_SIGN_NAME="" fi @@ -130,8 +155,8 @@ fi if [[ ! -z "$CFD_INSTALLER_NAME" ]]; then PKG_SIGN_NAME="${CFD_INSTALLER_NAME}" else - if [[ -n "$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)" ]]; then - PKG_SIGN_NAME=$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1) + if [[ -n "$(security find-certificate -c "Developer ID Installer" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)" ]]; then + PKG_SIGN_NAME=$(security find-certificate -c "Developer ID Installer" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1) else PKG_SIGN_NAME="" fi @@ -142,9 +167,16 @@ rm -rf "${TARGET_DIRECTORY}" export TARGET_OS="darwin" GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared + +# This allows apple tools to use the certificates in the keychain without requiring password input. +# This command always needs to run after the certificates have been loaded into the keychain +if [[ ! -z "$CFD_CODE_SIGN_PASS" ]]; then + security set-key-partition-list -S apple-tool:,apple: -s -k "${CFD_CODE_SIGN_PASS}" cloudflared_build_keychain +fi + # sign the cloudflared binary if [[ ! -z "$CODE_SIGN_NAME" ]]; then - codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME} + codesign --keychain $HOME/Library/Keychains/cloudflared_build_keychain.keychain-db -s "${CODE_SIGN_NAME}" -fv --options runtime --timestamp ${BINARY_NAME} # notarize the binary # TODO: TUN-5789 @@ -165,11 +197,13 @@ tar czf "$FILENAME" "${BINARY_NAME}" # build the installer package if [[ ! -z "$PKG_SIGN_NAME" ]]; then + pkgbuild --identifier com.cloudflare.${PRODUCT} \ --version ${VERSION} \ --scripts ${ARCH_TARGET_DIRECTORY}/scripts \ --root ${ARCH_TARGET_DIRECTORY}/contents \ --install-location /usr/local/bin \ + --keychain cloudflared_build_keychain \ --sign "${PKG_SIGN_NAME}" \ ${PKGNAME} @@ -187,3 +221,8 @@ fi # cleanup build directory because this script is not ran within containers, # which might lead to future issues in subsequent runs. rm -rf "${TARGET_DIRECTORY}" + +# cleanup the keychain +security default-keychain -d user -s login.keychain-db +security list-keychains -d user -s login.keychain-db +security delete-keychain cloudflared_build_keychain diff --git a/Makefile b/Makefile index dcd9cebd..a1ffedf0 100644 --- a/Makefile +++ b/Makefile @@ -237,6 +237,10 @@ github-release: python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) python3 github_message.py --release-version $(VERSION) +.PHONY: macos-release +macos-release: + python3 github_release.py --path $(PWD)/artifacts/ --release-version $(VERSION) + .PHONY: r2-linux-release r2-linux-release: python3 ./release_pkgs.py diff --git a/github_release.py b/github_release.py index 4620a139..c3a71739 100755 --- a/github_release.py +++ b/github_release.py @@ -61,7 +61,7 @@ def assert_tag_exists(repo, version): raise Exception("Tag {} not found".format(version)) -def get_or_create_release(repo, version, dry_run=False): +def get_or_create_release(repo, version, dry_run=False, is_draft=False): """ Get a Github Release matching the version tag or create a new one. If a conflict occurs on creation, attempt to fetch the Release on last time @@ -81,8 +81,11 @@ def get_or_create_release(repo, version, dry_run=False): return try: - logging.info("Creating release %s", version) - return repo.create_git_release(version, version, "") + if is_draft: + logging.info("Drafting release %s", version) + else: + logging.info("Creating release %s", version) + return repo.create_git_release(version, version, "", is_draft) except GithubException as e: errors = e.data.get("errors", []) if e.status == 422 and any( @@ -129,6 +132,10 @@ def parse_args(): "--dry-run", action="store_true", help="Do not create release or upload asset" ) + parser.add_argument( + "--draft", action="store_true", help="Create a draft release" + ) + args = parser.parse_args() is_valid = True if not args.release_version: @@ -292,7 +299,7 @@ def main(): for filename in onlyfiles: binary_path = os.path.join(args.path, filename) assert_asset_version(binary_path, args.release_version) - release = get_or_create_release(repo, args.release_version, args.dry_run) + release = get_or_create_release(repo, args.release_version, args.dry_run, args.draft) for filename in onlyfiles: binary_path = os.path.join(args.path, filename) upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id,