TUN-6209: Improve feedback process if release_pkgs to deb and rpm fail

This PR mostly raises exceptions so we are aware if release deb or
release pkgs fail. It also makes release_version optional if backup pkgs
are not needed.
This commit is contained in:
Sudarsan Reddy 2022-05-24 13:20:17 +01:00
parent 7ce2bb8b2f
commit 32739e9f98
2 changed files with 23 additions and 18 deletions

View File

@ -48,7 +48,7 @@ stretch: &stretch
- pip3 install pynacl==1.4.0 - pip3 install pynacl==1.4.0
- pip3 install pygithub==1.55 - pip3 install pygithub==1.55
- pip3 install boto3==1.22.9 - pip3 install boto3==1.22.9
- pip3 install gnupg==2.3.1 - pip3 install python-gnupg==0.4.9
post-cache: post-cache:
# build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
- ./build-packages.sh - ./build-packages.sh

View File

@ -8,7 +8,7 @@
them to be in an uploadable state. them to be in an uploadable state.
2. Upload these packages to a storage in a format that apt and yum expect. 2. Upload these packages to a storage in a format that apt and yum expect.
""" """
import subprocess from subprocess import Popen, PIPE
import os import os
import argparse import argparse
import base64 import base64
@ -102,19 +102,21 @@ class PkgCreator:
dist: contains all the pkgs and signed releases that are necessary for an apt download. dist: contains all the pkgs and signed releases that are necessary for an apt download.
""" """
def create_deb_pkgs(self, release, deb_file): def create_deb_pkgs(self, release, deb_file):
self._clean_build_resources() print(f"creating deb pkgs: {release} : {deb_file}")
subprocess.call(("reprepro", "includedeb", release, deb_file), timeout=120) p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
""" if p.returncode != 0:
This is mostly useful to clear previously built db, dist and pool resources. print(f"create deb_pkgs result => {out}, {err}")
""" raise
def _clean_build_resources(self):
subprocess.call(("reprepro", "clearvanished"), timeout=120)
# TODO https://jira.cfops.it/browse/TUN-6209 : Sign these packages. # TODO https://jira.cfops.it/browse/TUN-6209 : Sign these packages.
def create_rpm_pkgs(self, artifacts_path): def create_rpm_pkgs(self, artifacts_path):
self._setup_rpm_pkg_directories(artifacts_path) self._setup_rpm_pkg_directories(artifacts_path)
subprocess.call(("createrepo", "./rpm"), timeout=120) p = Popen(["createrepo", "./rpm"], stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
if p.returncode != 0:
print(f"create rpm_pkgs result => {out}, {err}")
raise
""" """
sets up the RPM directories in the following format: sets up the RPM directories in the following format:
@ -180,7 +182,7 @@ def upload_from_directories(pkg_uploader, directory, release, binary):
gpg_key_id: is an id indicating the key the package should be signed with. The public key of this id will be gpg_key_id: is an id indicating the key the package should be signed with. The public key of this id will be
uploaded to R2 so it can be presented to apt downloaders. uploaded to R2 so it can be presented to apt downloaders.
release_version: is the cloudflared release version. release_version: is the cloudflared release version. Only provide this if you want a permanent backup.
""" """
def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, release_version): def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, release_version):
# set configuration for package creation. # set configuration for package creation.
@ -205,6 +207,7 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary
upload_from_directories(pkg_uploader, "dists", None, binary_name) upload_from_directories(pkg_uploader, "dists", None, binary_name)
upload_from_directories(pkg_uploader, "pool", None, binary_name) upload_from_directories(pkg_uploader, "pool", None, binary_name)
if release_version:
print(f"uploading versioned release {release_version} to r2...") print(f"uploading versioned release {release_version} to r2...")
upload_from_directories(pkg_uploader, "dists", release_version, binary_name) upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
upload_from_directories(pkg_uploader, "pool", release_version, binary_name) upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
@ -216,6 +219,7 @@ def create_rpm_packaging(pkg_creator, pkg_uploader, artifacts_path, release_vers
print("uploading latest to r2...") print("uploading latest to r2...")
upload_from_directories(pkg_uploader, "rpm", None, binary_name) upload_from_directories(pkg_uploader, "rpm", None, binary_name)
if release_version:
print(f"uploading versioned release {release_version} to r2...") print(f"uploading versioned release {release_version} to r2...")
upload_from_directories(pkg_uploader, "rpm", release_version, binary_name) upload_from_directories(pkg_uploader, "rpm", release_version, binary_name)
@ -239,7 +243,7 @@ def parse_args():
) )
parser.add_argument( parser.add_argument(
"--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\ "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
prefixed with" prefixed with. Leave empty if you don't want tagged release versions backed up to R2."
) )
parser.add_argument( parser.add_argument(
@ -281,6 +285,7 @@ if __name__ == "__main__":
gpg_key_id = pkg_creator.import_gpg_keys(args.gpg_private_key, args.gpg_public_key) gpg_key_id = pkg_creator.import_gpg_keys(args.gpg_private_key, args.gpg_public_key)
pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret) pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
print(f"signing with gpg_key: {gpg_key_id}")
create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs, create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs,
"main", args.release_tag) "main", args.release_tag)