From 102631d98d9eb0a5d71b6d7459ceaead439404c2 Mon Sep 17 00:00:00 2001 From: Igor Postelnik Date: Tue, 14 Jun 2022 21:53:02 -0500 Subject: [PATCH] TUN-6395: Fix writing RPM repo data --- release_pkgs.py | 141 ++++++++++++++++++++++++++---------------------- 1 file changed, 78 insertions(+), 63 deletions(-) diff --git a/release_pkgs.py b/release_pkgs.py index 36f672ff..c5a26a76 100644 --- a/release_pkgs.py +++ b/release_pkgs.py @@ -8,23 +8,24 @@ them to be in an uploadable state. 2. Upload these packages to a storage in a format that apt and yum expect. """ -from subprocess import Popen, PIPE -import os import argparse import base64 -from pathlib import Path import logging +import os import shutil from hashlib import sha256 +from pathlib import Path +from subprocess import Popen, PIPE -import gnupg import boto3 +import gnupg from botocore.client import Config from botocore.exceptions import ClientError # The front facing R2 URL to access assets from. R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/' + class PkgUploader: def __init__(self, account_id, bucket_name, client_id, client_secret): self.account_id = account_id @@ -35,14 +36,14 @@ class PkgUploader: def upload_pkg_to_r2(self, filename, upload_file_path): endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com" token_secret_hash = sha256(self.client_secret.encode()).hexdigest() - + config = Config( - region_name = 'auto', + region_name='auto', s3={ "addressing_style": "path", } ) - + r2 = boto3.client( "s3", endpoint_url=endpoint_url, @@ -57,7 +58,7 @@ class PkgUploader: except ClientError as e: raise e - + class PkgCreator: """ The distribution conf is what dictates to reprepro, the debian packaging building @@ -72,15 +73,16 @@ class PkgCreator: description - (String) gpg_key_id - gpg key id of what you want to use to sign the packages.(String) """ - def create_distribution_conf(self, - file_path, - origin, - label, - releases, - archs, - components, - description, - gpg_key_id ): + + def create_distribution_conf(self, + file_path, + origin, + label, + releases, + archs, + components, + description, + gpg_key_id): with open(file_path, "w+") as distributions_file: for release in releases: distributions_file.write(f"Origin: {origin}\n") @@ -102,6 +104,7 @@ class PkgCreator: db and pool contain information and metadata about builds. We can ignore these. dist: contains all the pkgs and signed releases that are necessary for an apt download. """ + def create_deb_pkgs(self, release, deb_file): print(f"creating deb pkgs: {release} : {deb_file}") p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE) @@ -130,8 +133,9 @@ class PkgCreator: gpgcheck=1 gpgkey=https://pkg.cloudflare.com/cloudflare-main.gpg """ + def create_repo_file(self, file_path, binary_name, baseurl, gpgkey_url): - with open(file_path, "w+") as repo_file: + with open(os.path.join(file_path, binary_name + '.repo'), "w+") as repo_file: repo_file.write(f"[{binary_name}-stable]") repo_file.write(f"{binary_name}-stable") repo_file.write(f"baseurl={baseurl}/rpm") @@ -141,19 +145,19 @@ class PkgCreator: repo_file.write(f"gpgkey={gpgkey_url}") def _sign_rpms(self, file_path): - p = Popen(["rpm" , "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE) + p = Popen(["rpm", "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE) out, err = p.communicate() if p.returncode != 0: print(f"rpm sign result result => {out}, {err}") raise def _sign_repomd(self): - p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE) + p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE) out, err = p.communicate() if p.returncode != 0: print(f"sign repomd result => {out}, {err}") raise - + """ sets up and signs the RPM directories in the following format: - rpm @@ -163,9 +167,10 @@ class PkgCreator: this assumes the assets are in the format -.rpm """ + def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch64", "x86_64", "386"]): for arch in archs: - for root, _ , files in os.walk(artifacts_path): + for root, _, files in os.walk(artifacts_path): for file in files: if file.endswith(f"{arch}.rpm"): new_dir = f"./rpm/{arch}" @@ -174,11 +179,12 @@ class PkgCreator: new_path = os.path.join(new_dir, file) shutil.copyfile(old_path, new_path) self._sign_rpms(new_path) - + """ imports gpg keys into the system so reprepro and createrepo can use it to sign packages. it returns the GPG ID after a successful import """ + def import_gpg_keys(self, private_key, public_key): gpg = gnupg.GPG() private_key = base64.b64decode(private_key) @@ -192,12 +198,13 @@ class PkgCreator: basically rpm --import This enables us to sign rpms. """ + def import_rpm_key(self, public_key): file_name = "pb.key" with open(file_name, "wb") as f: public_key = base64.b64decode(public_key) f.write(public_key) - + p = Popen(["rpm", "--import", file_name], stdout=PIPE, stderr=PIPE) out, err = p.communicate() if p.returncode != 0: @@ -212,18 +219,21 @@ class PkgCreator: and the release will be uploaded to the default path. binary: name of the binary to upload """ + + def upload_from_directories(pkg_uploader, directory, release, binary): - for root, _ , files in os.walk(directory): + for root, _, files in os.walk(directory): for file in files: upload_file_name = os.path.join(binary, root, file) if release: upload_file_name = os.path.join(release, upload_file_name) - filename = os.path.join(root,file) - try: + filename = os.path.join(root, file) + try: pkg_uploader.upload_pkg_to_r2(filename, upload_file_name) except ClientError as e: logging.error(e) - return + return + """ 1. looks into a built_artifacts folder for cloudflared debs @@ -237,19 +247,22 @@ def upload_from_directories(pkg_uploader, directory, release, binary): release_version: is the cloudflared release version. Only provide this if you want a permanent backup. """ -def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, release_version): + + +def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, + release_version): # set configuration for package creation. print(f"initialising configuration for {binary_name} , {archs}") Path("./conf").mkdir(parents=True, exist_ok=True) pkg_creator.create_distribution_conf( - "./conf/distributions", - binary_name, - binary_name, - releases, - archs, - package_component, - f"apt repository for {binary_name}", - gpg_key_id) + "./conf/distributions", + binary_name, + binary_name, + releases, + archs, + package_component, + f"apt repository for {binary_name}", + gpg_key_id) # create deb pkgs for release in releases: @@ -266,6 +279,7 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary upload_from_directories(pkg_uploader, "dists", release_version, binary_name) upload_from_directories(pkg_uploader, "pool", release_version, binary_name) + def create_rpm_packaging( pkg_creator, pkg_uploader, @@ -294,60 +308,61 @@ def parse_args(): ) parser.add_argument( - "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name" + "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name" ) parser.add_argument( - "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID" + "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID" ) parser.add_argument( - "--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret" + "--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret" ) parser.add_argument( - "--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag" + "--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag" ) parser.add_argument( - "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\ + "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\ prefixed with. Leave empty if you don't want tagged release versions backed up to R2." ) parser.add_argument( - "--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for" + "--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for" ) parser.add_argument( - "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\ + "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\ packages" ) parser.add_argument( - "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\ + "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\ signing packages" ) parser.add_argument( - "--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\ + "--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\ downloaders can use to verify signing" ) parser.add_argument( - "--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders" + "--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders" ) parser.add_argument( - "--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic", - "xenial", "trusty"], - help="list of debian based releases that need to be packaged for" + "--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic", + "xenial", "trusty"], + help="list of debian based releases that need to be packaged for" ) parser.add_argument( - "--archs", default=["amd64", "386", "arm64", "arm"], help="list of architectures we want to package for. Note that\ + "--archs", default=["amd64", "386", "arm64", "arm"], help="list of architectures we want to package for. Note that\ it is the caller's responsiblity to ensure that these debs are already present in a directory. This script\ will not build binaries or create their debs." - ) + ) args = parser.parse_args() return args + if __name__ == "__main__": try: args = parse_args() @@ -362,15 +377,15 @@ if __name__ == "__main__": pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret) print(f"signing with gpg_key: {gpg_key_id}") create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs, - "main", args.release_tag) - + "main", args.release_tag) + create_rpm_packaging( - pkg_creator, - pkg_uploader, - "./built_artifacts", - args.release_tag, - args.binary, - gpg_key_name, - args.gpg_public_key_url, - args.pkg_upload_url, - ) + pkg_creator, + pkg_uploader, + "./built_artifacts", + args.release_tag, + args.binary, + gpg_key_name, + args.gpg_public_key_url, + args.pkg_upload_url, + )