curben
/
cloudflared-mirror
mirror of https://github.com/cloudflare/cloudflared.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

TUN-6395: Fix writing RPM repo data

This commit is contained in:
Igor Postelnik 2022-06-14 21:53:02 -05:00
parent 6c3d2fc339
commit 102631d98d
1 changed files with 78 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
release_pkgs.py
View File

@ -8,23 +8,24 @@
them to be in an uploadable state. them to be in an uploadable state.
2. Upload these packages to a storage in a format that apt and yum expect. 2. Upload these packages to a storage in a format that apt and yum expect.
""" """
from subprocess import Popen, PIPE
import os
import argparse import argparse
import base64 import base64
from pathlib import Path
import logging import logging
import os
import shutil import shutil
from hashlib import sha256 from hashlib import sha256
from pathlib import Path
from subprocess import Popen, PIPE
import gnupg
import boto3 import boto3
import gnupg
from botocore.client import Config from botocore.client import Config
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
# The front facing R2 URL to access assets from. # The front facing R2 URL to access assets from.
R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/' R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/'
class PkgUploader: class PkgUploader:
def __init__(self, account_id, bucket_name, client_id, client_secret): def __init__(self, account_id, bucket_name, client_id, client_secret):
self.account_id = account_id self.account_id = account_id
@ -37,7 +38,7 @@ class PkgUploader:
token_secret_hash = sha256(self.client_secret.encode()).hexdigest() token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
config = Config( config = Config(
region_name = 'auto', region_name='auto',
s3={ s3={
"addressing_style": "path", "addressing_style": "path",
} }
@ -72,15 +73,16 @@ class PkgCreator:
description - (String) description - (String)
gpg_key_id - gpg key id of what you want to use to sign the packages.(String) gpg_key_id - gpg key id of what you want to use to sign the packages.(String)
""" """
def create_distribution_conf(self, def create_distribution_conf(self,
file_path, file_path,
origin, origin,
label, label,
releases, releases,
archs, archs,
components, components,
description, description,
gpg_key_id ): gpg_key_id):
with open(file_path, "w+") as distributions_file: with open(file_path, "w+") as distributions_file:
for release in releases: for release in releases:
distributions_file.write(f"Origin: {origin}\n") distributions_file.write(f"Origin: {origin}\n")
@ -102,6 +104,7 @@ class PkgCreator:
db and pool contain information and metadata about builds. We can ignore these. db and pool contain information and metadata about builds. We can ignore these.
dist: contains all the pkgs and signed releases that are necessary for an apt download. dist: contains all the pkgs and signed releases that are necessary for an apt download.
""" """
def create_deb_pkgs(self, release, deb_file): def create_deb_pkgs(self, release, deb_file):
print(f"creating deb pkgs: {release} : {deb_file}") print(f"creating deb pkgs: {release} : {deb_file}")
p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE) p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE)
@ -130,8 +133,9 @@ class PkgCreator:
gpgcheck=1 gpgcheck=1
gpgkey=https://pkg.cloudflare.com/cloudflare-main.gpg gpgkey=https://pkg.cloudflare.com/cloudflare-main.gpg
""" """
def create_repo_file(self, file_path, binary_name, baseurl, gpgkey_url): def create_repo_file(self, file_path, binary_name, baseurl, gpgkey_url):
with open(file_path, "w+") as repo_file: with open(os.path.join(file_path, binary_name + '.repo'), "w+") as repo_file:
repo_file.write(f"[{binary_name}-stable]") repo_file.write(f"[{binary_name}-stable]")
repo_file.write(f"{binary_name}-stable") repo_file.write(f"{binary_name}-stable")
repo_file.write(f"baseurl={baseurl}/rpm") repo_file.write(f"baseurl={baseurl}/rpm")
@ -141,7 +145,7 @@ class PkgCreator:
repo_file.write(f"gpgkey={gpgkey_url}") repo_file.write(f"gpgkey={gpgkey_url}")
def _sign_rpms(self, file_path): def _sign_rpms(self, file_path):
p = Popen(["rpm" , "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE) p = Popen(["rpm", "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
out, err = p.communicate() out, err = p.communicate()
if p.returncode != 0: if p.returncode != 0:
print(f"rpm sign result result => {out}, {err}") print(f"rpm sign result result => {out}, {err}")
@ -163,9 +167,10 @@ class PkgCreator:
this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm
""" """
def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch64", "x86_64", "386"]): def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch64", "x86_64", "386"]):
for arch in archs: for arch in archs:
for root, _ , files in os.walk(artifacts_path): for root, _, files in os.walk(artifacts_path):
for file in files: for file in files:
if file.endswith(f"{arch}.rpm"): if file.endswith(f"{arch}.rpm"):
new_dir = f"./rpm/{arch}" new_dir = f"./rpm/{arch}"
@ -179,6 +184,7 @@ class PkgCreator:
imports gpg keys into the system so reprepro and createrepo can use it to sign packages. imports gpg keys into the system so reprepro and createrepo can use it to sign packages.
it returns the GPG ID after a successful import it returns the GPG ID after a successful import
""" """
def import_gpg_keys(self, private_key, public_key): def import_gpg_keys(self, private_key, public_key):
gpg = gnupg.GPG() gpg = gnupg.GPG()
private_key = base64.b64decode(private_key) private_key = base64.b64decode(private_key)
@ -192,6 +198,7 @@ class PkgCreator:
basically rpm --import <key_file> basically rpm --import <key_file>
This enables us to sign rpms. This enables us to sign rpms.
""" """
def import_rpm_key(self, public_key): def import_rpm_key(self, public_key):
file_name = "pb.key" file_name = "pb.key"
with open(file_name, "wb") as f: with open(file_name, "wb") as f:
@ -212,19 +219,22 @@ class PkgCreator:
and the release will be uploaded to the default path. and the release will be uploaded to the default path.
binary: name of the binary to upload binary: name of the binary to upload
""" """
def upload_from_directories(pkg_uploader, directory, release, binary): def upload_from_directories(pkg_uploader, directory, release, binary):
for root, _ , files in os.walk(directory): for root, _, files in os.walk(directory):
for file in files: for file in files:
upload_file_name = os.path.join(binary, root, file) upload_file_name = os.path.join(binary, root, file)
if release: if release:
upload_file_name = os.path.join(release, upload_file_name) upload_file_name = os.path.join(release, upload_file_name)
filename = os.path.join(root,file) filename = os.path.join(root, file)
try: try:
pkg_uploader.upload_pkg_to_r2(filename, upload_file_name) pkg_uploader.upload_pkg_to_r2(filename, upload_file_name)
except ClientError as e: except ClientError as e:
logging.error(e) logging.error(e)
return return
""" """
1. looks into a built_artifacts folder for cloudflared debs 1. looks into a built_artifacts folder for cloudflared debs
2. creates Packages.gz, InRelease (signed) files 2. creates Packages.gz, InRelease (signed) files
@ -237,19 +247,22 @@ def upload_from_directories(pkg_uploader, directory, release, binary):
release_version: is the cloudflared release version. Only provide this if you want a permanent backup. release_version: is the cloudflared release version. Only provide this if you want a permanent backup.
""" """
def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, release_version):
def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component,
release_version):
# set configuration for package creation. # set configuration for package creation.
print(f"initialising configuration for {binary_name} , {archs}") print(f"initialising configuration for {binary_name} , {archs}")
Path("./conf").mkdir(parents=True, exist_ok=True) Path("./conf").mkdir(parents=True, exist_ok=True)
pkg_creator.create_distribution_conf( pkg_creator.create_distribution_conf(
"./conf/distributions", "./conf/distributions",
binary_name, binary_name,
binary_name, binary_name,
releases, releases,
archs, archs,
package_component, package_component,
f"apt repository for {binary_name}", f"apt repository for {binary_name}",
gpg_key_id) gpg_key_id)
# create deb pkgs # create deb pkgs
for release in releases: for release in releases:
@ -266,6 +279,7 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary
upload_from_directories(pkg_uploader, "dists", release_version, binary_name) upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
upload_from_directories(pkg_uploader, "pool", release_version, binary_name) upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
def create_rpm_packaging( def create_rpm_packaging(
pkg_creator, pkg_creator,
pkg_uploader, pkg_uploader,
@ -294,60 +308,61 @@ def parse_args():
) )
parser.add_argument( parser.add_argument(
"--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name" "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
) )
parser.add_argument( parser.add_argument(
"--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID" "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
) )
parser.add_argument( parser.add_argument(
"--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret" "--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret"
) )
parser.add_argument( parser.add_argument(
"--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag" "--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag"
) )
parser.add_argument( parser.add_argument(
"--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\ "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
prefixed with. Leave empty if you don't want tagged release versions backed up to R2." prefixed with. Leave empty if you don't want tagged release versions backed up to R2."
) )
parser.add_argument( parser.add_argument(
"--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for" "--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for"
) )
parser.add_argument( parser.add_argument(
"--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\ "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
packages" packages"
) )
parser.add_argument( parser.add_argument(
"--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\ "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
signing packages" signing packages"
) )
parser.add_argument( parser.add_argument(
"--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\ "--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\
downloaders can use to verify signing" downloaders can use to verify signing"
) )
parser.add_argument( parser.add_argument(
"--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders" "--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders"
) )
parser.add_argument( parser.add_argument(
"--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic", "--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic",
"xenial", "trusty"], "xenial", "trusty"],
help="list of debian based releases that need to be packaged for" help="list of debian based releases that need to be packaged for"
) )
parser.add_argument( parser.add_argument(
"--archs", default=["amd64", "386", "arm64", "arm"], help="list of architectures we want to package for. Note that\ "--archs", default=["amd64", "386", "arm64", "arm"], help="list of architectures we want to package for. Note that\
it is the caller's responsiblity to ensure that these debs are already present in a directory. This script\ it is the caller's responsiblity to ensure that these debs are already present in a directory. This script\
will not build binaries or create their debs." will not build binaries or create their debs."
) )
args = parser.parse_args() args = parser.parse_args()
return args return args
if __name__ == "__main__": if __name__ == "__main__":
try: try:
args = parse_args() args = parse_args()
@ -362,15 +377,15 @@ if __name__ == "__main__":
pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret) pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
print(f"signing with gpg_key: {gpg_key_id}") print(f"signing with gpg_key: {gpg_key_id}")
create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs, create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs,
"main", args.release_tag) "main", args.release_tag)
create_rpm_packaging( create_rpm_packaging(
pkg_creator, pkg_creator,
pkg_uploader, pkg_uploader,
"./built_artifacts", "./built_artifacts",
args.release_tag, args.release_tag,
args.binary, args.binary,
gpg_key_name, gpg_key_name,
args.gpg_public_key_url, args.gpg_public_key_url,
args.pkg_upload_url, args.pkg_upload_url,
) )