TUN-6054: Create and upload deb packages to R2
This PR does the following:
1. Creates packages.gz, signed InRelease files for debs in
built_artifacts for configured debian releases.
2. Uploads them to Cloudflare R2.
3. Adds a Workers KV entry that talks about where these assets are
uploaded.
This commit is contained in:
Sudarsan Reddy
parent
8250708b37
commit
1e71202c89
|
|
@ -0,0 +1,229 @@
|
|||
"""
|
||||
This is a utility for creating deb and rpm packages, signing them
|
||||
and uploading them to a storage and adding metadata to workers KV.
|
||||
|
||||
It has two over-arching responsiblities:
|
||||
1. Create deb and yum repositories from .deb and .rpm files.
|
||||
This is also responsible for signing the packages and generally preparing
|
||||
them to be in an uploadable state.
|
||||
2. Upload these packages to a storage and add metadata cross reference
|
||||
for these to be accessed.
|
||||
"""
|
||||
import requests
|
||||
import subprocess
|
||||
import os
|
||||
import io
|
||||
import shutil
|
||||
import logging
|
||||
from hashlib import sha256
|
||||
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
|
||||
# The front facing R2 URL to access assets from.
|
||||
R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/'
|
||||
|
||||
class PkgUploader:
|
||||
def __init__(self, kv_api_token, namespace, account_id, bucket_name, client_id, client_secret):
|
||||
self.kv_api_token = kv_api_token
|
||||
self.namespace = namespace
|
||||
self.account_id = account_id
|
||||
self.bucket_name = bucket_name
|
||||
self.client_id = client_id
|
||||
self.client_secret = client_secret
|
||||
|
||||
def send_to_kv(self, key, value):
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer " + self.kv_api_token,
|
||||
}
|
||||
|
||||
kv_url = f"{BASE_KV_URL}{self.account_id}/storage/kv/namespaces/{self.namespace}/values/{key}"
|
||||
response = requests.put(
|
||||
kv_url,
|
||||
headers=headers,
|
||||
data=value
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
jsonResponse = response.json()
|
||||
errors = jsonResponse["errors"]
|
||||
if len(errors) > 0:
|
||||
raise Exception("failed to send to workers kv: {0}", errors[0])
|
||||
else:
|
||||
raise Exception("recieved error code: {0}", response.status_code)
|
||||
|
||||
|
||||
def send_pkg_info(self, binary, flavor, asset_name, arch, uploaded_package_location):
|
||||
key = f"pkg_{binary}_{flavor}_{arch}_{asset_name}"
|
||||
print(f"writing key:{key} , value: {uploaded_package_location}")
|
||||
self.send_to_kv(key, uploaded_package_location)
|
||||
|
||||
|
||||
def upload_pkg_to_r2(self, filename, upload_file_path):
|
||||
endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com"
|
||||
token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
|
||||
|
||||
config = Config(
|
||||
s3={
|
||||
"addressing_style": "path",
|
||||
}
|
||||
)
|
||||
|
||||
r2 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint_url,
|
||||
aws_access_key_id=self.client_id,
|
||||
aws_secret_access_key=token_secret_hash,
|
||||
config=config,
|
||||
)
|
||||
|
||||
print(f"uploading asset: {filename} to {upload_file_path}...")
|
||||
try:
|
||||
r2.upload_file(filename, self.bucket_name, upload_file_path)
|
||||
except ClientError as e:
|
||||
raise e
|
||||
|
||||
|
||||
class PkgCreator:
|
||||
"""
|
||||
The distribution conf is what dictates to reprepro, the debian packaging building
|
||||
and signing tool we use, what distros to support, what GPG key to use for signing
|
||||
and what to call the debian binary etc. This function creates it "./conf/distributions".
|
||||
|
||||
origin - name of your package (String)
|
||||
label - label of your package (could be same as the name) (String)
|
||||
flavor - flavor you want this to be distributed for (List of Strings)
|
||||
components - could be a channel like main/stable/beta
|
||||
archs - Architecture (List of Strings)
|
||||
description - (String)
|
||||
gpg_key_id - gpg key id of what you want to use to sign the packages.(String)
|
||||
"""
|
||||
def create_distribution_conf(self,
|
||||
file_path,
|
||||
origin,
|
||||
label,
|
||||
flavors,
|
||||
archs,
|
||||
components,
|
||||
description,
|
||||
gpg_key_id ):
|
||||
with open(file_path, "w") as distributions_file:
|
||||
for flavor in flavors:
|
||||
distributions_file.write(f"Origin: {origin}\n")
|
||||
distributions_file.write(f"Label: {label}\n")
|
||||
distributions_file.write(f"Codename: {flavor}\n")
|
||||
archs_list = " ".join(archs)
|
||||
distributions_file.write(f"Architectures: {archs_list}\n")
|
||||
distributions_file.write(f"Components: {components}\n")
|
||||
distributions_file.write(f"Description: {description} - {flavor}\n")
|
||||
distributions_file.write(f"SignWith: {gpg_key_id}\n")
|
||||
distributions_file.write("\n")
|
||||
return distributions_file
|
||||
|
||||
"""
|
||||
Uses the reprepro tool to generate packages, sign them and create the InRelease as specified
|
||||
by the distribution_conf file.
|
||||
|
||||
This function creates three folders db, pool and dist.
|
||||
db and pool contain information and metadata about builds. We can ignore these.
|
||||
dist: contains all the pkgs and signed releases that are necessary for an apt download.
|
||||
"""
|
||||
def create_deb_pkgs(self, flavor, deb_file):
|
||||
self._clean_build_resources()
|
||||
subprocess.call(("reprepro", "includedeb", flavor, deb_file))
|
||||
|
||||
"""
|
||||
This is mostly useful to clear previously built db, dist and pool resources.
|
||||
"""
|
||||
def _clean_build_resources(self):
|
||||
subprocess.call(("reprepro", "clearvanished"))
|
||||
|
||||
def upload_from_directories(pkg_uploader, directory, arch, release):
|
||||
for root, _ , files in os.walk(directory):
|
||||
for file in files:
|
||||
root_elements = root.split("/")
|
||||
upload_file_name = os.path.join(root, arch, release, file)
|
||||
flavor_prefix = root_elements[1]
|
||||
if root_elements[0] == "pool":
|
||||
upload_file_name = os.path.join(root, file)
|
||||
flavor_prefix = "deb"
|
||||
filename = os.path.join(root,file)
|
||||
try:
|
||||
pkg_uploader.upload_pkg_to_r2(filename, upload_file_name)
|
||||
except ClientError as e:
|
||||
logging.error(e)
|
||||
return
|
||||
|
||||
# save to workers kv in the following formats
|
||||
# Example:
|
||||
# key : pkg_cloudflared_bullseye_InRelease,
|
||||
# value: https://r2.cloudflarestorage.com/dists/bullseye/amd64/2022_3_4/InRelease
|
||||
r2_asset_url = f"{R2_ASSET_URL}{upload_file_name}"
|
||||
pkg_uploader.send_pkg_info("cloudflared", flavor_prefix, upload_file_name, arch, r2_asset_url)
|
||||
|
||||
# TODO https://jira.cfops.it/browse/TUN-6163: Add a key for latest version.
|
||||
|
||||
"""
|
||||
1. looks into a built_artifacts folder for cloudflared debs
|
||||
2. creates Packages.gz, InRelease (signed) files
|
||||
3. uploads them to Cloudflare R2 and
|
||||
4. adds a Workers KV reference
|
||||
|
||||
pkg_creator, pkg_uploader: are instantiations of the two classes above.
|
||||
|
||||
gpg_key_id: is an id indicating the key the package should be signed with. The public key of this id will be
|
||||
uploaded to R2 so it can be presented to apt downloaders.
|
||||
|
||||
release_version: is the cloudflared release version.
|
||||
"""
|
||||
def create_deb_packaging(pkg_creator, pkg_uploader, flavors, gpg_key_id, binary_name, arch, package_component, release_version):
|
||||
# set configuration for package creation.
|
||||
print(f"initialising configuration for {binary_name} , {arch}")
|
||||
pkg_creator.create_distribution_conf(
|
||||
"./conf/distributions",
|
||||
binary_name,
|
||||
binary_name,
|
||||
flavors,
|
||||
[arch],
|
||||
package_component,
|
||||
f"apt repository for {binary_name}",
|
||||
gpg_key_id)
|
||||
|
||||
# create deb pkgs
|
||||
for flavor in flavors:
|
||||
print(f"creating deb pkgs for {flavor} and {arch}...")
|
||||
pkg_creator.create_deb_pkgs(flavor, f"./built_artifacts/cloudflared-linux-{arch}.deb")
|
||||
|
||||
print("uploading to r2...")
|
||||
upload_from_directories(pkg_uploader, "dists", arch, release_version)
|
||||
upload_from_directories(pkg_uploader, "pool", arch, release_version)
|
||||
|
||||
print("cleaning up directories...")
|
||||
shutil.rmtree("./dists")
|
||||
shutil.rmtree("./pool")
|
||||
shutil.rmtree("./db")
|
||||
|
||||
#TODO: https://jira.cfops.it/browse/TUN-6146 will extract this into it's own command line script.
|
||||
if __name__ == "__main__":
|
||||
# initialise pkg creator
|
||||
pkg_creator = PkgCreator()
|
||||
|
||||
# initialise pkg uploader
|
||||
bucket_name = os.getenv('R2_BUCKET_NAME')
|
||||
client_id = os.getenv('R2_CLIENT_ID')
|
||||
client_secret = os.getenv('R2_CLIENT_SECRET')
|
||||
tunnel_account_id = '5ab4e9dfbd435d24068829fda0077963'
|
||||
kv_namespace = os.getenv('KV_NAMESPACE')
|
||||
kv_api_token = os.getenv('KV_API_TOKEN')
|
||||
release_version = os.getenv('RELEASE_VERSION')
|
||||
gpg_key_id = os.getenv('GPG_KEY_ID')
|
||||
|
||||
pkg_uploader = PkgUploader(kv_api_token, kv_namespace, tunnel_account_id, bucket_name, client_id, client_secret)
|
||||
|
||||
archs = ["amd64", "386", "arm64"]
|
||||
flavors = ["bullseye", "buster", "bionic"]
|
||||
for arch in archs:
|
||||
create_deb_packaging(pkg_creator, pkg_uploader, flavors, gpg_key_id, "cloudflared", arch, "main", release_version)
|
||||
Loading…
Reference in New Issue