TUN-6175: Simply debian packaging by structural upload

The way apt works is:

1. It looks at the release file based on the `deb` added to sources.list.
2. It uses this release file to find the relative location of Packages or Packages.gz
3. It uses the pool information from packages to find the relative location of where the .deb file is located and then downloads and installs it.

This PR seeks to take advantage of this information by simply arranging
the files in a way apt expects thereby eliminating the need for an
orchestrating endpoint.
This commit is contained in:
Sudarsan Reddy 2022-05-05 16:04:48 +01:00
parent 46c147a1b2
commit 2cf43abe8c
1 changed files with 30 additions and 73 deletions

View File

@ -6,14 +6,10 @@
1. Create deb and yum repositories from .deb and .rpm files. 1. Create deb and yum repositories from .deb and .rpm files.
This is also responsible for signing the packages and generally preparing This is also responsible for signing the packages and generally preparing
them to be in an uploadable state. them to be in an uploadable state.
2. Upload these packages to a storage and add metadata cross reference 2. Upload these packages to a storage in a format that apt and yum expect.
for these to be accessed.
""" """
import requests
import subprocess import subprocess
import os import os
import io
import shutil
import logging import logging
from hashlib import sha256 from hashlib import sha256
@ -21,52 +17,22 @@ import boto3
from botocore.client import Config from botocore.client import Config
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
# The front facing R2 URL to access assets from. # The front facing R2 URL to access assets from.
R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/' R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/'
class PkgUploader: class PkgUploader:
def __init__(self, kv_api_token, namespace, account_id, bucket_name, client_id, client_secret): def __init__(self, account_id, bucket_name, client_id, client_secret):
self.kv_api_token = kv_api_token
self.namespace = namespace
self.account_id = account_id self.account_id = account_id
self.bucket_name = bucket_name self.bucket_name = bucket_name
self.client_id = client_id self.client_id = client_id
self.client_secret = client_secret self.client_secret = client_secret
def send_to_kv(self, key, value):
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer " + self.kv_api_token,
}
kv_url = f"{BASE_KV_URL}{self.account_id}/storage/kv/namespaces/{self.namespace}/values/{key}"
response = requests.put(
kv_url,
headers=headers,
data=value
)
if response.status_code != 200:
jsonResponse = response.json()
errors = jsonResponse["errors"]
if len(errors) > 0:
raise Exception("failed to send to workers kv: {0}", errors[0])
else:
raise Exception("recieved error code: {0}", response.status_code)
def send_pkg_info(self, binary, flavor, asset_name, arch, uploaded_package_location):
key = f"pkg_{binary}_{flavor}_{arch}_{asset_name}"
print(f"writing key:{key} , value: {uploaded_package_location}")
self.send_to_kv(key, uploaded_package_location)
def upload_pkg_to_r2(self, filename, upload_file_path): def upload_pkg_to_r2(self, filename, upload_file_path):
endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com" endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com"
token_secret_hash = sha256(self.client_secret.encode()).hexdigest() token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
config = Config( config = Config(
region_name = 'auto',
s3={ s3={
"addressing_style": "path", "addressing_style": "path",
} }
@ -141,15 +107,19 @@ class PkgCreator:
def _clean_build_resources(self): def _clean_build_resources(self):
subprocess.call(("reprepro", "clearvanished")) subprocess.call(("reprepro", "clearvanished"))
def upload_from_directories(pkg_uploader, directory, arch, release): """
Walks through a directory and uploads it's assets to R2.
directory : root directory to walk through (String).
release: release string. If this value is none, a specific release path will not be created
and the release will be uploaded to the default path.
binary: name of the binary to upload
"""
def upload_from_directories(pkg_uploader, directory, release, binary):
for root, _ , files in os.walk(directory): for root, _ , files in os.walk(directory):
for file in files: for file in files:
root_elements = root.split("/") upload_file_name = os.path.join(binary, root, file)
upload_file_name = os.path.join(root, arch, release, file) if release:
flavor_prefix = root_elements[1] upload_file_name = os.path.join(release, upload_file_name)
if root_elements[0] == "pool":
upload_file_name = os.path.join(root, file)
flavor_prefix = "deb"
filename = os.path.join(root,file) filename = os.path.join(root,file)
try: try:
pkg_uploader.upload_pkg_to_r2(filename, upload_file_name) pkg_uploader.upload_pkg_to_r2(filename, upload_file_name)
@ -157,20 +127,10 @@ def upload_from_directories(pkg_uploader, directory, arch, release):
logging.error(e) logging.error(e)
return return
# save to workers kv in the following formats
# Example:
# key : pkg_cloudflared_bullseye_InRelease,
# value: https://r2.cloudflarestorage.com/dists/bullseye/amd64/2022_3_4/InRelease
r2_asset_url = f"{R2_ASSET_URL}{upload_file_name}"
pkg_uploader.send_pkg_info("cloudflared", flavor_prefix, upload_file_name, arch, r2_asset_url)
# TODO https://jira.cfops.it/browse/TUN-6163: Add a key for latest version.
""" """
1. looks into a built_artifacts folder for cloudflared debs 1. looks into a built_artifacts folder for cloudflared debs
2. creates Packages.gz, InRelease (signed) files 2. creates Packages.gz, InRelease (signed) files
3. uploads them to Cloudflare R2 and 3. uploads them to Cloudflare R2
4. adds a Workers KV reference
pkg_creator, pkg_uploader: are instantiations of the two classes above. pkg_creator, pkg_uploader: are instantiations of the two classes above.
@ -179,32 +139,32 @@ def upload_from_directories(pkg_uploader, directory, arch, release):
release_version: is the cloudflared release version. release_version: is the cloudflared release version.
""" """
def create_deb_packaging(pkg_creator, pkg_uploader, flavors, gpg_key_id, binary_name, arch, package_component, release_version): def create_deb_packaging(pkg_creator, pkg_uploader, flavors, gpg_key_id, binary_name, archs, package_component, release_version):
# set configuration for package creation. # set configuration for package creation.
print(f"initialising configuration for {binary_name} , {arch}") print(f"initialising configuration for {binary_name} , {archs}")
pkg_creator.create_distribution_conf( pkg_creator.create_distribution_conf(
"./conf/distributions", "./conf/distributions",
binary_name, binary_name,
binary_name, binary_name,
flavors, flavors,
[arch], archs,
package_component, package_component,
f"apt repository for {binary_name}", f"apt repository for {binary_name}",
gpg_key_id) gpg_key_id)
# create deb pkgs # create deb pkgs
for flavor in flavors: for flavor in flavors:
print(f"creating deb pkgs for {flavor} and {arch}...") for arch in archs:
pkg_creator.create_deb_pkgs(flavor, f"./built_artifacts/cloudflared-linux-{arch}.deb") print(f"creating deb pkgs for {flavor} and {arch}...")
pkg_creator.create_deb_pkgs(flavor, f"./built_artifacts/cloudflared-linux-{arch}.deb")
print("uploading to r2...") print("uploading latest to r2...")
upload_from_directories(pkg_uploader, "dists", arch, release_version) upload_from_directories(pkg_uploader, "dists", None, binary_name)
upload_from_directories(pkg_uploader, "pool", arch, release_version) upload_from_directories(pkg_uploader, "pool", None, binary_name)
print("cleaning up directories...") print(f"uploading versioned release {release_version} to r2...")
shutil.rmtree("./dists") upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
shutil.rmtree("./pool") upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
shutil.rmtree("./db")
#TODO: https://jira.cfops.it/browse/TUN-6146 will extract this into it's own command line script. #TODO: https://jira.cfops.it/browse/TUN-6146 will extract this into it's own command line script.
if __name__ == "__main__": if __name__ == "__main__":
@ -215,15 +175,12 @@ if __name__ == "__main__":
bucket_name = os.getenv('R2_BUCKET_NAME') bucket_name = os.getenv('R2_BUCKET_NAME')
client_id = os.getenv('R2_CLIENT_ID') client_id = os.getenv('R2_CLIENT_ID')
client_secret = os.getenv('R2_CLIENT_SECRET') client_secret = os.getenv('R2_CLIENT_SECRET')
tunnel_account_id = '5ab4e9dfbd435d24068829fda0077963' tunnel_account_id = os.getenv('R2_ACCOUNT_ID')
kv_namespace = os.getenv('KV_NAMESPACE')
kv_api_token = os.getenv('KV_API_TOKEN')
release_version = os.getenv('RELEASE_VERSION') release_version = os.getenv('RELEASE_VERSION')
gpg_key_id = os.getenv('GPG_KEY_ID') gpg_key_id = os.getenv('GPG_KEY_ID')
pkg_uploader = PkgUploader(kv_api_token, kv_namespace, tunnel_account_id, bucket_name, client_id, client_secret) pkg_uploader = PkgUploader(tunnel_account_id, bucket_name, client_id, client_secret)
archs = ["amd64", "386", "arm64"] archs = ["amd64", "386", "arm64"]
flavors = ["bullseye", "buster", "bionic"] flavors = ["bullseye", "buster", "bionic"]
for arch in archs: create_deb_packaging(pkg_creator, pkg_uploader, flavors, gpg_key_id, "cloudflared", archs, "main", release_version)
create_deb_packaging(pkg_creator, pkg_uploader, flavors, gpg_key_id, "cloudflared", arch, "main", release_version)