From 4b6437cc60f0f0a7fe732816c549abad36e371fb Mon Sep 17 00:00:00 2001 From: Sudarsan Reddy Date: Thu, 12 May 2022 08:32:09 +0100 Subject: [PATCH] TUN-5943: Add RPM support This PR extends release_pkgs.py to now also support uploading rpm based assets to R2. The packages are not signed yet and will be done in a subsequent PR. This PR - Packs the .rpm assets into relevant directories - Calls createrepo on them to make them yum repo ready - Uploads them to R2 --- Makefile | 4 ++++ cfsetup.yaml | 7 ++++++- release_pkgs.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4fda3d6a..f82dbe8f 100644 --- a/Makefile +++ b/Makefile @@ -261,6 +261,10 @@ github-release: cloudflared github-release-built-pkgs: python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) +.PHONY: release-pkgs-linux +release-pkgs-linux: + python3 ./release_pkgs.py + .PHONY: github-message github-message: python3 github_message.py --release-version $(VERSION) diff --git a/cfsetup.yaml b/cfsetup.yaml index 4dc51ad4..cdad6ce8 100644 --- a/cfsetup.yaml +++ b/cfsetup.yaml @@ -40,14 +40,19 @@ stretch: &stretch - libffi-dev - python3-setuptools - python3-pip + - reprepro + - createrepo pre-cache: &github_release_pkgs_pre_cache - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin - chmod a+x /usr/local/bin/wixl - pip3 install pynacl==1.4.0 - pip3 install pygithub==1.55 + - pip3 install boto3==1.22.9 post-cache: # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts - ./build-packages.sh + # publish packages to linux repos + # TODO TUN-6180: Uncomment this - make release-pkgs-linux # release the packages built and moved to /cfsetup/built_artifacts - make github-release-built-pkgs # handle FIPS separately so that we built with gofips compiler @@ -255,4 +260,4 @@ centos-7: - export PATH=$PATH:/usr/local/go/bin - export GOOS=linux - export GOARCH=amd64 - - make publish-rpm \ No newline at end of file + - make publish-rpm diff --git a/release_pkgs.py b/release_pkgs.py index 3671676a..67b1fb3d 100644 --- a/release_pkgs.py +++ b/release_pkgs.py @@ -12,6 +12,7 @@ import subprocess import os import argparse import logging +import shutil from hashlib import sha256 import boto3 @@ -100,13 +101,38 @@ class PkgCreator: """ def create_deb_pkgs(self, release, deb_file): self._clean_build_resources() - subprocess.call(("reprepro", "includedeb", release, deb_file)) + subprocess.call(("reprepro", "includedeb", release, deb_file), timeout=120) """ This is mostly useful to clear previously built db, dist and pool resources. """ def _clean_build_resources(self): - subprocess.call(("reprepro", "clearvanished")) + subprocess.call(("reprepro", "clearvanished"), timeout=120) + + # TODO https://jira.cfops.it/browse/TUN-6209 : Sign these packages. + def create_rpm_pkgs(self, artifacts_path): + self._setup_rpm_pkg_directories(artifacts_path) + subprocess.call(("createrepo", "./rpm"), timeout=120) + + """ + sets up the RPM directories in the following format: + - rpm + - aarch64 + - x86_64 + - 386 + + this assumes the assets are in the format -.rpm + """ + def _setup_rpm_pkg_directories(self, artifacts_path, archs=["aarch64", "x86_64", "386"]): + for arch in archs: + for root, _ , files in os.walk(artifacts_path): + for file in files: + if file.endswith(f"{arch}.rpm"): + new_dir = f"./rpm/{arch}" + os.makedirs(new_dir, exist_ok=True) + old_path = os.path.join(root, file) + new_path = os.path.join(new_dir, file) + shutil.copyfile(old_path, new_path) """ Walks through a directory and uploads it's assets to R2. @@ -167,6 +193,17 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary upload_from_directories(pkg_uploader, "dists", release_version, binary_name) upload_from_directories(pkg_uploader, "pool", release_version, binary_name) +def create_rpm_packaging(pkg_creator, pkg_uploader, artifacts_path, release_version, binary_name): + print(f"creating rpm pkgs...") + pkg_creator.create_rpm_pkgs(artifacts_path) + + print("uploading latest to r2...") + upload_from_directories(pkg_uploader, "rpm", None, binary_name) + + print(f"uploading versioned release {release_version} to r2...") + upload_from_directories(pkg_uploader, "rpm", release_version, binary_name) + + def parse_args(): parser = argparse.ArgumentParser( description="Creates linux releases and uploads them in a packaged format" @@ -223,3 +260,5 @@ if __name__ == "__main__": pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret) create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, args.gpg_key_id, args.binary, args.archs, "main", args.release_tag) + + create_rpm_packaging(pkg_creator, pkg_uploader, "./built_artifacts", args.release_tag, args.binary )