TUN-5943: Add RPM support

This PR extends release_pkgs.py to now also support uploading rpm based
assets to R2. The packages are not signed yet  and will be done in a
subsequent PR.

This PR
- Packs the .rpm assets into relevant directories
- Calls createrepo on them to make them yum repo ready
- Uploads them to R2
This commit is contained in:
Sudarsan Reddy 2022-05-12 08:32:09 +01:00
parent f7fd4ea71c
commit 4b6437cc60
3 changed files with 51 additions and 3 deletions

View File

@ -261,6 +261,10 @@ github-release: cloudflared
github-release-built-pkgs: github-release-built-pkgs:
python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
.PHONY: release-pkgs-linux
release-pkgs-linux:
python3 ./release_pkgs.py
.PHONY: github-message .PHONY: github-message
github-message: github-message:
python3 github_message.py --release-version $(VERSION) python3 github_message.py --release-version $(VERSION)

View File

@ -40,14 +40,19 @@ stretch: &stretch
- libffi-dev - libffi-dev
- python3-setuptools - python3-setuptools
- python3-pip - python3-pip
- reprepro
- createrepo
pre-cache: &github_release_pkgs_pre_cache pre-cache: &github_release_pkgs_pre_cache
- wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
- chmod a+x /usr/local/bin/wixl - chmod a+x /usr/local/bin/wixl
- pip3 install pynacl==1.4.0 - pip3 install pynacl==1.4.0
- pip3 install pygithub==1.55 - pip3 install pygithub==1.55
- pip3 install boto3==1.22.9
post-cache: post-cache:
# build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
- ./build-packages.sh - ./build-packages.sh
# publish packages to linux repos
# TODO TUN-6180: Uncomment this - make release-pkgs-linux
# release the packages built and moved to /cfsetup/built_artifacts # release the packages built and moved to /cfsetup/built_artifacts
- make github-release-built-pkgs - make github-release-built-pkgs
# handle FIPS separately so that we built with gofips compiler # handle FIPS separately so that we built with gofips compiler
@ -255,4 +260,4 @@ centos-7:
- export PATH=$PATH:/usr/local/go/bin - export PATH=$PATH:/usr/local/go/bin
- export GOOS=linux - export GOOS=linux
- export GOARCH=amd64 - export GOARCH=amd64
- make publish-rpm - make publish-rpm

View File

@ -12,6 +12,7 @@ import subprocess
import os import os
import argparse import argparse
import logging import logging
import shutil
from hashlib import sha256 from hashlib import sha256
import boto3 import boto3
@ -100,13 +101,38 @@ class PkgCreator:
""" """
def create_deb_pkgs(self, release, deb_file): def create_deb_pkgs(self, release, deb_file):
self._clean_build_resources() self._clean_build_resources()
subprocess.call(("reprepro", "includedeb", release, deb_file)) subprocess.call(("reprepro", "includedeb", release, deb_file), timeout=120)
""" """
This is mostly useful to clear previously built db, dist and pool resources. This is mostly useful to clear previously built db, dist and pool resources.
""" """
def _clean_build_resources(self): def _clean_build_resources(self):
subprocess.call(("reprepro", "clearvanished")) subprocess.call(("reprepro", "clearvanished"), timeout=120)
# TODO https://jira.cfops.it/browse/TUN-6209 : Sign these packages.
def create_rpm_pkgs(self, artifacts_path):
self._setup_rpm_pkg_directories(artifacts_path)
subprocess.call(("createrepo", "./rpm"), timeout=120)
"""
sets up the RPM directories in the following format:
- rpm
- aarch64
- x86_64
- 386
this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm
"""
def _setup_rpm_pkg_directories(self, artifacts_path, archs=["aarch64", "x86_64", "386"]):
for arch in archs:
for root, _ , files in os.walk(artifacts_path):
for file in files:
if file.endswith(f"{arch}.rpm"):
new_dir = f"./rpm/{arch}"
os.makedirs(new_dir, exist_ok=True)
old_path = os.path.join(root, file)
new_path = os.path.join(new_dir, file)
shutil.copyfile(old_path, new_path)
""" """
Walks through a directory and uploads it's assets to R2. Walks through a directory and uploads it's assets to R2.
@ -167,6 +193,17 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary
upload_from_directories(pkg_uploader, "dists", release_version, binary_name) upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
upload_from_directories(pkg_uploader, "pool", release_version, binary_name) upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
def create_rpm_packaging(pkg_creator, pkg_uploader, artifacts_path, release_version, binary_name):
print(f"creating rpm pkgs...")
pkg_creator.create_rpm_pkgs(artifacts_path)
print("uploading latest to r2...")
upload_from_directories(pkg_uploader, "rpm", None, binary_name)
print(f"uploading versioned release {release_version} to r2...")
upload_from_directories(pkg_uploader, "rpm", release_version, binary_name)
def parse_args(): def parse_args():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Creates linux releases and uploads them in a packaged format" description="Creates linux releases and uploads them in a packaged format"
@ -223,3 +260,5 @@ if __name__ == "__main__":
pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret) pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, args.gpg_key_id, args.binary, create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, args.gpg_key_id, args.binary,
args.archs, "main", args.release_tag) args.archs, "main", args.release_tag)
create_rpm_packaging(pkg_creator, pkg_uploader, "./built_artifacts", args.release_tag, args.binary )