TUN-4771: Upload deb, rpm and msi packages to github

- cfsetup now has a build command `github-release-pkgs` to release linux
   and msi packages to github.
 - github_message.py now has an option to upload all assets in a provided
   directory.

Makefile

@@ -239,6 +239,10 @@ release: bin/equinox
 github-release: cloudflared
 	python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION)
 
+.PHONY: github-release-built-pkgs
+github-release-built-pkgs:
+	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
+
 .PHONY: github-message
 github-message:
 	python3 github_message.py --release-version $(VERSION)

cfsetup.yaml

@@ -27,10 +27,35 @@ stretch: &stretch
       - libmsi-dev
       - libgcab-dev
     pre-cache:
+      # TODO: https://jira.cfops.it/browse/TUN-4792 Replace this wixl with the official one once msitools supports
+      # environment.
       - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
       - chmod a+x /usr/local/bin/wixl
     post-cache:
       - ./build-packages.sh
+  github-release-pkgs:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go_fips
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+      - python3-setuptools
+      - python3-pip
+    pre-cache:
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+      - pip3 install pygithub
+    post-cache:
+      # build all packages and move them to /cfsetup/built_artifacts
+      - ./build-packages.sh
+      # release the packages built and moved to /cfsetup/built_artifacts
+      - make github-release-built-pkgs
   build-deb:
     build_dir: *build_dir
     builddeps: &build_deb_deps

github_release.py

@@ -10,6 +10,9 @@ import shutil
 import hashlib
 import requests
 import tarfile
+from os import listdir
+from os.path import isfile, join
+import re
 
 from github import Github, GithubException, UnknownObjectException
 
@@ -135,7 +138,7 @@ def parse_args():
         logging.error("Missing asset path")
         is_valid = False
 
-    if not args.name:
+    if not args.name and not os.path.isdir(args.path):
         logging.error("Missing asset name")
         is_valid = False
 
@@ -161,6 +164,57 @@ def parse_args():
     parser.print_usage()
     exit(1)
 
+def upload_asset(release, filepath, filename, release_version, kv_account_id, namespace_id, kv_api_token):
+    logging.info(f"Uploading asset: {filename}")
+    release.upload_asset(filepath, name=filename)
+
+    # check and extract if the file is a tar and gzipped file (as is the case with the macos builds)
+    binary_path = filepath
+    if binary_path.endswith("tgz"):
+        try:
+            shutil.rmtree('cfd')
+        except OSError:
+            pass
+        zipfile = tarfile.open(binary_path, "r:gz")
+        zipfile.extractall('cfd') # specify which folder to extract to
+        zipfile.close()
+
+        binary_path = os.path.join(os.getcwd(), 'cfd', 'cloudflared')
+
+    # send the sha256 (the checksum) to workers kv
+    pkg_hash = get_sha256(binary_path)
+    send_hash(pkg_hash, filename, release_version, kv_account_id, namespace_id, kv_api_token)
+
+    # create the artifacts directory if it doesn't exist
+    artifact_path = os.path.join(os.getcwd(), 'artifacts')
+    if not os.path.isdir(artifact_path):
+        os.mkdir(artifact_path)
+
+    # copy the binary to the path
+    copy_path = os.path.join(artifact_path, filename)
+    try:
+        shutil.copy(filepath, copy_path)
+    except shutil.SameFileError:
+        pass # the macOS release copy fails with being the same file (already in the artifacts directory)
+
+def compile_regex() :
+    # The capture groups are:
+    # 1: the text cloudflared
+    # 2: compiled architecture
+    # 3: possible extensions
+    capture_string = r'(cloudflared).+(amd64|x86_64|386|arm)(\.msi|\.deb|\.rpm|\.exe)?'
+    return re.compile(capture_string)
+
+def extract_filename(regex, filename):
+    match_groups = regex.match(filename)
+    os = "linux"
+    name = match_groups.group(1)
+    arch = match_groups.group(2)
+    ext = match_groups.group(3)
+    if ext in ['.msi', '.exe']:
+        os = "windows"
+    return "{}-{}-{}{}".format(name, os, arch, ext if ext is not None else "")
+
 
 def main():
     """ Attempts to upload Asset to Github Release. Creates Release if it doesnt exist """
@@ -174,40 +228,20 @@ def main():
             logging.info("Skipping asset upload because of dry-run")
             return
 
-        release.upload_asset(args.path, name=args.name)
+        if os.path.isdir(args.path):
-
+            regexp = compile_regex()
-        # check and extract if the file is a tar and gzipped file (as is the case with the macos builds)
+            onlyfiles = [f for f in listdir(args.path) if isfile(join(args.path, f))]
-        binary_path = args.path
+            for filename in onlyfiles:
-        if binary_path.endswith("tgz"):
+                binary_path = os.path.join(args.path, filename)
-            try:
+                filename = extract_filename(regexp, filename)
-                shutil.rmtree('cfd') 
+                upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id,
-            except OSError as e:
+                args.kv_api_token)
-                pass
+        else:
-            zipfile = tarfile.open(binary_path, "r:gz")
+            upload_asset(release, args.path, args.name, args.release_version, args.kv_account_id, args.namespace_id,
-            zipfile.extractall('cfd') # specify which folder to extract to
+                args.kv_api_token)
-            zipfile.close()
-
-            binary_path = os.path.join(os.getcwd(), 'cfd', 'cloudflared') 
-
-        # send the sha256 (the checksum) to workers kv
-        pkg_hash = get_sha256(binary_path)
-        send_hash(pkg_hash, args.name, args.release_version, args.kv_account_id, args.namespace_id, args.kv_api_token)
-
-        # create the artifacts directory if it doesn't exist 
-        artifact_path = os.path.join(os.getcwd(), 'artifacts') 
-        if not os.path.isdir(artifact_path):
-            os.mkdir(artifact_path)
-
-        # copy the binary to the path
-        copy_path = os.path.join(artifact_path, args.name)
-        try:
-            shutil.copy(args.path, copy_path)
-        except shutil.SameFileError:
-            pass # the macOS release copy fails with being the same file (already in the artifacts directory). Catching to ignore.
 
     except Exception as e:
         logging.exception(e)
         exit(1)
 
-
 main()