Merge branch 'cloudflare:master' into master

Makefile

@@ -249,6 +249,11 @@ gitlab-release:
 r2-linux-release:
 	python3 ./release_pkgs.py
 
+.PHONY: r2-next-linux-release
+# Publishes to a separate R2 repository during GPG key rollover, using dual-key signing.
+r2-next-linux-release:
+	python3 ./release_pkgs.py --upload-repo-file
+
 .PHONY: capnp
 capnp:
 	which capnp  # https://capnproto.org/install.html

cfsetup.yaml

@@ -211,7 +211,7 @@ bookworm: &bookworm
       - make github-release
   r2-linux-release:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &r2-linux-release-deps
       - *pinned_go
       - build-essential
       - fakeroot
@@ -231,4 +231,13 @@ bookworm: &bookworm
       - pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9
       - make r2-linux-release
 
+  r2-next-linux-release:
+    build_dir: *build_dir
+    builddeps: *r2-linux-release-deps
+    post-cache:
+      - python3 -m venv env
+      - . env/bin/activate
+      - pip install pynacl==1.4.0 pygithub==1.55 boto3==1.22.9 python-gnupg==0.4.9
+      - make r2-next-linux-release
+
 trixie: *bookworm

release_pkgs.py

@@ -133,7 +133,8 @@ class PkgCreator:
     """
 
     def create_repo_file(self, file_path, binary_name, baseurl, gpgkey_url):
-        with open(os.path.join(file_path, binary_name + '.repo'), "w+") as repo_file:
+        repo_file = os.path.join(file_path, binary_name + '.repo')
+        with open(repo_file, "w+") as repo_file:
             repo_file.write(f"[{binary_name}-stable]")
             repo_file.write(f"{binary_name}-stable")
             repo_file.write(f"baseurl={baseurl}/rpm")
@@ -141,8 +142,10 @@ class PkgCreator:
             repo_file.write("type=rpm")
             repo_file.write("gpgcheck=1")
             repo_file.write(f"gpgkey={gpgkey_url}")
+        return repo_file
 
-    def _sign_rpms(self, file_path):
+
+    def _sign_rpms(self, file_path, gpg_key_name):
         p = Popen(["rpm", "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
@@ -176,7 +179,7 @@ class PkgCreator:
                         old_path = os.path.join(root, file)
                         new_path = os.path.join(new_dir, file)
                         shutil.copyfile(old_path, new_path)
-                        self._sign_rpms(new_path)
+                        self._sign_rpms(new_path, gpg_key_name)
 
     """
         imports gpg keys into the system so reprepro and createrepo can use it to sign packages.
@@ -192,6 +195,18 @@ class PkgCreator:
         data = gpg.list_keys(secret=True)
         return (data[0]["fingerprint"], data[0]["uids"][0])
 
+    def import_multiple_gpg_keys(self, primary_private_key, primary_public_key, secondary_private_key=None, secondary_public_key=None):
+        """
+        Import one or two GPG keypairs. Returns a list of (fingerprint, uid) with the primary first.
+        """
+        results = []
+        if primary_private_key and primary_public_key:
+            results.append(self.import_gpg_keys(primary_private_key, primary_public_key))
+        if secondary_private_key and secondary_public_key:
+            # Ensure secondary is imported and appended
+            results.append(self.import_gpg_keys(secondary_private_key, secondary_public_key))
+        return results
+
     """
         basically rpm --import <key_file>
         This enables us to sign rpms.
@@ -247,11 +262,13 @@ def upload_from_directories(pkg_uploader, directory, release, binary):
 """
 
 
-def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component,
+def create_deb_packaging(pkg_creator, pkg_uploader, releases, primary_gpg_key_id, secondary_gpg_key_id, binary_name, archs, package_component,
                          release_version):
     # set configuration for package creation.
     print(f"initialising configuration for {binary_name} , {archs}")
     Path("./conf").mkdir(parents=True, exist_ok=True)
+    # If in rollover mode (secondary provided), tell reprepro to sign with both keys.
+    sign_with_ids = primary_gpg_key_id if not secondary_gpg_key_id else f"{primary_gpg_key_id} {secondary_gpg_key_id}"
     pkg_creator.create_distribution_conf(
         "./conf/distributions",
         binary_name,
@@ -260,7 +277,7 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary
         archs,
         package_component,
         f"apt repository for {binary_name}",
-        gpg_key_id)
+        sign_with_ids)
 
     # create deb pkgs
     for release in releases:
@@ -287,15 +304,19 @@ def create_rpm_packaging(
         gpg_key_name,
         base_url,
         gpg_key_url,
+        upload_repo_file=False,
 ):
     print(f"creating rpm pkgs...")
     pkg_creator.create_rpm_pkgs(artifacts_path, gpg_key_name)
-    pkg_creator.create_repo_file(artifacts_path, binary_name, base_url, gpg_key_url)
+    repo_file = pkg_creator.create_repo_file(artifacts_path, binary_name, base_url, gpg_key_url)
+
+    print("Uploading repo file")
+    pkg_uploader.upload_pkg_to_r2(repo_file, binary_name + "repo")
 
     print("uploading latest to r2...")
     upload_from_directories(pkg_uploader, "rpm", None, binary_name)
 
-    if release_version:
+    if upload_repo_file:
         print(f"uploading versioned release {release_version} to r2...")
         upload_from_directories(pkg_uploader, "rpm", release_version, binary_name)
 
@@ -336,11 +357,23 @@ def parse_args():
             signing packages"
     )
 
+    # Optional secondary keypair for key rollover
+    parser.add_argument(
+        "--gpg-private-key-2", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY_2"), help="Secondary GPG private key for rollover"
+    )
+    parser.add_argument(
+        "--gpg-public-key-2", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY_2"), help="Secondary GPG public key for rollover"
+    )
+
     parser.add_argument(
         "--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\
             downloaders can use to verify signing"
     )
 
+    parser.add_argument(
+        "--gpg-public-key-url-2", default=os.environ.get("GPG_PUBLIC_KEY_URL_2"), help="Secondary GPG public key url for rollover"
+    )
+
     parser.add_argument(
         "--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders"
     )
@@ -355,6 +388,10 @@ def parse_args():
             it is the caller's responsiblity to ensure that these debs are already present in a directory. This script\
             will not build binaries or create their debs."
     )
+
+    parser.add_argument(
+        "--upload-repo-file", action='store_true', help="Upload RPM repo file to R2"
+    )
     args = parser.parse_args()
 
     return args
@@ -368,13 +405,36 @@ if __name__ == "__main__":
         exit(1)
 
     pkg_creator = PkgCreator()
-    (gpg_key_id, gpg_key_name) = pkg_creator.import_gpg_keys(args.gpg_private_key, args.gpg_public_key)
+    # Import one or two keypairs; primary first
+    key_results = pkg_creator.import_multiple_gpg_keys(
+        args.gpg_private_key,
+        args.gpg_public_key,
+        args.gpg_private_key_2,
+        args.gpg_public_key_2,
+    )
+    if not key_results or len(key_results) == 0:
+        raise SystemExit("No GPG keys were provided for signing")
+    primary_gpg_key_id, primary_gpg_key_name = key_results[0]
+    secondary_gpg_key_id = None
+    secondary_gpg_key_name = None
+    if len(key_results) > 1:
+        secondary_gpg_key_id, secondary_gpg_key_name = key_results[1]
+    # Import RPM public keys (one or two)
     pkg_creator.import_rpm_key(args.gpg_public_key)
 
     pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
-    print(f"signing with gpg_key: {gpg_key_id}")
+    print(f"signing with primary gpg_key: {primary_gpg_key_id} and secondary gpg_key: {secondary_gpg_key_id}")
-    create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs,
+    create_deb_packaging(
-                         "main", args.release_tag)
+        pkg_creator,
+        pkg_uploader,
+        args.deb_based_releases,
+        primary_gpg_key_id,
+        secondary_gpg_key_id,
+        args.binary,
+        args.archs,
+        "main",
+        args.release_tag,
+    )
 
     create_rpm_packaging(
         pkg_creator,
@@ -382,7 +442,8 @@ if __name__ == "__main__":
         "./built_artifacts",
         args.release_tag,
         args.binary,
-        gpg_key_name,
+        primary_gpg_key_name,
-        args.gpg_public_key_url,
         args.pkg_upload_url,
+        args.gpg_public_key_url,
+        args.upload_repo_file,
     )