TUN-5551: Reintroduce FIPS compliance for linux amd64 now as separate binaries

This is a cherry-pick of 157f5d1412
followed by build/CI changes so that amd64/linux FIPS compliance is
provided by new/separate binaries/artifacts/packages.

The reasoning being that FIPS compliance places excessive requirements
in the encryption algorithms used for regular users that do not care
about that. This can cause cloudflared to reject HTTPS origins that
would otherwise be accepted without FIPS checks.

This way, by having separate binaries, existing ones remain as they
were, and only FIPS-needy users will opt-in to the new FIPS binaries.

Makefile

@@ -4,22 +4,33 @@ MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut
 #e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
 
 ifeq ($(FIPS), true)
-	GO_BUILD_TAGS := $(GO_BUILD_TAGS) fips
+	BINARY_NAME := cloudflared-fips
-endif
+else
-
+	BINARY_NAME := cloudflared
-ifneq ($(GO_BUILD_TAGS),)
-	GO_BUILD_TAGS := -tags $(GO_BUILD_TAGS)
 endif
 
 ifeq ($(NIGHTLY), true)
+	# We do not release FIPS in NIGHTLY, so no need to consider that case here.
 	DEB_PACKAGE_NAME := cloudflared-nightly
 	NIGHTLY_FLAGS := --conflicts cloudflared --replaces cloudflared
 else
-	DEB_PACKAGE_NAME := cloudflared
+	DEB_PACKAGE_NAME := $(BINARY_NAME)
 endif
 
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
-VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
+VERSION_FLAGS := -X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"
+
+LINK_FLAGS :=
+ifeq ($(FIPS), true)
+	LINK_FLAGS := -linkmode=external -extldflags=-static $(LINK_FLAGS)
+	# Prevent linking with libc regardless of CGO enabled or not.
+	GO_BUILD_TAGS := $(GO_BUILD_TAGS) osusergo netgo fips
+endif
+
+LDFLAGS := -ldflags='$(VERSION_FLAGS) $(LINK_FLAGS)'
+ifneq ($(GO_BUILD_TAGS),)
+	GO_BUILD_TAGS := -tags "$(GO_BUILD_TAGS)"
+endif
 
 IMPORT_PATH   := github.com/cloudflare/cloudflared
 PACKAGE_DIR   := $(CURDIR)/packaging
@@ -61,9 +72,9 @@ else
 endif
 
 ifeq ($(TARGET_OS), windows)
-	EXECUTABLE_PATH=./cloudflared.exe
+	EXECUTABLE_PATH=./$(BINARY_NAME).exe
 else
-	EXECUTABLE_PATH=./cloudflared
+	EXECUTABLE_PATH=./$(BINARY_NAME)
 endif
 
 ifeq ($(FLAVOR), centos-7)
@@ -83,14 +94,12 @@ clean:
 cloudflared:
 ifeq ($(FIPS), true)
 	$(info Building cloudflared with go-fips)
-	-test -f fips/fips.go && mv fips/fips.go fips/fips.go.linux-amd64
+	cp -f fips/fips.go.linux-amd64 cmd/cloudflared/fips.go
-	mv fips/fips.go.linux-amd64 fips/fips.go
 endif
-
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
-
 ifeq ($(FIPS), true)
-	mv fips/fips.go fips/fips.go.linux-amd64
+	rm -f cmd/cloudflared/fips.go
+	./check-fips.sh cloudflared
 endif
 
 .PHONY: container
@@ -100,10 +109,10 @@ container:
 .PHONY: test
 test: vet
 ifndef CI
-	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+	go test -v -mod=vendor -race $(LDFLAGS) ./...
 else
 	@mkdir -p .cover
-	go test -v -mod=vendor -race $(VERSION_FLAGS) -coverprofile=".cover/c.out" ./...
+	go test -v -mod=vendor -race $(LDFLAGS) -coverprofile=".cover/c.out" ./...
 	go tool cover -html ".cover/c.out" -o .cover/all.html
 endif
 
@@ -112,10 +121,10 @@ test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 define publish_package
-	chmod 664 cloudflared*.$(1); \
+	chmod 664 $(BINARY_NAME)*.$(1); \
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t ecdsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -p -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
+		scp -p -4 $(BINARY_NAME)*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/$(BINARY_NAME)/; \
 	done
 endef
 
@@ -127,6 +136,8 @@ publish-deb: cloudflared-deb
 publish-rpm: cloudflared-rpm
 	$(call publish_package,rpm,yum)
 
+# When we build packages, the package name will be FIPS-aware.
+# But we keep the binary installed by it to be named "cloudflared" regardless.
 define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared

build-packages-fips.sh

@@ -0,0 +1,25 @@
+VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
+echo $VERSION
+
+# This controls the directory the built artifacts go into
+export ARTIFACT_DIR=built_artifacts/
+mkdir -p $ARTIFACT_DIR
+
+arch=("amd64")
+export TARGET_ARCH=$arch
+export TARGET_OS=linux
+export FIPS=true
+# For BoringCrypto to link, we need CGO enabled. Otherwise compilation fails.
+export CGO_ENABLED=1
+
+make cloudflared-deb
+mv cloudflared-fips\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-fips-linux-$arch.deb
+
+# rpm packages invert the - and _ and use x86_64 instead of amd64.
+RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g')
+RPMARCH="x86_64"
+make cloudflared-rpm
+mv cloudflared-fips-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-fips-linux-$RPMARCH.rpm
+
+# finally move the linux binary as well.
+mv ./cloudflared $ARTIFACT_DIR/cloudflared-fips-linux-$arch

build-packages.sh

@@ -1,6 +1,9 @@
-VERSION=$(git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
+VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
 echo $VERSION
+
+# Avoid depending on C code since we don't need it.
 export CGO_ENABLED=0
+
 # This controls the directory the built artifacts go into
 export ARTIFACT_DIR=built_artifacts/
 mkdir -p $ARTIFACT_DIR
@@ -14,8 +17,7 @@ for arch in ${windowsArchs[@]}; do
 done
 
 
-export FIPS=true
+linuxArchs=("386" "amd64" "arm" "arm64")
-linuxArchs=("amd64" "386" "arm" "arm64")
 export TARGET_OS=linux
 for arch in ${linuxArchs[@]}; do
     export TARGET_ARCH=$arch
@@ -37,4 +39,3 @@ for arch in ${linuxArchs[@]}; do
     # finally move the linux binary as well.
     mv ./cloudflared $ARTIFACT_DIR/cloudflared-linux-$arch
 done
-

cfsetup.yaml

@@ -1,20 +1,10 @@
 pinned_go: &pinned_go go=1.17-1
-pinned_go_fips: &pinned_go_fips go-boring=1.16.6-6
+pinned_go_fips: &pinned_go_fips go-boring=1.16.6-7
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: buster
 stretch: &stretch
   build:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go_fips
-      - build-essential
-    post-cache:
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - export FIPS=true
-      - make cloudflared
-  build-non-fips:  # helpful to catch problems with non-fips (only used for releasing non-linux artifacts) before releases
     build_dir: *build_dir
     builddeps:
       - *pinned_go
@@ -23,11 +13,22 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
-  build-all-packages: #except osxpkg
+  build-fips:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
       - build-essential
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - export FIPS=true
+      - make cloudflared
+  # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64)
+  github-release-pkgs:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
       - fakeroot
       - rubygem-fpm
       - rpm
@@ -35,15 +36,21 @@ stretch: &stretch
       # libmsi and libgcab are libraries the wixl binary depends on.
       - libmsi-dev
       - libgcab-dev
-    pre-cache:
+      - python3-dev
-      # TODO: https://jira.cfops.it/browse/TUN-4792 Replace this wixl with the official one once msitools supports
+      - libffi-dev
-      # environment.
+      - python3-setuptools
+      - python3-pip
+    pre-cache: &github_release_pkgs_pre_cache
       - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
       - chmod a+x /usr/local/bin/wixl
+      - pip3 install pygithub
     post-cache:
-      - export FIPS=true
+      # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
       - ./build-packages.sh
-  github-release-pkgs:
+      # release the packages built and moved to /cfsetup/built_artifacts
+      - make github-release-built-pkgs
+  # handle FIPS separately so that we built with gofips compiler
+  github-fips-release-pkgs:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
@@ -59,18 +66,25 @@ stretch: &stretch
       - libffi-dev
       - python3-setuptools
       - python3-pip
-    pre-cache:
+    pre-cache: *github_release_pkgs_pre_cache
-      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
-      - chmod a+x /usr/local/bin/wixl
-      - pip3 install pygithub
     post-cache:
-      # build all packages and move them to /cfsetup/built_artifacts
+      # same logic as above, but for FIPS packages only
-      - ./build-packages.sh
+      - ./build-packages-fips.sh
-      # release the packages built and moved to /cfsetup/built_artifacts
       - make github-release-built-pkgs
   build-deb:
     build_dir: *build_dir
     builddeps: &build_deb_deps
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - make cloudflared-deb
+  build-fips-deb:
+    build_dir: *build_dir
+    builddeps:
       - *pinned_go_fips
       - build-essential
       - fakeroot
@@ -86,7 +100,6 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
-      - export FIPS=true
       - export NIGHTLY=true
       - make cloudflared-deb
   build-deb-arm64:
@@ -99,7 +112,7 @@ stretch: &stretch
   publish-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go_fips
+      - *pinned_go
       - build-essential
       - fakeroot
       - rubygem-fpm
@@ -107,7 +120,6 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
-      - export FIPS=true
       - make publish-deb
   github-release-macos-amd64:
     build_dir: *build_dir
@@ -123,14 +135,27 @@ stretch: &stretch
     post-cache:
       - make github-mac-upload
   test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - gotest-to-teamcity
+    pre-cache: &test_pre_cache
+      - go get golang.org/x/tools/cmd/goimports
+      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - export PATH="$HOME/go/bin:$PATH"
+      - ./fmt-check.sh
+      - make test | gotest-to-teamcity
+  test-fips:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
       - build-essential
       - gotest-to-teamcity
-    pre-cache:
+    pre-cache: *test_pre_cache
-      - go get golang.org/x/tools/cmd/goimports
-      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64

check-fips.sh

@@ -0,0 +1,15 @@
+# Pass the path to the executable to check for FIPS compliance
+exe=$1
+
+if [ "$(go tool nm "${exe}" | grep -c '_Cfunc__goboringcrypto_')" -eq 0 ]; then
+    # Asserts that executable is using FIPS-compliant boringcrypto
+    echo "${exe}: missing goboring symbols" >&2
+    exit 1
+fi
+if [ "$(go tool nm "${exe}" | grep -c 'crypto/internal/boring/sig.FIPSOnly')" -eq 0 ]; then
+    # Asserts that executable is using FIPS-only schemes
+    echo "${exe}: missing fipsonly symbols" >&2
+    exit 1
+fi
+
+echo "${exe} is FIPS-compliant"

cmd/cloudflared/generic_service.go

@@ -1,3 +1,4 @@
+//go:build !windows && !darwin && !linux
 // +build !windows,!darwin,!linux
 
 package main

cmd/cloudflared/linux_service.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package main

cmd/cloudflared/macos_service.go

@@ -1,3 +1,4 @@
+//go:build darwin
 // +build darwin
 
 package main

cmd/cloudflared/tunnel/configuration_test.go

@@ -1,4 +1,6 @@
+//go:build ignore
 // +build ignore
+
 // TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
 
 package tunnel

cmd/cloudflared/tunnel/signal_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package tunnel

cmd/cloudflared/updater/workers_service_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package updater

cmd/cloudflared/windows_service.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package main

h2mux/h2_compressor_brotli.go

@@ -1,3 +1,4 @@
+//go:build cgo
 // +build cgo
 
 package h2mux

h2mux/h2_compressor_none.go

@@ -1,3 +1,4 @@
+//go:build !cgo
 // +build !cgo
 
 package h2mux

origin/proxy_posix_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package origin

sshgen/sshgen_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package sshgen

token/launch_browser_darwin.go

@@ -1,3 +1,4 @@
+//go:build darwin
 // +build darwin
 
 package token

token/launch_browser_other.go

@@ -1,3 +1,4 @@
+//go:build !windows && !darwin && !linux && !netbsd && !freebsd && !openbsd
 // +build !windows,!darwin,!linux,!netbsd,!freebsd,!openbsd
 
 package token

token/launch_browser_unix.go

@@ -1,3 +1,4 @@
+//go:build linux || freebsd || openbsd || netbsd
 // +build linux freebsd openbsd netbsd
 
 package token

token/launch_browser_windows.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package token

token/token_test.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package token

watcher/file_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package watcher