Merge branch 'cloudflare:master' into tunnel-health

This commit is contained in:
Mads Jon Nielsen 2024-07-22 16:47:55 +02:00 committed by GitHub
commit 037f056d0c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 163 additions and 200 deletions

View File

@ -12,8 +12,6 @@ export GO111MODULE=on
# build 'cloudflared-darwin-amd64.tgz'
mkdir -p artifacts
FILENAME="$(pwd)/artifacts/cloudflared-darwin-amd64.tgz"
PKGNAME="$(pwd)/artifacts/cloudflared-amd64.pkg"
TARGET_DIRECTORY=".build"
BINARY_NAME="cloudflared"
VERSION=$(git describe --tags --always --dirty="-dev")
@ -28,7 +26,6 @@ export PATH="$PATH:/usr/local/bin"
mkdir -p ../src/github.com/cloudflare/
cp -r . ../src/github.com/cloudflare/cloudflared
cd ../src/github.com/cloudflare/cloudflared
GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared
# Add code signing private key to the key chain
if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then
@ -138,47 +135,59 @@ else
fi
fi
# sign the cloudflared binary
if [[ ! -z "$CODE_SIGN_NAME" ]]; then
# cleanup the build directory because the previous execution might have failed without cleaning up.
rm -rf "${TARGET_DIRECTORY}"
archs=("amd64" "arm64")
export TARGET_OS=darwin
for arch in ${archs[@]}; do
FILENAME="$(pwd)/artifacts/cloudflared-darwin-$arch.tgz"
PKGNAME="$(pwd)/artifacts/cloudflared-$arch.pkg"
TARGET_ARCH=$arch GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared
# sign the cloudflared binary
if [[ ! -z "$CODE_SIGN_NAME" ]]; then
codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
# notarize the binary
# TODO: TUN-5789
fi
fi
# creating build directory
rm -rf $TARGET_DIRECTORY
mkdir "${TARGET_DIRECTORY}"
mkdir "${TARGET_DIRECTORY}/contents"
cp -r ".mac_resources/scripts" "${TARGET_DIRECTORY}/scripts"
ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${arch}-build"
# creating build directory
rm -rf $ARCH_TARGET_DIRECTORY
mkdir -p "${ARCH_TARGET_DIRECTORY}"
mkdir -p "${ARCH_TARGET_DIRECTORY}/contents"
cp -r ".mac_resources/scripts" "${ARCH_TARGET_DIRECTORY}/scripts"
# copy cloudflared into the build directory
cp ${BINARY_NAME} "${TARGET_DIRECTORY}/contents/${PRODUCT}"
# copy cloudflared into the build directory
cp ${BINARY_NAME} "${ARCH_TARGET_DIRECTORY}/contents/${PRODUCT}"
# compress cloudflared into a tar and gzipped file
tar czf "$FILENAME" "${BINARY_NAME}"
# compress cloudflared into a tar and gzipped file
tar czf "$FILENAME" "${BINARY_NAME}"
# build the installer package
if [[ ! -z "$PKG_SIGN_NAME" ]]; then
# build the installer package
if [[ ! -z "$PKG_SIGN_NAME" ]]; then
pkgbuild --identifier com.cloudflare.${PRODUCT} \
--version ${VERSION} \
--scripts ${TARGET_DIRECTORY}/scripts \
--root ${TARGET_DIRECTORY}/contents \
--scripts ${ARCH_TARGET_DIRECTORY}/scripts \
--root ${ARCH_TARGET_DIRECTORY}/contents \
--install-location /usr/local/bin \
--sign "${PKG_SIGN_NAME}" \
${PKGNAME}
# notarize the package
# TODO: TUN-5789
else
else
pkgbuild --identifier com.cloudflare.${PRODUCT} \
--version ${VERSION} \
--scripts ${TARGET_DIRECTORY}/scripts \
--root ${TARGET_DIRECTORY}/contents \
--scripts ${ARCH_TARGET_DIRECTORY}/scripts \
--root ${ARCH_TARGET_DIRECTORY}/contents \
--install-location /usr/local/bin \
${PKGNAME}
fi
fi
done
# cleaning up the build directory
rm -rf $TARGET_DIRECTORY
# cleanup build directory because this script is not ran within containers,
# which might lead to future issues in subsequent runs.
rm -rf "${TARGET_DIRECTORY}"

View File

@ -9,8 +9,7 @@ windowsArchs=("amd64" "386")
for arch in ${windowsArchs[@]}; do
export TARGET_ARCH=$arch
# Copy exe into final directory
cp ./artifacts/cloudflared-windows-$arch.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
cp ./artifacts/cloudflared-windows-$arch.exe ./cloudflared.exe
cp $ARTIFACT_DIR/cloudflared-windows-$arch.exe ./cloudflared.exe
make cloudflared-msi
# Copy msi into final directory
mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi

View File

@ -5,41 +5,6 @@ $ProgressPreference = "SilentlyContinue"
$WorkingDirectory = Get-Location
$CloudflaredDirectory = "$WorkingDirectory\go\src\github.com\cloudflare\cloudflared"
Write-Output "Installing python..."
$PythonVersion = "3.10.11"
$PythonZipFile = "$env:Temp\python-$PythonVersion-embed-amd64.zip"
$PipInstallFile = "$env:Temp\get-pip.py"
$PythonZipUrl = "https://www.python.org/ftp/python/$PythonVersion/python-$PythonVersion-embed-amd64.zip"
$PythonPath = "$WorkingDirectory\Python"
$PythonBinPath = "$PythonPath\python.exe"
# Download Python zip file
Invoke-WebRequest -Uri $PythonZipUrl -OutFile $PythonZipFile
# Download Python pip file
Invoke-WebRequest -Uri "https://bootstrap.pypa.io/get-pip.py" -OutFile $PipInstallFile
# Extract Python files
Expand-Archive $PythonZipFile -DestinationPath $PythonPath -Force
# Add Python to PATH
$env:Path = "$PythonPath\Scripts;$PythonPath;$($env:Path)"
Write-Output "Installed to $PythonPath"
# Install pip
& $PythonBinPath $PipInstallFile
# Add package paths in pythonXX._pth to unblock python -m pip
$PythonImportPathFile = "$PythonPath\python310._pth"
$ComponentTestsDir = "$CloudflaredDirectory\component-tests\"
@($ComponentTestsDir, "Lib\site-packages", $(Get-Content $PythonImportPathFile)) | Set-Content $PythonImportPathFile
# Test Python installation
& $PythonBinPath --version
& $PythonBinPath -m pip --version
go env
go version
@ -48,8 +13,8 @@ $env:CGO_ENABLED = 1
$env:TARGET_ARCH = "amd64"
$env:Path = "$Env:Temp\go\bin;$($env:Path)"
& $PythonBinPath --version
& $PythonBinPath -m pip --version
python --version
python -m pip --version
cd $CloudflaredDirectory
@ -72,11 +37,11 @@ if ($LASTEXITCODE -ne 0) { throw "Failed unit tests" }
Write-Output "Running component tests"
& $PythonBinPath -m pip install --upgrade -r component-tests/requirements.txt
& $PythonBinPath component-tests/setup.py --type create
& $PythonBinPath -m pytest component-tests -o log_cli=true --log-cli-level=INFO
python -m pip --disable-pip-version-check install --upgrade -r component-tests/requirements.txt
python component-tests/setup.py --type create
python -m pytest component-tests -o log_cli=true --log-cli-level=INFO
if ($LASTEXITCODE -ne 0) {
& $PythonBinPath component-tests/setup.py --type cleanup
python component-tests/setup.py --type cleanup
throw "Failed component tests"
}
& $PythonBinPath component-tests/setup.py --type cleanup
python component-tests/setup.py --type cleanup

View File

@ -218,38 +218,14 @@ cloudflared-pkg: cloudflared cloudflared.1
cloudflared-msi:
wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs
.PHONY: cloudflared-darwin-amd64.tgz
cloudflared-darwin-amd64.tgz: cloudflared
tar czf cloudflared-darwin-amd64.tgz cloudflared
rm cloudflared
.PHONY: github-release
github-release: cloudflared
python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION)
.PHONY: github-release-built-pkgs
github-release-built-pkgs:
github-release:
python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
.PHONY: release-pkgs-linux
release-pkgs-linux:
python3 ./release_pkgs.py
.PHONY: github-message
github-message:
python3 github_message.py --release-version $(VERSION)
.PHONY: github-mac-upload
github-mac-upload:
python3 github_release.py --path artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz
python3 github_release.py --path artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg
.PHONY: github-windows-upload
github-windows-upload:
python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.exe --release-version $(VERSION) --name cloudflared-windows-amd64.exe
python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.msi --release-version $(VERSION) --name cloudflared-windows-amd64.msi
python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe
python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi
.PHONY: r2-linux-release
r2-linux-release:
python3 ./release_pkgs.py
.PHONY: capnp
capnp:

View File

@ -1,3 +1,14 @@
2024.7.1
- 2024-07-16 TUN-8543: use -p flag to create intermediate directories
2024.7.0
- 2024-07-05 TUN-8520: add macos arm64 build
- 2024-07-05 TUN-8523: refactor makefile and cfsetup
- 2024-07-02 TUN-8504: Use pre-installed python version instead of downloading it on Windows builds
- 2024-06-26 TUN-8489: Add default noop logger for capnprpc
- 2024-06-25 TUN-8487: Add user-agent for quick-tunnel requests
- 2023-12-12 TUN-8057: cloudflared uses new PQ curve ID
2024.6.1
- 2024-06-12 TUN-8461: Don't log Failed to send session payload if the error is EOF
- 2024-06-07 TUN-8456: Update quic-go to 0.45 and collect mtu and congestion control metrics

View File

@ -3,34 +3,27 @@ pinned_go: &pinned_go go-boring=1.22.2-1
build_dir: &build_dir /cfsetup_build
default-flavor: bullseye
buster: &buster
build:
build-linux:
build_dir: *build_dir
builddeps: &build_deps
- *pinned_go
- build-essential
- gotest-to-teamcity
- fakeroot
- rubygem-fpm
- rpm
- libffi-dev
- reprepro
- createrepo
pre-cache: &build_pre_cache
- export GOCACHE=/cfsetup_build/.cache/go-build
- go install golang.org/x/tools/cmd/goimports@latest
post-cache:
# TODO: TUN-8126 this is temporary to make sure packages can be built before release
- ./build-packages.sh
# Build binary for component test
- GOOS=linux GOARCH=amd64 make cloudflared
build-fips:
build-linux-fips:
build_dir: *build_dir
builddeps: *build_deps
pre-cache: *build_pre_cache
post-cache:
- export FIPS=true
# TODO: TUN-8126 this is temporary to make sure packages can be built before release
- ./build-packages-fips.sh
# Build binary for component test
- GOOS=linux GOARCH=amd64 make cloudflared
cover:
@ -39,28 +32,21 @@ buster: &buster
pre-cache: *build_pre_cache
post-cache:
- make cover
# except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64)
github-release-pkgs:
# except FIPS and macos
build-linux-release:
build_dir: *build_dir
builddeps:
builddeps: &build_deps_release
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
- rpm
- wget
# libmsi and libgcab are libraries the wixl binary depends on.
- libmsi-dev
- libgcab-dev
- python3-dev
- libffi-dev
- python3-setuptools
- python3-dev
- python3-pip
- reprepro
- createrepo
pre-cache: &github_release_pkgs_pre_cache
- wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
- chmod a+x /usr/local/bin/wixl
- python3-setuptools
- wget
pre-cache: &build_release_pre_cache
- pip3 install pynacl==1.4.0
- pip3 install pygithub==1.55
- pip3 install boto3==1.22.9
@ -68,32 +54,14 @@ buster: &buster
post-cache:
# build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
- ./build-packages.sh
# release the packages built and moved to /cfsetup/built_artifacts
- make github-release-built-pkgs
# publish packages to linux repos
- make release-pkgs-linux
# handle FIPS separately so that we built with gofips compiler
github-fips-release-pkgs:
build-linux-fips-release:
build_dir: *build_dir
builddeps:
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
- rpm
- wget
# libmsi and libgcab are libraries the wixl binary depends on.
- libmsi-dev
- libgcab-dev
- python3-dev
- libffi-dev
- python3-setuptools
- python3-pip
pre-cache: *github_release_pkgs_pre_cache
builddeps: *build_deps_release
pre-cache: *build_release_pre_cache
post-cache:
# same logic as above, but for FIPS packages only
- ./build-packages-fips.sh
- make github-release-built-pkgs
generate-versions-file:
build_dir: *build_dir
builddeps:
@ -152,21 +120,7 @@ buster: &buster
- export GOOS=linux
- export GOARCH=arm64
- make cloudflared-deb
github-release-macos-amd64:
build_dir: *build_dir
builddeps: &build_pygithub
- *pinned_go
- build-essential
- python3-dev
- libffi-dev
- python3-setuptools
- python3-pip
pre-cache: &install_pygithub
- pip3 install pynacl==1.4.0
- pip3 install pygithub==1.55
post-cache:
- make github-mac-upload
github-release-windows:
package-windows:
build_dir: *build_dir
builddeps:
- *pinned_go
@ -186,10 +140,16 @@ buster: &buster
- pip3 install pygithub==1.55
post-cache:
- .teamcity/package-windows.sh
- make github-windows-upload
test:
build_dir: *build_dir
builddeps: *build_deps
builddeps: &build_deps_tests
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
- rpm
- libffi-dev
- gotest-to-teamcity
pre-cache: *build_pre_cache
post-cache:
- export GOOS=linux
@ -199,7 +159,7 @@ buster: &buster
- make test | gotest-to-teamcity
test-fips:
build_dir: *build_dir
builddeps: *build_deps
builddeps: *build_deps_tests
pre-cache: *build_pre_cache
post-cache:
- export GOOS=linux
@ -210,7 +170,7 @@ buster: &buster
- make test | gotest-to-teamcity
component-test:
build_dir: *build_dir
builddeps:
builddeps: &build_deps_component_test
- *pinned_go
- python3.7
- python3-pip
@ -230,24 +190,48 @@ buster: &buster
- python3 component-tests/setup.py --type cleanup
component-test-fips:
build_dir: *build_dir
builddeps:
- *pinned_go
- python3.7
- python3-pip
- python3-setuptools
# procps installs the ps command which is needed in test_sysv_service because the init script
# uses ps pid to determine if the agent is running
- procps
builddeps: *build_deps_component_test
pre-cache-copy-paths:
- component-tests/requirements.txt
pre-cache: *component_test_pre_cache
post-cache: *component_test_post_cache
github-message-release:
github-release:
build_dir: *build_dir
builddeps: *build_pygithub
pre-cache: *install_pygithub
builddeps:
- *pinned_go
- build-essential
- python3-dev
- libffi-dev
- python3-setuptools
- python3-pip
pre-cache:
- pip3 install pynacl==1.4.0
- pip3 install pygithub==1.55
post-cache:
- make github-release
- make github-message
r2-linux-release:
build_dir: *build_dir
builddeps:
- *pinned_go
- build-essential
- fakeroot
- rubygem-fpm
- rpm
- wget
- python3-dev
- libffi-dev
- python3-setuptools
- python3-pip
- reprepro
- createrepo
pre-cache:
- pip3 install pynacl==1.4.0
- pip3 install pygithub==1.55
- pip3 install boto3==1.22.9
- pip3 install python-gnupg==0.4.9
post-cache:
- make r2-linux-release
bullseye: *buster
bookworm: *buster

View File

@ -35,7 +35,13 @@ func RunQuickTunnel(sc *subcommandContext) error {
Timeout: httpTimeout,
}
resp, err := client.Post(fmt.Sprintf("%s/tunnel", sc.c.String("quick-service")), "application/json", nil)
req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/tunnel", sc.c.String("quick-service")), nil)
if err != nil {
return errors.Wrap(err, "failed to build quick tunnel request")
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("User-Agent", buildInfo.UserAgent())
resp, err := client.Do(req)
if err != nil {
return errors.Wrap(err, "failed to request quick Tunnel")
}

View File

@ -12,7 +12,7 @@ import (
// to https://pqtunnels.cloudflareresearch.com.
const (
PQKex = tls.CurveID(0xfe31) // X25519Kyber768Draft00
PQKex = tls.CurveID(0x6399) // X25519Kyber768Draft00
PQKexName = "X25519Kyber768Draft00"
)

View File

@ -32,7 +32,7 @@ func NewCloudflaredClient(ctx context.Context, stream io.ReadWriteCloser, reques
return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(rpcStreamProtocolSignature), n)
}
transport := tunnelrpc.SafeTransport(stream)
conn := rpc.NewConn(transport)
conn := tunnelrpc.NewClientConn(transport)
client := pogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn)
return &CloudflaredClient{
client: client,

View File

@ -6,8 +6,6 @@ import (
"io"
"time"
"zombiezen.com/go/capnproto2/rpc"
"github.com/cloudflare/cloudflared/tunnelrpc"
"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
)
@ -58,7 +56,7 @@ func (s *CloudflaredServer) handleRPC(ctx context.Context, stream io.ReadWriteCl
defer transport.Close()
main := pogs.CloudflaredServer_ServerToClient(s.sessionManager, s.configManager)
rpcConn := rpc.NewConn(transport, rpc.MainInterface(main.Client))
rpcConn := tunnelrpc.NewServerConn(transport, main.Client)
defer rpcConn.Close()
// We ignore the errors here because if cloudflared fails to handle a request, we will just move on.

View File

@ -31,7 +31,7 @@ func NewSessionClient(ctx context.Context, stream io.ReadWriteCloser, requestTim
return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(rpcStreamProtocolSignature), n)
}
transport := tunnelrpc.SafeTransport(stream)
conn := rpc.NewConn(transport)
conn := tunnelrpc.NewClientConn(transport)
return &SessionClient{
client: pogs.NewSessionManager_PogsClient(conn.Bootstrap(ctx), conn),
transport: transport,

View File

@ -6,8 +6,6 @@ import (
"io"
"time"
"zombiezen.com/go/capnproto2/rpc"
"github.com/cloudflare/cloudflared/tunnelrpc"
"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
)
@ -48,7 +46,7 @@ func (s *SessionManagerServer) Serve(ctx context.Context, stream io.ReadWriteClo
defer transport.Close()
main := pogs.SessionManager_ServerToClient(s.sessionManager)
rpcConn := rpc.NewConn(transport, rpc.MainInterface(main.Client))
rpcConn := tunnelrpc.NewServerConn(transport, main.Client)
defer rpcConn.Close()
select {

View File

@ -35,7 +35,7 @@ type registrationClient struct {
func NewRegistrationClient(ctx context.Context, stream io.ReadWriteCloser, requestTimeout time.Duration) RegistrationClient {
transport := SafeTransport(stream)
conn := rpc.NewConn(transport)
conn := NewClientConn(transport)
client := pogs.NewRegistrationServer_PogsClient(conn.Bootstrap(ctx), conn)
return &registrationClient{
client: client,

View File

@ -4,8 +4,6 @@ import (
"context"
"io"
"zombiezen.com/go/capnproto2/rpc"
"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
)
@ -28,8 +26,7 @@ func (s *RegistrationServer) Serve(ctx context.Context, stream io.ReadWriteClose
defer transport.Close()
main := pogs.RegistrationServer_ServerToClient(s.registrationServer)
rpcConn := rpc.NewConn(transport, rpc.MainInterface(main.Client))
defer rpcConn.Close()
rpcConn := NewServerConn(transport, main.Client)
select {
case <-rpcConn.Done():

View File

@ -1,10 +1,12 @@
package tunnelrpc
import (
"context"
"io"
"time"
"github.com/pkg/errors"
capnp "zombiezen.com/go/capnproto2"
"zombiezen.com/go/capnproto2/rpc"
)
@ -67,3 +69,21 @@ func isTemporaryError(e error) bool {
t, ok := e.(temp)
return ok && t.Temporary()
}
// NoopCapnpLogger provides a logger to discard all capnp rpc internal logging messages as
// they are by default provided to stdout if no logger interface is provided. These logging
// messages in cloudflared have typically not provided a high amount of pratical value
// as the messages are extremely verbose and don't provide a good insight into the message
// contents or rpc method names.
type noopCapnpLogger struct{}
func (noopCapnpLogger) Infof(ctx context.Context, format string, args ...interface{}) {}
func (noopCapnpLogger) Errorf(ctx context.Context, format string, args ...interface{}) {}
func NewClientConn(transport rpc.Transport) *rpc.Conn {
return rpc.NewConn(transport, rpc.ConnLog(noopCapnpLogger{}))
}
func NewServerConn(transport rpc.Transport, client capnp.Client) *rpc.Conn {
return rpc.NewConn(transport, rpc.MainInterface(client), rpc.ConnLog(noopCapnpLogger{}))
}