Compare commits

...

5 Commits

Author SHA1 Message Date
Adam Chalmers 42ac30f2fb Release 2020.9.2 2020-09-23 15:30:35 -05:00
Adam Chalmers c065fae792 TUN-3410: Request the v1 Tunnelstore API 2020-09-23 20:24:53 +00:00
Lee Valentine 4117162109 TRAFFIC-448: build cloudflare for junos and publish to s3 2020-09-23 18:04:01 +00:00
Bojan Zelic fe554668e5 updater service exit code should be 11 2020-09-23 18:02:18 +04:00
Dalton 691e86a564 AUTH-3109 upload the checksum to workers kv on github releases 2020-09-22 19:43:30 +00:00
9 changed files with 372 additions and 37 deletions

2
.gitignore vendored
View File

@ -5,6 +5,7 @@ guide/public
/.GOPATH
/bin
.idea
.build
.vscode
\#*\#
cscope.*
@ -12,6 +13,7 @@ cloudflared
cloudflared.pkg
cloudflared.exe
cloudflared.msi
cloudflared-x86-64*
!cmd/cloudflared/
.DS_Store
*-session.log

View File

@ -2,7 +2,7 @@ VERSION := $(shell git describe --tags --always --dirty="-dev" --match "[0
DATE := $(shell date -u '+%Y-%m-%d-%H%M UTC')
VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
MSI_VERSION := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut -c2-)
#MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
#MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
#e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
IMPORT_PATH := github.com/cloudflare/cloudflared
@ -44,6 +44,8 @@ else ifeq ($(LOCAL_OS),darwin)
TARGET_OS ?= darwin
else ifeq ($(LOCAL_OS),windows)
TARGET_OS ?= windows
else ifeq ($(LOCAL_OS),freebsd)
TARGET_OS ?= freebsd
else
$(error This system's OS $(LOCAL_OS) isn't supported)
endif
@ -125,6 +127,58 @@ cloudflared-darwin-amd64.tgz: cloudflared
tar czf cloudflared-darwin-amd64.tgz cloudflared
rm cloudflared
.PHONY: cloudflared-junos
cloudflared-junos: cloudflared jetez-certificate.pem jetez-key.pem
jetez --source . \
-j jet.yaml \
--key jetez-key.pem \
--cert jetez-certificate.pem \
--version $(VERSION)
rm jetez-*.pem
jetez-certificate.pem:
ifndef JETEZ_CERT
$(error JETEZ_CERT not defined)
endif
@echo "Writing JetEZ certificate"
@echo "$$JETEZ_CERT" > jetez-certificate.pem
jetez-key.pem:
ifndef JETEZ_KEY
$(error JETEZ_KEY not defined)
endif
@echo "Writing JetEZ key"
@echo "$$JETEZ_KEY" > jetez-key.pem
.PHONY: publish-cloudflared-junos
publish-cloudflared-junos: cloudflared-junos cloudflared-x86-64.latest.s3
ifndef S3_ENDPOINT
$(error S3_HOST not defined)
endif
ifndef S3_URI
$(error S3_URI not defined)
endif
ifndef S3_ACCESS_KEY
$(error S3_ACCESS_KEY not defined)
endif
ifndef S3_SECRET_KEY
$(error S3_SECRET_KEY not defined)
endif
sha256sum cloudflared-x86-64-$(VERSION).tgz | awk '{printf $$1}' > cloudflared-x86-64-$(VERSION).tgz.shasum
s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
put cloudflared-x86-64-$(VERSION).tgz $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz
s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
put cloudflared-x86-64-$(VERSION).tgz.shasum $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz.shasum
dpkg --compare-versions "$(VERSION)" gt "$(shell cat cloudflared-x86-64.latest.s3)" && \
echo -n "$(VERSION)" > cloudflared-x86-64.latest && \
s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
put cloudflared-x86-64.latest $(S3_URI)/cloudflared-x86-64.latest || \
echo "Latest version not updated"
cloudflared-x86-64.latest.s3:
s4cmd --endpoint-url $(S3_ENDPOINT) --force \
get $(S3_URI)/cloudflared-x86-64.latest cloudflared-x86-64.latest.s3
.PHONY: homebrew-upload
homebrew-upload: cloudflared-darwin-amd64.tgz
aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $$^ $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz

View File

@ -1,3 +1,25 @@
2020.9.2
- 2020-09-22 TRAFFIC-448: build cloudflare for junos and publish to s3
- 2020-09-22 TUN-3410: Request the v1 Tunnelstore API
- 2020-09-17 AUTH-3103 CI build fixes
- 2020-09-18 AUTH-3110-use-cfsetup-precache
- 2020-09-17 TUN-3295: Show route command results
- 2020-09-16 TUN-3291: cloudflared tunnel run -h explains how to use flags from parent command
- 2020-09-18 AUTH-3109 upload the checksum to workers kv on github releases
- 2020-09-17 updater service exit code should be 11
- 2020-09-01 TUN-3216: UI improvements
- 2020-08-25 Rebased and passed TunnelEventChan to LogServerInfo in new ReconnectTunnel function
- 2020-08-25 TUN-3321: Add box around logs on UI
- 2020-08-26 TUN-3328: Filter out free tunnel has started log from UI
- 2020-08-27 TUN-3333: Add text to UI explaining how to exit
- 2020-08-27 TUN-3335: Dynamically set connection table size for UI
- 2020-08-10 TUN-3238: Update UI when connection re-connects
- 2020-08-17 TUN-3261: Display connections on UI for free classic tunnels
- 2020-07-24 TUN-3201: Create base cloudflared UI structure
- 2020-07-29 TUN-3200: Add connection information to UI
- 2020-07-24 TUN-3255: Update UI to display URL instead of hostname
- 2020-07-29 TUN-3198: Handle errors while running tunnel UI
2020.9.1
- 2020-09-14 TUN-3395: Unhide named tunnel subcommands, tweak help
- 2020-09-15 TUN-3395: Improve help for list command

View File

@ -209,6 +209,35 @@ stretch: &stretch
pre-cache: *install_pygithub
post-cache:
- make github-message
build-junos:
build_dir: *build_dir
builddeps:
- *pinned_go
- build-essential
- python3
- genisoimage
- jetez
pre-cache:
- ln -s /usr/bin/genisoimage /usr/bin/mkisofs
post-cache:
- export GOOS=freebsd
- export GOARCH=amd64
- make cloudflared-junos
publish-junos:
build_dir: *build_dir
builddeps:
- *pinned_go
- build-essential
- python3
- genisoimage
- jetez
- s4cmd
pre-cache:
- ln -s /usr/bin/genisoimage /usr/bin/mkisofs
post-cache:
- export GOOS=freebsd
- export GOARCH=amd64
- make publish-cloudflared-junos
jessie: *stretch

View File

@ -67,7 +67,7 @@ Description=Update Argo Tunnel
After=network.target
[Service]
ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
`,
},
{

View File

@ -7,6 +7,8 @@ import argparse
import logging
import os
import shutil
import hashlib
import requests
from github import Github, GithubException, UnknownObjectException
@ -15,6 +17,37 @@ logging.basicConfig(format=FORMAT)
CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
GITHUB_CONFLICT_CODE = "already_exists"
BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
UPDATER_PREFIX = 'update'
def get_sha256(filename):
""" get the sha256 of a file """
sha256_hash = hashlib.sha256()
with open(filename,"rb") as f:
for byte_block in iter(lambda: f.read(4096),b""):
sha256_hash.update(byte_block)
return sha256_hash.hexdigest()
def send_hash(pkg_hash, name, version, account, namespace, api_token):
""" send the checksum of a file to workers kv """
key = '{0}_{1}_{2}'.format(UPDATER_PREFIX, version, name)
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer " + api_token,
}
response = requests.put(
BASE_KV_URL + account + "/storage/kv/namespaces/" + namespace + "/values/" + key,
headers=headers,
data=pkg_hash
)
if response.status_code != 200:
jsonResponse = response.json()
errors = jsonResponse["errors"]
if len(errors) > 0:
raise Exception("failed to upload checksum: {0}", errors[0])
def assert_tag_exists(repo, version):
""" Raise exception if repo does not contain a tag matching version """
@ -78,6 +111,15 @@ def parse_args():
parser.add_argument(
"--name", default=os.environ.get("ASSET_NAME"), help="Asset Name"
)
parser.add_argument(
"--namespace-id", default=os.environ.get("KV_NAMESPACE"), help="workersKV namespace id"
)
parser.add_argument(
"--kv-account-id", default=os.environ.get("KV_ACCOUNT"), help="workersKV account id"
)
parser.add_argument(
"--kv-api-token", default=os.environ.get("KV_API_TOKEN"), help="workersKV API Token"
)
parser.add_argument(
"--dry-run", action="store_true", help="Do not create release or upload asset"
)
@ -99,6 +141,18 @@ def parse_args():
if not args.api_key:
logging.error("Missing API key")
is_valid = False
if not args.namespace_id:
logging.error("Missing KV namespace id")
is_valid = False
if not args.kv_account_id:
logging.error("Missing KV account id")
is_valid = False
if not args.kv_api_token:
logging.error("Missing KV API token")
is_valid = False
if is_valid:
return args
@ -121,6 +175,10 @@ def main():
release.upload_asset(args.path, name=args.name)
# send the sha256 (the checksum) to workers kv
pkg_hash = get_sha256(args.path)
send_hash(pkg_hash, args.name, args.release_version, args.kv_account_id, args.namespace_id, args.kv_api_token)
# create the artifacts directory if it doesn't exist
artifact_path = os.path.join(os.getcwd(), 'artifacts')
if not os.path.isdir(artifact_path):

9
jet.yaml Normal file
View File

@ -0,0 +1,9 @@
---
basename: "cloudflared"
comment: "Cloudflare Argo Tunnel"
copyright: "Cloudflare, Inc"
arch: "x86"
abi: "64"
files:
- source: cloudflared
destination: /var/db/scripts/jet/cloudflared

View File

@ -27,6 +27,7 @@ var (
ErrUnauthorized = errors.New("unauthorized")
ErrBadRequest = errors.New("incorrect request parameters")
ErrNotFound = errors.New("not found")
ErrAPINoSuccess = errors.New("API call failed")
)
type Tunnel struct {
@ -39,7 +40,7 @@ type Tunnel struct {
type Connection struct {
ColoName string `json:"colo_name"`
ID uuid.UUID `json:"uuid"`
ID uuid.UUID `json:"id"`
IsPendingReconnect bool `json:"is_pending_reconnect"`
}
@ -91,11 +92,9 @@ func (dr *DNSRoute) MarshalJSON() ([]byte, error) {
func (dr *DNSRoute) UnmarshalResult(body io.Reader) (RouteResult, error) {
var result DNSRouteResult
if err := json.NewDecoder(body).Decode(&result); err != nil {
return nil, err
}
err := parseResponse(body, &result)
result.route = dr
return &result, nil
return &result, err
}
func (dr *DNSRoute) RecordType() string {
@ -152,11 +151,9 @@ func (lr *LBRoute) RecordType() string {
func (lr *LBRoute) UnmarshalResult(body io.Reader) (RouteResult, error) {
var result LBRouteResult
if err := json.NewDecoder(body).Decode(&result); err != nil {
return nil, err
}
err := parseResponse(body, &result)
result.route = lr
return &result, nil
return &result, err
}
func (res *LBRouteResult) SuccessSummary() string {
@ -313,16 +310,18 @@ func (r *RESTClient) ListTunnels(filter *Filter) ([]*Tunnel, error) {
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
var tunnels []*Tunnel
if err := json.NewDecoder(resp.Body).Decode(&tunnels); err != nil {
return nil, errors.Wrap(err, "failed to decode response")
}
return tunnels, nil
return parseListTunnels(resp.Body)
}
return nil, r.statusCodeToError("list tunnels", resp)
}
func parseListTunnels(body io.ReadCloser) ([]*Tunnel, error) {
var tunnels []*Tunnel
err := parseResponse(body, &tunnels)
return tunnels, err
}
func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID) error {
endpoint := r.baseEndpoints.accountLevel
endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
@ -370,15 +369,48 @@ func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (
req.Header.Set("Content-Type", jsonContentType)
}
req.Header.Add("X-Auth-User-Service-Key", r.authToken)
req.Header.Add("Accept", "application/json;version=1")
return r.client.Do(req)
}
func parseResponse(reader io.Reader, data interface{}) error {
// Schema for Tunnelstore responses in the v1 API.
// Roughly, it's a wrapper around a particular result that adds failures/errors/etc
var result struct {
Result json.RawMessage `json:"result"`
Success bool `json:"success"`
Errors []string `json:"errors"`
}
// First, parse the wrapper and check the API call succeeded
if err := json.NewDecoder(reader).Decode(&result); err != nil {
return errors.Wrap(err, "failed to decode response")
}
if err := checkErrors(result.Errors); err != nil {
return err
}
if !result.Success {
return ErrAPINoSuccess
}
// At this point we know the API call succeeded, so, parse out the inner
// result into the datatype provided as a parameter.
return json.Unmarshal(result.Result, &data)
}
func unmarshalTunnel(reader io.Reader) (*Tunnel, error) {
var tunnel Tunnel
if err := json.NewDecoder(reader).Decode(&tunnel); err != nil {
return nil, errors.Wrap(err, "failed to decode response")
err := parseResponse(reader, &tunnel)
return &tunnel, err
}
func checkErrors(errs []string) error {
if len(errs) == 0 {
return nil
}
return &tunnel, nil
if len(errs) == 1 {
return fmt.Errorf("API error: %s", errs[0])
}
allErrs := strings.Join(errs, "; ")
return fmt.Errorf("API errors: %s", allErrs)
}
func (r *RESTClient) statusCodeToError(op string, resp *http.Response) error {

View File

@ -1,9 +1,16 @@
package tunnelstore
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"reflect"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
@ -12,7 +19,7 @@ func TestDNSRouteUnmarshalResult(t *testing.T) {
userHostname: "example.com",
}
result, err := route.UnmarshalResult(strings.NewReader(`{"cname": "new"}`))
result, err := route.UnmarshalResult(strings.NewReader(`{"success": true, "result": {"cname": "new"}}`))
assert.NoError(t, err)
assert.Equal(t, &DNSRouteResult{
@ -20,8 +27,19 @@ func TestDNSRouteUnmarshalResult(t *testing.T) {
CName: ChangeNew,
}, result)
_, err = route.UnmarshalResult(strings.NewReader(`abc`))
assert.NotNil(t, err)
badJSON := []string{
`abc`,
`{"success": false, "result": {"cname": "new"}}`,
`{"errors": ["foo"], "result": {"cname": "new"}}`,
`{"errors": ["foo", "bar"], "result": {"cname": "new"}}`,
`{"result": {"cname": "new"}}`,
`{"result": {"cname": "new"}}`,
}
for _, j := range badJSON {
_, err = route.UnmarshalResult(strings.NewReader(j))
assert.NotNil(t, err)
}
}
func TestLBRouteUnmarshalResult(t *testing.T) {
@ -30,7 +48,7 @@ func TestLBRouteUnmarshalResult(t *testing.T) {
lbPool: "pool",
}
result, err := route.UnmarshalResult(strings.NewReader(`{"pool": "unchanged", "load_balancer": "updated"}`))
result, err := route.UnmarshalResult(strings.NewReader(`{"success": true, "result": {"pool": "unchanged", "load_balancer": "updated"}}`))
assert.NoError(t, err)
assert.Equal(t, &LBRouteResult{
@ -39,8 +57,18 @@ func TestLBRouteUnmarshalResult(t *testing.T) {
Pool: ChangeUnchanged,
}, result)
_, err = route.UnmarshalResult(strings.NewReader(`abc`))
assert.NotNil(t, err)
badJSON := []string{
`abc`,
`{"success": false, "result": {"pool": "unchanged", "load_balancer": "updated"}}`,
`{"errors": ["foo"], "result": {"pool": "unchanged", "load_balancer": "updated"}}`,
`{"errors": ["foo", "bar"], "result": {"pool": "unchanged", "load_balancer": "updated"}}`,
`{"result": {"pool": "unchanged", "load_balancer": "updated"}}`,
}
for _, j := range badJSON {
_, err = route.UnmarshalResult(strings.NewReader(j))
assert.NotNil(t, err)
}
}
func TestLBRouteResultSuccessSummary(t *testing.T) {
@ -54,25 +82,126 @@ func TestLBRouteResultSuccessSummary(t *testing.T) {
pool Change
expected string
}{
{ChangeNew, ChangeNew, "Created load balancer lb.example.com and added a new pool POOL with this tunnel as an origin" },
{ChangeNew, ChangeUpdated, "Created load balancer lb.example.com with an existing pool POOL which was updated to use this tunnel as an origin" },
{ChangeNew, ChangeUnchanged, "Created load balancer lb.example.com with an existing pool POOL which already has this tunnel as an origin" },
{ChangeUpdated, ChangeNew, "Added new pool POOL with this tunnel as an origin to load balancer lb.example.com" },
{ChangeUpdated, ChangeUpdated, "Updated pool POOL to use this tunnel as an origin and added it to load balancer lb.example.com" },
{ChangeUpdated, ChangeUnchanged, "Added pool POOL, which already has this tunnel as an origin, to load balancer lb.example.com" },
{ChangeUnchanged, ChangeNew, "Something went wrong: failed to modify load balancer lb.example.com with pool POOL; please check traffic manager configuration in the dashboard" },
{ChangeUnchanged, ChangeUpdated, "Added this tunnel as an origin in pool POOL which is already used by load balancer lb.example.com" },
{ChangeUnchanged, ChangeUnchanged, "Load balancer lb.example.com already uses pool POOL which has this tunnel as an origin" },
{"", "", "Something went wrong: failed to modify load balancer lb.example.com with pool POOL; please check traffic manager configuration in the dashboard" },
{"a", "b", "Something went wrong: failed to modify load balancer lb.example.com with pool POOL; please check traffic manager configuration in the dashboard" },
{ChangeNew, ChangeNew, "Created load balancer lb.example.com and added a new pool POOL with this tunnel as an origin"},
{ChangeNew, ChangeUpdated, "Created load balancer lb.example.com with an existing pool POOL which was updated to use this tunnel as an origin"},
{ChangeNew, ChangeUnchanged, "Created load balancer lb.example.com with an existing pool POOL which already has this tunnel as an origin"},
{ChangeUpdated, ChangeNew, "Added new pool POOL with this tunnel as an origin to load balancer lb.example.com"},
{ChangeUpdated, ChangeUpdated, "Updated pool POOL to use this tunnel as an origin and added it to load balancer lb.example.com"},
{ChangeUpdated, ChangeUnchanged, "Added pool POOL, which already has this tunnel as an origin, to load balancer lb.example.com"},
{ChangeUnchanged, ChangeNew, "Something went wrong: failed to modify load balancer lb.example.com with pool POOL; please check traffic manager configuration in the dashboard"},
{ChangeUnchanged, ChangeUpdated, "Added this tunnel as an origin in pool POOL which is already used by load balancer lb.example.com"},
{ChangeUnchanged, ChangeUnchanged, "Load balancer lb.example.com already uses pool POOL which has this tunnel as an origin"},
{"", "", "Something went wrong: failed to modify load balancer lb.example.com with pool POOL; please check traffic manager configuration in the dashboard"},
{"a", "b", "Something went wrong: failed to modify load balancer lb.example.com with pool POOL; please check traffic manager configuration in the dashboard"},
}
for i, tt := range tests {
res := &LBRouteResult{
route: route,
LoadBalancer: tt.lb,
Pool: tt.pool,
}
}
actual := res.SuccessSummary()
assert.Equal(t, tt.expected, actual, "case %d", i+1)
}
}
func Test_parseListTunnels(t *testing.T) {
type args struct {
body string
}
tests := []struct {
name string
args args
want []*Tunnel
wantErr bool
}{
{
name: "empty list",
args: args{body: `{"success": true, "result": []}`},
want: []*Tunnel{},
},
{
name: "success is false",
args: args{body: `{"success": false, "result": []}`},
wantErr: true,
},
{
name: "errors are present",
args: args{body: `{"errors": ["foo"], "result": []}`},
wantErr: true,
},
{
name: "invalid response",
args: args{body: `abc`},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
got, err := parseListTunnels(body)
if (err != nil) != tt.wantErr {
t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseListTunnels() = %v, want %v", got, tt.want)
}
})
}
}
func Test_unmarshalTunnel(t *testing.T) {
type args struct {
reader io.Reader
}
tests := []struct {
name string
args args
want *Tunnel
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := unmarshalTunnel(tt.args.reader)
if (err != nil) != tt.wantErr {
t.Errorf("unmarshalTunnel() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("unmarshalTunnel() = %v, want %v", got, tt.want)
}
})
}
}
func TestUnmarshalTunnelOk(t *testing.T) {
jsonBody := `{"success": true, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}`
expected := Tunnel{
ID: uuid.Nil,
Name: "test",
CreatedAt: time.Time{},
Connections: []Connection{},
}
actual, err := unmarshalTunnel(bytes.NewReader([]byte(jsonBody)))
assert.NoError(t, err)
assert.Equal(t, &expected, actual)
}
func TestUnmarshalTunnelErr(t *testing.T) {
tests := []string{
`abc`,
`{"success": true, "result": abc}`,
`{"success": false, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
`{"errors": ["foo"], "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
}
for i, test := range tests {
_, err := unmarshalTunnel(bytes.NewReader([]byte(test)))
assert.Error(t, err, fmt.Sprintf("Test #%v failed", i))
}
}