Release 2020.9.2

.gitignore

@@ -5,6 +5,7 @@ guide/public
 /.GOPATH
 /bin
 .idea
+.build
 .vscode
 \#*\#
 cscope.*
@@ -12,6 +13,7 @@ cloudflared
 cloudflared.pkg
 cloudflared.exe
 cloudflared.msi
+cloudflared-x86-64*
 !cmd/cloudflared/
 .DS_Store
 *-session.log

Makefile

@@ -44,6 +44,8 @@ else ifeq ($(LOCAL_OS),darwin)
     TARGET_OS ?= darwin
 else ifeq ($(LOCAL_OS),windows)
     TARGET_OS ?= windows
+else ifeq ($(LOCAL_OS),freebsd)
+    TARGET_OS ?= freebsd
 else
     $(error This system's OS $(LOCAL_OS) isn't supported)
 endif
@@ -125,6 +127,58 @@ cloudflared-darwin-amd64.tgz: cloudflared
 	tar czf cloudflared-darwin-amd64.tgz cloudflared
 	rm cloudflared
 
+.PHONY: cloudflared-junos
+cloudflared-junos: cloudflared jetez-certificate.pem jetez-key.pem
+	jetez --source . \
+		  -j jet.yaml \
+		  --key jetez-key.pem \
+		  --cert jetez-certificate.pem \
+		  --version $(VERSION)
+	rm jetez-*.pem
+
+jetez-certificate.pem:
+ifndef JETEZ_CERT
+	$(error JETEZ_CERT not defined)
+endif
+	@echo "Writing JetEZ certificate"
+	@echo "$$JETEZ_CERT" > jetez-certificate.pem
+
+jetez-key.pem:
+ifndef JETEZ_KEY
+	$(error JETEZ_KEY not defined)
+endif
+	@echo "Writing JetEZ key"
+	@echo "$$JETEZ_KEY" > jetez-key.pem
+
+.PHONY: publish-cloudflared-junos
+publish-cloudflared-junos: cloudflared-junos cloudflared-x86-64.latest.s3
+ifndef S3_ENDPOINT
+	$(error S3_HOST not defined)
+endif
+ifndef S3_URI
+	$(error S3_URI not defined)
+endif
+ifndef S3_ACCESS_KEY
+	$(error S3_ACCESS_KEY not defined)
+endif
+ifndef S3_SECRET_KEY
+	$(error S3_SECRET_KEY not defined)
+endif
+	sha256sum cloudflared-x86-64-$(VERSION).tgz | awk '{printf $$1}' > cloudflared-x86-64-$(VERSION).tgz.shasum
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz.shasum $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz.shasum
+	dpkg --compare-versions "$(VERSION)" gt "$(shell cat cloudflared-x86-64.latest.s3)" && \
+		echo -n "$(VERSION)" > cloudflared-x86-64.latest && \
+		s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+			put cloudflared-x86-64.latest $(S3_URI)/cloudflared-x86-64.latest || \
+		echo "Latest version not updated"
+
+cloudflared-x86-64.latest.s3:
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force \
+		get $(S3_URI)/cloudflared-x86-64.latest cloudflared-x86-64.latest.s3
+
 .PHONY: homebrew-upload
 homebrew-upload: cloudflared-darwin-amd64.tgz
 	aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $$^ $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz

RELEASE_NOTES

@@ -1,3 +1,25 @@
+2020.9.2
+- 2020-09-22 TRAFFIC-448: build cloudflare for junos and publish to s3
+- 2020-09-22 TUN-3410: Request the v1 Tunnelstore API
+- 2020-09-17 AUTH-3103 CI build fixes
+- 2020-09-18 AUTH-3110-use-cfsetup-precache
+- 2020-09-17 TUN-3295: Show route command results
+- 2020-09-16 TUN-3291: cloudflared tunnel run -h explains how to use flags from parent command
+- 2020-09-18 AUTH-3109 upload the checksum to workers kv on github releases
+- 2020-09-17 updater service exit code should be 11
+- 2020-09-01 TUN-3216: UI improvements
+- 2020-08-25 Rebased and passed TunnelEventChan to LogServerInfo in new ReconnectTunnel function
+- 2020-08-25 TUN-3321: Add box around logs on UI
+- 2020-08-26 TUN-3328: Filter out free tunnel has started log from UI
+- 2020-08-27 TUN-3333: Add text to UI explaining how to exit
+- 2020-08-27 TUN-3335: Dynamically set connection table size for UI
+- 2020-08-10 TUN-3238: Update UI when connection re-connects
+- 2020-08-17 TUN-3261: Display connections on UI for free classic tunnels
+- 2020-07-24 TUN-3201: Create base cloudflared UI structure
+- 2020-07-29 TUN-3200: Add connection information to UI
+- 2020-07-24 TUN-3255: Update UI to display URL instead of hostname
+- 2020-07-29 TUN-3198: Handle errors while running tunnel UI
+
 2020.9.1
 - 2020-09-14 TUN-3395: Unhide named tunnel subcommands, tweak help
 - 2020-09-15 TUN-3395: Improve help for list command

cfsetup.yaml

@@ -209,6 +209,35 @@ stretch: &stretch
     pre-cache: *install_pygithub
     post-cache:
       - make github-message
+  build-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+      - jetez
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make cloudflared-junos
+  publish-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+      - jetez
+      - s4cmd
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make publish-cloudflared-junos
 
 jessie: *stretch
 

cmd/cloudflared/linux_service.go

@@ -67,7 +67,7 @@ Description=Update Argo Tunnel
 After=network.target
 
 [Service]
-ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
+ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
 `,
 	},
 	{

github_release.py

@@ -7,6 +7,8 @@ import argparse
 import logging
 import os
 import shutil
+import hashlib
+import requests
 
 from github import Github, GithubException, UnknownObjectException
 
@@ -15,6 +17,37 @@ logging.basicConfig(format=FORMAT)
 
 CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
 GITHUB_CONFLICT_CODE = "already_exists"
+BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
+UPDATER_PREFIX = 'update'
+
+def get_sha256(filename):
+    """ get the sha256 of a file """
+    sha256_hash = hashlib.sha256()
+    with open(filename,"rb") as f:
+        for byte_block in iter(lambda: f.read(4096),b""):
+            sha256_hash.update(byte_block)
+        return sha256_hash.hexdigest()
+
+def send_hash(pkg_hash, name, version, account, namespace, api_token):
+    """ send the checksum of a file to workers kv """
+    key = '{0}_{1}_{2}'.format(UPDATER_PREFIX, version, name)
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": "Bearer " + api_token,
+    }
+    response = requests.put(
+            BASE_KV_URL + account + "/storage/kv/namespaces/" + namespace + "/values/" + key,
+            headers=headers,
+            data=pkg_hash
+    )
+
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to upload checksum: {0}", errors[0])
+
+
 
 def assert_tag_exists(repo, version):
     """ Raise exception if repo does not contain a tag matching version """
@@ -78,6 +111,15 @@ def parse_args():
     parser.add_argument(
         "--name", default=os.environ.get("ASSET_NAME"), help="Asset Name"
     )
+    parser.add_argument(
+        "--namespace-id", default=os.environ.get("KV_NAMESPACE"), help="workersKV namespace id"
+    )
+    parser.add_argument(
+        "--kv-account-id", default=os.environ.get("KV_ACCOUNT"), help="workersKV account id"
+    )
+    parser.add_argument(
+        "--kv-api-token", default=os.environ.get("KV_API_TOKEN"), help="workersKV API Token"
+    )
     parser.add_argument(
         "--dry-run", action="store_true", help="Do not create release or upload asset"
     )
@@ -100,6 +142,18 @@ def parse_args():
         logging.error("Missing API key")
         is_valid = False
     
+    if not args.namespace_id:
+        logging.error("Missing KV namespace id")
+        is_valid = False
+
+    if not args.kv_account_id:
+        logging.error("Missing KV account id")
+        is_valid = False
+
+    if not args.kv_api_token:
+        logging.error("Missing KV API token")
+        is_valid = False
+
     if is_valid:
         return args
 
@@ -121,6 +175,10 @@ def main():
 
         release.upload_asset(args.path, name=args.name)
 
+        # send the sha256 (the checksum) to workers kv
+        pkg_hash = get_sha256(args.path)
+        send_hash(pkg_hash, args.name, args.release_version, args.kv_account_id, args.namespace_id, args.kv_api_token)
+
         # create the artifacts directory if it doesn't exist 
         artifact_path = os.path.join(os.getcwd(), 'artifacts') 
         if not os.path.isdir(artifact_path):

jet.yaml

@@ -0,0 +1,9 @@
+---
+basename: "cloudflared"
+comment: "Cloudflare Argo Tunnel"
+copyright: "Cloudflare, Inc"
+arch: "x86"
+abi: "64"
+files:
+    - source: cloudflared
+      destination: /var/db/scripts/jet/cloudflared

tunnelstore/client.go

@@ -27,6 +27,7 @@ var (
 	ErrUnauthorized       = errors.New("unauthorized")
 	ErrBadRequest         = errors.New("incorrect request parameters")
 	ErrNotFound           = errors.New("not found")
+	ErrAPINoSuccess       = errors.New("API call failed")
 )
 
 type Tunnel struct {
@@ -39,7 +40,7 @@ type Tunnel struct {
 
 type Connection struct {
 	ColoName           string    `json:"colo_name"`
-	ID                 uuid.UUID `json:"uuid"`
+	ID                 uuid.UUID `json:"id"`
 	IsPendingReconnect bool      `json:"is_pending_reconnect"`
 }
 
@@ -91,11 +92,9 @@ func (dr *DNSRoute) MarshalJSON() ([]byte, error) {
 
 func (dr *DNSRoute) UnmarshalResult(body io.Reader) (RouteResult, error) {
 	var result DNSRouteResult
-	if err := json.NewDecoder(body).Decode(&result); err != nil {
+	err := parseResponse(body, &result)
-		return nil, err
-	}
 	result.route = dr
-	return &result, nil
+	return &result, err
 }
 
 func (dr *DNSRoute) RecordType() string {
@@ -152,11 +151,9 @@ func (lr *LBRoute) RecordType() string {
 
 func (lr *LBRoute) UnmarshalResult(body io.Reader) (RouteResult, error) {
 	var result LBRouteResult
-	if err := json.NewDecoder(body).Decode(&result); err != nil {
+	err := parseResponse(body, &result)
-		return nil, err
-	}
 	result.route = lr
-	return &result, nil
+	return &result, err
 }
 
 func (res *LBRouteResult) SuccessSummary() string {
@@ -313,16 +310,18 @@ func (r *RESTClient) ListTunnels(filter *Filter) ([]*Tunnel, error) {
 	defer resp.Body.Close()
 
 	if resp.StatusCode == http.StatusOK {
-		var tunnels []*Tunnel
+		return parseListTunnels(resp.Body)
-		if err := json.NewDecoder(resp.Body).Decode(&tunnels); err != nil {
-			return nil, errors.Wrap(err, "failed to decode response")
-		}
-		return tunnels, nil
 	}
 
 	return nil, r.statusCodeToError("list tunnels", resp)
 }
 
+func parseListTunnels(body io.ReadCloser) ([]*Tunnel, error) {
+	var tunnels []*Tunnel
+	err := parseResponse(body, &tunnels)
+	return tunnels, err
+}
+
 func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID) error {
 	endpoint := r.baseEndpoints.accountLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
@@ -370,15 +369,48 @@ func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (
 		req.Header.Set("Content-Type", jsonContentType)
 	}
 	req.Header.Add("X-Auth-User-Service-Key", r.authToken)
+	req.Header.Add("Accept", "application/json;version=1")
 	return r.client.Do(req)
 }
 
+func parseResponse(reader io.Reader, data interface{}) error {
+	// Schema for Tunnelstore responses in the v1 API.
+	// Roughly, it's a wrapper around a particular result that adds failures/errors/etc
+	var result struct {
+		Result  json.RawMessage `json:"result"`
+		Success bool            `json:"success"`
+		Errors  []string        `json:"errors"`
+	}
+	// First, parse the wrapper and check the API call succeeded
+	if err := json.NewDecoder(reader).Decode(&result); err != nil {
+		return errors.Wrap(err, "failed to decode response")
+	}
+	if err := checkErrors(result.Errors); err != nil {
+		return err
+	}
+	if !result.Success {
+		return ErrAPINoSuccess
+	}
+	// At this point we know the API call succeeded, so, parse out the inner
+	// result into the datatype provided as a parameter.
+	return json.Unmarshal(result.Result, &data)
+}
+
 func unmarshalTunnel(reader io.Reader) (*Tunnel, error) {
 	var tunnel Tunnel
-	if err := json.NewDecoder(reader).Decode(&tunnel); err != nil {
+	err := parseResponse(reader, &tunnel)
-		return nil, errors.Wrap(err, "failed to decode response")
+	return &tunnel, err
 }
-	return &tunnel, nil
+
+func checkErrors(errs []string) error {
+	if len(errs) == 0 {
+		return nil
+	}
+	if len(errs) == 1 {
+		return fmt.Errorf("API error: %s", errs[0])
+	}
+	allErrs := strings.Join(errs, "; ")
+	return fmt.Errorf("API errors: %s", allErrs)
 }
 
 func (r *RESTClient) statusCodeToError(op string, resp *http.Response) error {

tunnelstore/client_test.go

@@ -1,9 +1,16 @@
 package tunnelstore
 
 import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"reflect"
 	"strings"
 	"testing"
+	"time"
 
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -12,7 +19,7 @@ func TestDNSRouteUnmarshalResult(t *testing.T) {
 		userHostname: "example.com",
 	}
 
-	result, err := route.UnmarshalResult(strings.NewReader(`{"cname": "new"}`))
+	result, err := route.UnmarshalResult(strings.NewReader(`{"success": true, "result": {"cname": "new"}}`))
 
 	assert.NoError(t, err)
 	assert.Equal(t, &DNSRouteResult{
@@ -20,9 +27,20 @@ func TestDNSRouteUnmarshalResult(t *testing.T) {
 		CName: ChangeNew,
 	}, result)
 
-	_, err = route.UnmarshalResult(strings.NewReader(`abc`))
+	badJSON := []string{
+		`abc`,
+		`{"success": false, "result": {"cname": "new"}}`,
+		`{"errors": ["foo"], "result": {"cname": "new"}}`,
+		`{"errors": ["foo", "bar"], "result": {"cname": "new"}}`,
+		`{"result": {"cname": "new"}}`,
+		`{"result": {"cname": "new"}}`,
+	}
+
+	for _, j := range badJSON {
+		_, err = route.UnmarshalResult(strings.NewReader(j))
 		assert.NotNil(t, err)
 	}
+}
 
 func TestLBRouteUnmarshalResult(t *testing.T) {
 	route := &LBRoute{
@@ -30,7 +48,7 @@ func TestLBRouteUnmarshalResult(t *testing.T) {
 		lbPool: "pool",
 	}
 
-	result, err := route.UnmarshalResult(strings.NewReader(`{"pool": "unchanged", "load_balancer": "updated"}`))
+	result, err := route.UnmarshalResult(strings.NewReader(`{"success": true, "result": {"pool": "unchanged", "load_balancer": "updated"}}`))
 
 	assert.NoError(t, err)
 	assert.Equal(t, &LBRouteResult{
@@ -39,9 +57,19 @@ func TestLBRouteUnmarshalResult(t *testing.T) {
 		Pool:         ChangeUnchanged,
 	}, result)
 
-	_, err = route.UnmarshalResult(strings.NewReader(`abc`))
+	badJSON := []string{
+		`abc`,
+		`{"success": false, "result": {"pool": "unchanged", "load_balancer": "updated"}}`,
+		`{"errors": ["foo"], "result": {"pool": "unchanged", "load_balancer": "updated"}}`,
+		`{"errors": ["foo", "bar"], "result": {"pool": "unchanged", "load_balancer": "updated"}}`,
+		`{"result": {"pool": "unchanged", "load_balancer": "updated"}}`,
+	}
+
+	for _, j := range badJSON {
+		_, err = route.UnmarshalResult(strings.NewReader(j))
 		assert.NotNil(t, err)
 	}
+}
 
 func TestLBRouteResultSuccessSummary(t *testing.T) {
 	route := &LBRoute{
@@ -76,3 +104,104 @@ func TestLBRouteResultSuccessSummary(t *testing.T) {
 		assert.Equal(t, tt.expected, actual, "case %d", i+1)
 	}
 }
+
+func Test_parseListTunnels(t *testing.T) {
+	type args struct {
+		body string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []*Tunnel
+		wantErr bool
+	}{
+		{
+			name: "empty list",
+			args: args{body: `{"success": true, "result": []}`},
+			want: []*Tunnel{},
+		},
+		{
+			name:    "success is false",
+			args:    args{body: `{"success": false, "result": []}`},
+			wantErr: true,
+		},
+		{
+			name:    "errors are present",
+			args:    args{body: `{"errors": ["foo"], "result": []}`},
+			wantErr: true,
+		},
+		{
+			name:    "invalid response",
+			args:    args{body: `abc`},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
+			got, err := parseListTunnels(body)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseListTunnels() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_unmarshalTunnel(t *testing.T) {
+	type args struct {
+		reader io.Reader
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *Tunnel
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := unmarshalTunnel(tt.args.reader)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("unmarshalTunnel() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unmarshalTunnel() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestUnmarshalTunnelOk(t *testing.T) {
+
+	jsonBody := `{"success": true, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}`
+	expected := Tunnel{
+		ID:          uuid.Nil,
+		Name:        "test",
+		CreatedAt:   time.Time{},
+		Connections: []Connection{},
+	}
+	actual, err := unmarshalTunnel(bytes.NewReader([]byte(jsonBody)))
+	assert.NoError(t, err)
+	assert.Equal(t, &expected, actual)
+}
+
+func TestUnmarshalTunnelErr(t *testing.T) {
+
+	tests := []string{
+		`abc`,
+		`{"success": true, "result": abc}`,
+		`{"success": false, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
+		`{"errors": ["foo"], "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
+	}
+
+	for i, test := range tests {
+		_, err := unmarshalTunnel(bytes.NewReader([]byte(test)))
+		assert.Error(t, err, fmt.Sprintf("Test #%v failed", i))
+	}
+}