Merge branch 'master' into add-tunnel-warp-routing-flag
This commit is contained in:
Helio Machado
GitHub
commit
83e7eeca79
|
|
@ -1,3 +1,7 @@
|
|||
## 2023.7.0
|
||||
### New Features
|
||||
- You can now enable additional diagnostics over the management.argotunnel.com service for your active cloudflared connectors via a new runtime flag `--management-diagnostics` (or env `TUNNEL_MANAGEMENT_DIAGNOSTICS`). This feature is provided as opt-in and requires the flag to enable. Endpoints such as /metrics provides your prometheus metrics endpoint another mechanism to be reached. Additionally /debug/pprof/(goroutine|heap) are also introduced to allow for remotely retrieving active pprof information from a running cloudflared connector.
|
||||
|
||||
## 2023.4.1
|
||||
### New Features
|
||||
- You can now stream your logs from your remote cloudflared to your local terminal with `cloudflared tail <TUNNEL-ID>`. This new feature requires the remote cloudflared to be version 2023.4.1 or higher.
|
||||
|
|
|
|||
52
Makefile
52
Makefile
|
|
@ -209,58 +209,6 @@ cloudflared-darwin-amd64.tgz: cloudflared
|
|||
tar czf cloudflared-darwin-amd64.tgz cloudflared
|
||||
rm cloudflared
|
||||
|
||||
.PHONY: cloudflared-junos
|
||||
cloudflared-junos: cloudflared jetez-certificate.pem jetez-key.pem
|
||||
jetez --source . \
|
||||
-j jet.yaml \
|
||||
--key jetez-key.pem \
|
||||
--cert jetez-certificate.pem \
|
||||
--version $(VERSION)
|
||||
rm jetez-*.pem
|
||||
|
||||
jetez-certificate.pem:
|
||||
ifndef JETEZ_CERT
|
||||
$(error JETEZ_CERT not defined)
|
||||
endif
|
||||
@echo "Writing JetEZ certificate"
|
||||
@echo "$$JETEZ_CERT" > jetez-certificate.pem
|
||||
|
||||
jetez-key.pem:
|
||||
ifndef JETEZ_KEY
|
||||
$(error JETEZ_KEY not defined)
|
||||
endif
|
||||
@echo "Writing JetEZ key"
|
||||
@echo "$$JETEZ_KEY" > jetez-key.pem
|
||||
|
||||
.PHONY: publish-cloudflared-junos
|
||||
publish-cloudflared-junos: cloudflared-junos cloudflared-x86-64.latest.s3
|
||||
ifndef S3_ENDPOINT
|
||||
$(error S3_HOST not defined)
|
||||
endif
|
||||
ifndef S3_URI
|
||||
$(error S3_URI not defined)
|
||||
endif
|
||||
ifndef S3_ACCESS_KEY
|
||||
$(error S3_ACCESS_KEY not defined)
|
||||
endif
|
||||
ifndef S3_SECRET_KEY
|
||||
$(error S3_SECRET_KEY not defined)
|
||||
endif
|
||||
sha256sum cloudflared-x86-64-$(VERSION).tgz | awk '{printf $$1}' > cloudflared-x86-64-$(VERSION).tgz.shasum
|
||||
s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
|
||||
put cloudflared-x86-64-$(VERSION).tgz $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz
|
||||
s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
|
||||
put cloudflared-x86-64-$(VERSION).tgz.shasum $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz.shasum
|
||||
dpkg --compare-versions "$(VERSION)" gt "$(shell cat cloudflared-x86-64.latest.s3)" && \
|
||||
echo -n "$(VERSION)" > cloudflared-x86-64.latest && \
|
||||
s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
|
||||
put cloudflared-x86-64.latest $(S3_URI)/cloudflared-x86-64.latest || \
|
||||
echo "Latest version not updated"
|
||||
|
||||
cloudflared-x86-64.latest.s3:
|
||||
s4cmd --endpoint-url $(S3_ENDPOINT) --force \
|
||||
get $(S3_URI)/cloudflared-x86-64.latest cloudflared-x86-64.latest.s3
|
||||
|
||||
.PHONY: homebrew-upload
|
||||
homebrew-upload: cloudflared-darwin-amd64.tgz
|
||||
aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $$^ $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz
|
||||
|
|
|
|||
|
|
@ -1,3 +1,20 @@
|
|||
2023.7.1
|
||||
- 2023-07-13 TUN-7582: Correct changelog wording for --management-diagnostics
|
||||
- 2023-07-12 TUN-7575: Add option to disable PTMU discovery over QUIC
|
||||
|
||||
2023.7.0
|
||||
- 2023-07-06 TUN-7558: Flush on Writes for StreamBasedOriginProxy
|
||||
- 2023-07-05 TUN-7553: Add flag to enable management diagnostic services
|
||||
- 2023-07-05 TUN-7564: Support cf-trace-id for cloudflared access
|
||||
- 2023-07-05 TUN-7477: Decrement UDP sessions on shutdown
|
||||
- 2023-07-03 TUN-7545: Add support for full bidirectionally streaming with close signal propagation
|
||||
- 2023-06-30 TUN-7549: Add metrics route to management service
|
||||
- 2023-06-30 TUN-7551: Complete removal of raven-go to sentry-go
|
||||
- 2023-06-30 TUN-7550: Add pprof endpoint to management service
|
||||
- 2023-06-29 TUN-7543: Add --debug-stream flag to cloudflared access ssh
|
||||
- 2023-06-26 TUN-6011: Remove docker networks from ICMP Proxy test
|
||||
- 2023-06-20 AUTH-5328 Pass cloudflared_token_check param when running cloudflared access login
|
||||
|
||||
2023.6.1
|
||||
- 2023-06-19 TUN-7480: Added a timeout for unregisterUDP.
|
||||
- 2023-06-16 TUN-7477: Add UDP/TCP session metrics
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
apiVersion: backstage.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: cloudflared
|
||||
description: Client for Cloudflare Tunnels
|
||||
annotations:
|
||||
backstage.io/source-location: url:https://bitbucket.cfdata.org/projects/TUN/repos/cloudflared/browse
|
||||
cloudflare.com/software-excellence-opt-in: "true"
|
||||
cloudflare.com/jira-project-key: "TUN"
|
||||
cloudflare.com/jira-project-component: "Cloudflare Tunnel"
|
||||
tags:
|
||||
- internal
|
||||
spec:
|
||||
type: "service"
|
||||
lifecycle: "Active"
|
||||
owner: "teams/tunnel-teams-routing"
|
||||
|
|
@ -3,7 +3,7 @@ package cfapi
|
|||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
|
@ -49,7 +49,7 @@ func Test_parseListTunnels(t *testing.T) {
|
|||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
|
||||
body := io.NopCloser(bytes.NewReader([]byte(tt.args.body)))
|
||||
got, err := parseListTunnels(body)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)
|
||||
|
|
|
|||
41
cfsetup.yaml
41
cfsetup.yaml
|
|
@ -121,7 +121,7 @@ buster: &buster
|
|||
- export FIPS=true
|
||||
- export ORIGINAL_NAME=true
|
||||
- make cloudflared-deb
|
||||
build-fips-internal-deb-nightly:
|
||||
build-internal-deb-nightly-amd64:
|
||||
build_dir: *build_dir
|
||||
builddeps: *build_fips_deb_deps
|
||||
post-cache:
|
||||
|
|
@ -131,6 +131,16 @@ buster: &buster
|
|||
- export FIPS=true
|
||||
- export ORIGINAL_NAME=true
|
||||
- make cloudflared-deb
|
||||
build-internal-deb-nightly-arm64:
|
||||
build_dir: *build_dir
|
||||
builddeps: *build_fips_deb_deps
|
||||
post-cache:
|
||||
- export GOOS=linux
|
||||
- export GOARCH=arm64
|
||||
- export NIGHTLY=true
|
||||
#- export FIPS=true # TUN-7595
|
||||
- export ORIGINAL_NAME=true
|
||||
- make cloudflared-deb
|
||||
build-deb-arm64:
|
||||
build_dir: *build_dir
|
||||
builddeps: *build_deb_deps
|
||||
|
|
@ -244,35 +254,6 @@ buster: &buster
|
|||
pre-cache: *install_pygithub
|
||||
post-cache:
|
||||
- make github-message
|
||||
build-junos:
|
||||
build_dir: *build_dir
|
||||
builddeps:
|
||||
- *pinned_go
|
||||
- build-essential
|
||||
- python3
|
||||
- genisoimage
|
||||
pre-cache:
|
||||
- ln -s /usr/bin/genisoimage /usr/bin/mkisofs
|
||||
post-cache:
|
||||
- export CGO_ENABLED=0
|
||||
- export GOOS=freebsd
|
||||
- export GOARCH=amd64
|
||||
- make cloudflared
|
||||
publish-junos:
|
||||
build_dir: *build_dir
|
||||
builddeps:
|
||||
- *pinned_go
|
||||
- build-essential
|
||||
- python3
|
||||
- genisoimage
|
||||
- jetez
|
||||
- s4cmd
|
||||
pre-cache:
|
||||
- ln -s /usr/bin/genisoimage /usr/bin/mkisofs
|
||||
post-cache:
|
||||
- export GOOS=freebsd
|
||||
- export GOARCH=amd64
|
||||
- make publish-cloudflared-junos
|
||||
|
||||
bullseye: *buster
|
||||
bookworm: *buster
|
||||
|
|
|
|||
|
|
@ -148,9 +148,9 @@ func ssh(c *cli.Context) error {
|
|||
func buildRequestHeaders(values []string) http.Header {
|
||||
headers := make(http.Header)
|
||||
for _, valuePair := range values {
|
||||
split := strings.Split(valuePair, ":")
|
||||
if len(split) > 1 {
|
||||
headers.Add(strings.TrimSpace(split[0]), strings.TrimSpace(split[1]))
|
||||
header, value, found := strings.Cut(valuePair, ":")
|
||||
if found {
|
||||
headers.Add(strings.TrimSpace(header), strings.TrimSpace(value))
|
||||
}
|
||||
}
|
||||
return headers
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ func TestBuildRequestHeaders(t *testing.T) {
|
|||
headers.Add("client", "value")
|
||||
headers.Add("secret", "safe-value")
|
||||
|
||||
values := buildRequestHeaders([]string{"client: value", "secret: safe-value", "trash"})
|
||||
values := buildRequestHeaders([]string{"client: value", "secret: safe-value", "trash", "cf-trace-id: 000:000:0:1:asd"})
|
||||
assert.Equal(t, headers.Get("client"), values.Get("client"))
|
||||
assert.Equal(t, headers.Get("secret"), values.Get("secret"))
|
||||
assert.Equal(t, headers.Get("cf-trace-id"), values.Get("000:000:0:1:asd"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
|
|||
Name: "install",
|
||||
Usage: "Install cloudflared as a system service",
|
||||
Action: cliutil.ConfiguredAction(installLinuxService),
|
||||
Flags: []cli.Flag{
|
||||
noUpdateServiceFlag,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "uninstall",
|
||||
|
|
@ -39,15 +42,17 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
|
|||
// The directory and files that are used by the service.
|
||||
// These are hard-coded in the templates below.
|
||||
const (
|
||||
serviceConfigDir = "/etc/cloudflared"
|
||||
serviceConfigFile = "config.yml"
|
||||
serviceCredentialFile = "cert.pem"
|
||||
serviceConfigPath = serviceConfigDir + "/" + serviceConfigFile
|
||||
cloudflaredService = "cloudflared.service"
|
||||
serviceConfigDir = "/etc/cloudflared"
|
||||
serviceConfigFile = "config.yml"
|
||||
serviceCredentialFile = "cert.pem"
|
||||
serviceConfigPath = serviceConfigDir + "/" + serviceConfigFile
|
||||
cloudflaredService = "cloudflared.service"
|
||||
cloudflaredUpdateService = "cloudflared-update.service"
|
||||
cloudflaredUpdateTimer = "cloudflared-update.timer"
|
||||
)
|
||||
|
||||
var systemdTemplates = []ServiceTemplate{
|
||||
{
|
||||
var systemdAllTemplates = map[string]ServiceTemplate{
|
||||
cloudflaredService: {
|
||||
Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredService),
|
||||
Content: `[Unit]
|
||||
Description=cloudflared
|
||||
|
|
@ -64,8 +69,8 @@ RestartSec=5s
|
|||
WantedBy=multi-user.target
|
||||
`,
|
||||
},
|
||||
{
|
||||
Path: "/etc/systemd/system/cloudflared-update.service",
|
||||
cloudflaredUpdateService: {
|
||||
Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredUpdateService),
|
||||
Content: `[Unit]
|
||||
Description=Update cloudflared
|
||||
After=network.target
|
||||
|
|
@ -74,8 +79,8 @@ After=network.target
|
|||
ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
|
||||
`,
|
||||
},
|
||||
{
|
||||
Path: "/etc/systemd/system/cloudflared-update.timer",
|
||||
cloudflaredUpdateTimer: {
|
||||
Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredUpdateTimer),
|
||||
Content: `[Unit]
|
||||
Description=Update cloudflared
|
||||
|
||||
|
|
@ -106,7 +111,7 @@ var sysvTemplate = ServiceTemplate{
|
|||
# Description: cloudflared agent
|
||||
### END INIT INFO
|
||||
name=$(basename $(readlink -f $0))
|
||||
cmd="{{.Path}} --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
|
||||
cmd="{{.Path}} --pidfile /var/run/$name.pid {{ range .ExtraArgs }} {{ . }}{{ end }}"
|
||||
pid_file="/var/run/$name.pid"
|
||||
stdout_log="/var/log/$name.log"
|
||||
stderr_log="/var/log/$name.err"
|
||||
|
|
@ -178,6 +183,14 @@ exit 0
|
|||
`,
|
||||
}
|
||||
|
||||
var (
|
||||
noUpdateServiceFlag = &cli.BoolFlag{
|
||||
Name: "no-update-service",
|
||||
Usage: "Disable auto-update of the cloudflared linux service, which restarts the server to upgrade for new versions.",
|
||||
Value: false,
|
||||
}
|
||||
)
|
||||
|
||||
func isSystemd() bool {
|
||||
if _, err := os.Stat("/run/systemd/system"); err == nil {
|
||||
return true
|
||||
|
|
@ -196,6 +209,9 @@ func installLinuxService(c *cli.Context) error {
|
|||
Path: etPath,
|
||||
}
|
||||
|
||||
// Check if the "no update flag" is set
|
||||
autoUpdate := !c.IsSet(noUpdateServiceFlag.Name)
|
||||
|
||||
var extraArgsFunc func(c *cli.Context, log *zerolog.Logger) ([]string, error)
|
||||
if c.NArg() == 0 {
|
||||
extraArgsFunc = buildArgsForConfig
|
||||
|
|
@ -213,10 +229,10 @@ func installLinuxService(c *cli.Context) error {
|
|||
switch {
|
||||
case isSystemd():
|
||||
log.Info().Msgf("Using Systemd")
|
||||
err = installSystemd(&templateArgs, log)
|
||||
err = installSystemd(&templateArgs, autoUpdate, log)
|
||||
default:
|
||||
log.Info().Msgf("Using SysV")
|
||||
err = installSysv(&templateArgs, log)
|
||||
err = installSysv(&templateArgs, autoUpdate, log)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
|
|
@ -261,7 +277,20 @@ credentials-file: CREDENTIALS-FILE
|
|||
}, nil
|
||||
}
|
||||
|
||||
func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
|
||||
func installSystemd(templateArgs *ServiceTemplateArgs, autoUpdate bool, log *zerolog.Logger) error {
|
||||
var systemdTemplates []ServiceTemplate
|
||||
if autoUpdate {
|
||||
systemdTemplates = []ServiceTemplate{
|
||||
systemdAllTemplates[cloudflaredService],
|
||||
systemdAllTemplates[cloudflaredUpdateService],
|
||||
systemdAllTemplates[cloudflaredUpdateTimer],
|
||||
}
|
||||
} else {
|
||||
systemdTemplates = []ServiceTemplate{
|
||||
systemdAllTemplates[cloudflaredService],
|
||||
}
|
||||
}
|
||||
|
||||
for _, serviceTemplate := range systemdTemplates {
|
||||
err := serviceTemplate.Generate(templateArgs)
|
||||
if err != nil {
|
||||
|
|
@ -273,10 +302,14 @@ func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) erro
|
|||
log.Err(err).Msgf("systemctl enable %s error", cloudflaredService)
|
||||
return err
|
||||
}
|
||||
if err := runCommand("systemctl", "start", "cloudflared-update.timer"); err != nil {
|
||||
log.Err(err).Msg("systemctl start cloudflared-update.timer error")
|
||||
return err
|
||||
|
||||
if autoUpdate {
|
||||
if err := runCommand("systemctl", "start", cloudflaredUpdateTimer); err != nil {
|
||||
log.Err(err).Msgf("systemctl start %s error", cloudflaredUpdateTimer)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := runCommand("systemctl", "daemon-reload"); err != nil {
|
||||
log.Err(err).Msg("systemctl daemon-reload error")
|
||||
return err
|
||||
|
|
@ -284,12 +317,19 @@ func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) erro
|
|||
return runCommand("systemctl", "start", cloudflaredService)
|
||||
}
|
||||
|
||||
func installSysv(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
|
||||
func installSysv(templateArgs *ServiceTemplateArgs, autoUpdate bool, log *zerolog.Logger) error {
|
||||
confPath, err := sysvTemplate.ResolvePath()
|
||||
if err != nil {
|
||||
log.Err(err).Msg("error resolving system path")
|
||||
return err
|
||||
}
|
||||
|
||||
if autoUpdate {
|
||||
templateArgs.ExtraArgs = append([]string{"--autoupdate-freq 24h0m0s"}, templateArgs.ExtraArgs...)
|
||||
} else {
|
||||
templateArgs.ExtraArgs = append([]string{"--no-autoupdate"}, templateArgs.ExtraArgs...)
|
||||
}
|
||||
|
||||
if err := sysvTemplate.Generate(templateArgs); err != nil {
|
||||
log.Err(err).Msg("error generating system template")
|
||||
return err
|
||||
|
|
@ -327,19 +367,35 @@ func uninstallLinuxService(c *cli.Context) error {
|
|||
}
|
||||
|
||||
func uninstallSystemd(log *zerolog.Logger) error {
|
||||
if err := runCommand("systemctl", "disable", cloudflaredService); err != nil {
|
||||
log.Err(err).Msgf("systemctl disable %s error", cloudflaredService)
|
||||
return err
|
||||
// Get only the installed services
|
||||
installedServices := make(map[string]ServiceTemplate)
|
||||
for serviceName, serviceTemplate := range systemdAllTemplates {
|
||||
if err := runCommand("systemctl", "status", serviceName); err == nil {
|
||||
installedServices[serviceName] = serviceTemplate
|
||||
} else {
|
||||
log.Info().Msgf("Service '%s' not installed, skipping its uninstall", serviceName)
|
||||
}
|
||||
}
|
||||
if err := runCommand("systemctl", "stop", cloudflaredService); err != nil {
|
||||
log.Err(err).Msgf("systemctl stop %s error", cloudflaredService)
|
||||
return err
|
||||
|
||||
if _, exists := installedServices[cloudflaredService]; exists {
|
||||
if err := runCommand("systemctl", "disable", cloudflaredService); err != nil {
|
||||
log.Err(err).Msgf("systemctl disable %s error", cloudflaredService)
|
||||
return err
|
||||
}
|
||||
if err := runCommand("systemctl", "stop", cloudflaredService); err != nil {
|
||||
log.Err(err).Msgf("systemctl stop %s error", cloudflaredService)
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := runCommand("systemctl", "stop", "cloudflared-update.timer"); err != nil {
|
||||
log.Err(err).Msg("systemctl stop cloudflared-update.timer error")
|
||||
return err
|
||||
|
||||
if _, exists := installedServices[cloudflaredUpdateTimer]; exists {
|
||||
if err := runCommand("systemctl", "stop", cloudflaredUpdateTimer); err != nil {
|
||||
log.Err(err).Msgf("systemctl stop %s error", cloudflaredUpdateTimer)
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, serviceTemplate := range systemdTemplates {
|
||||
|
||||
for _, serviceTemplate := range installedServices {
|
||||
if err := serviceTemplate.Remove(); err != nil {
|
||||
log.Err(err).Msg("error removing service template")
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import (
|
|||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
|
|
@ -64,7 +63,7 @@ func (st *ServiceTemplate) Generate(args *ServiceTemplateArgs) error {
|
|||
return fmt.Errorf("error creating %s: %v", plistFolder, err)
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
|
||||
err = os.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing %s: %v", resolvedPath, err)
|
||||
}
|
||||
|
|
@ -103,7 +102,7 @@ func runCommand(command string, args ...string) error {
|
|||
return fmt.Errorf("error starting %s: %v", command, err)
|
||||
}
|
||||
|
||||
output, _ := ioutil.ReadAll(stderr)
|
||||
output, _ := io.ReadAll(stderr)
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s %v returned with error code %v due to: %v", command, args, err, string(output))
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"runtime/trace"
|
||||
|
|
@ -12,7 +11,7 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-systemd/daemon"
|
||||
"github.com/coreos/go-systemd/v22/daemon"
|
||||
"github.com/facebookgo/grace/gracenet"
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/google/uuid"
|
||||
|
|
@ -82,6 +81,11 @@ const (
|
|||
// udpUnregisterSessionTimeout is how long we wait before we stop trying to unregister a UDP session from the edge
|
||||
udpUnregisterSessionTimeoutFlag = "udp-unregister-session-timeout"
|
||||
|
||||
// quicDisablePathMTUDiscovery sets if QUIC should not perform PTMU discovery and use a smaller (safe) packet size.
|
||||
// Packets will then be at most 1252 (IPv4) / 1232 (IPv6) bytes in size.
|
||||
// Note that this may result in packet drops for UDP proxying, since we expect being able to send at least 1280 bytes of inner packets.
|
||||
quicDisablePathMTUDiscovery = "quic-disable-pmtu-discovery"
|
||||
|
||||
// uiFlag is to enable launching cloudflared in interactive UI mode
|
||||
uiFlag = "ui"
|
||||
|
||||
|
|
@ -300,7 +304,7 @@ func StartServer(
|
|||
}
|
||||
|
||||
if c.IsSet("trace-output") {
|
||||
tmpTraceFile, err := ioutil.TempFile("", "trace")
|
||||
tmpTraceFile, err := os.CreateTemp("", "trace")
|
||||
if err != nil {
|
||||
log.Err(err).Msg("Failed to create new temporary file to save trace output")
|
||||
}
|
||||
|
|
@ -413,6 +417,7 @@ func StartServer(
|
|||
|
||||
mgmt := management.New(
|
||||
c.String("management-hostname"),
|
||||
c.Bool("management-diagnostics"),
|
||||
serviceIP,
|
||||
clientID,
|
||||
c.String(connectorLabelFlag),
|
||||
|
|
@ -691,6 +696,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
|
|||
Value: 5 * time.Second,
|
||||
Hidden: true,
|
||||
}),
|
||||
altsrc.NewBoolFlag(&cli.BoolFlag{
|
||||
Name: quicDisablePathMTUDiscovery,
|
||||
EnvVars: []string{"TUNNEL_DISABLE_QUIC_PMTU"},
|
||||
Usage: "Use this option to disable PTMU discovery for QUIC connections. This will result in lower packet sizes. Not however, that this may cause instability for UDP proxying.",
|
||||
Value: false,
|
||||
Hidden: true,
|
||||
}),
|
||||
altsrc.NewStringFlag(&cli.StringFlag{
|
||||
Name: connectorLabelFlag,
|
||||
Usage: "Use this option to give a meaningful label to a specific connector. When a tunnel starts up, a connector id unique to the tunnel is generated. This is a uuid. To make it easier to identify a connector, we will use the hostname of the machine the tunnel is running on along with the connector ID. This option exists if one wants to have more control over what their individual connectors are called.",
|
||||
|
|
@ -770,6 +782,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
|
|||
EnvVars: []string{"TUNNEL_WARP_ROUTING"},
|
||||
Hidden: shouldHide,
|
||||
}),
|
||||
altsrc.NewBoolFlag(&cli.BoolFlag{
|
||||
Name: "management-diagnostics",
|
||||
Usage: "Enables the in-depth diagnostic routes to be made available over the management service (/debug/pprof, /metrics, etc.)",
|
||||
EnvVars: []string{"TUNNEL_MANAGEMENT_DIAGNOSTICS"},
|
||||
Value: false,
|
||||
}),
|
||||
selectProtocolFlag,
|
||||
overwriteDNSFlag,
|
||||
}...)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import (
|
|||
"github.com/rs/zerolog"
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/urfave/cli/v2/altsrc"
|
||||
"golang.org/x/crypto/ssh/terminal"
|
||||
"golang.org/x/term"
|
||||
|
||||
"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
|
||||
"github.com/cloudflare/cloudflared/config"
|
||||
|
|
@ -240,6 +240,7 @@ func prepareTunnelConfig(
|
|||
PQKexIdx: pqKexIdx,
|
||||
MaxEdgeAddrRetries: uint8(c.Int("max-edge-addr-retries")),
|
||||
UDPUnregisterSessionTimeout: c.Duration(udpUnregisterSessionTimeoutFlag),
|
||||
DisableQUICPathMTUDiscovery: c.Bool(quicDisablePathMTUDiscovery),
|
||||
}
|
||||
packetConfig, err := newPacketConfig(c, log)
|
||||
if err != nil {
|
||||
|
|
@ -279,7 +280,7 @@ func gracePeriod(c *cli.Context) (time.Duration, error) {
|
|||
}
|
||||
|
||||
func isRunningFromTerminal() bool {
|
||||
return terminal.IsTerminal(int(os.Stdout.Fd()))
|
||||
return term.IsTerminal(int(os.Stdout.Fd()))
|
||||
}
|
||||
|
||||
// Remove any duplicates from the slice
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package tunnel
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
)
|
||||
|
||||
|
|
@ -23,5 +22,5 @@ func (fs realFileSystem) validFilePath(path string) bool {
|
|||
}
|
||||
|
||||
func (fs realFileSystem) readFile(filePath string) ([]byte, error) {
|
||||
return ioutil.ReadFile(filePath)
|
||||
return os.ReadFile(filePath)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package tunnel
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
|
@ -66,7 +65,7 @@ func login(c *cli.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
|
||||
if err := os.WriteFile(path, resourceData, 0600); err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("error writing cert to %s", path))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import (
|
|||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
|
@ -241,7 +240,7 @@ func writeTunnelCredentials(filePath string, credentials *connection.Credentials
|
|||
if err != nil {
|
||||
return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
|
||||
}
|
||||
return ioutil.WriteFile(filePath, body, 400)
|
||||
return os.WriteFile(filePath, body, 0400)
|
||||
}
|
||||
|
||||
func buildListCommand() *cli.Command {
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import (
|
|||
"github.com/facebookgo/grace/gracenet"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/urfave/cli/v2"
|
||||
"golang.org/x/crypto/ssh/terminal"
|
||||
"golang.org/x/term"
|
||||
|
||||
"github.com/cloudflare/cloudflared/config"
|
||||
"github.com/cloudflare/cloudflared/logger"
|
||||
|
|
@ -304,7 +304,7 @@ func wasInstalledFromPackageManager() bool {
|
|||
}
|
||||
|
||||
func isRunningFromTerminal() bool {
|
||||
return terminal.IsTerminal(int(os.Stdout.Fd()))
|
||||
return term.IsTerminal(int(os.Stdout.Fd()))
|
||||
}
|
||||
|
||||
func IsSysV() bool {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -224,7 +223,7 @@ func TestUpdateService(t *testing.T) {
|
|||
require.Equal(t, v.Version(), mostRecentVersion)
|
||||
|
||||
require.NoError(t, v.Apply())
|
||||
dat, err := ioutil.ReadFile(testFilePath)
|
||||
dat, err := os.ReadFile(testFilePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, string(dat), mostRecentVersion)
|
||||
|
|
@ -243,7 +242,7 @@ func TestBetaUpdateService(t *testing.T) {
|
|||
require.Equal(t, v.Version(), mostRecentBetaVersion)
|
||||
|
||||
require.NoError(t, v.Apply())
|
||||
dat, err := ioutil.ReadFile(testFilePath)
|
||||
dat, err := os.ReadFile(testFilePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, string(dat), mostRecentBetaVersion)
|
||||
|
|
@ -289,7 +288,7 @@ func TestForcedUpdateService(t *testing.T) {
|
|||
require.Equal(t, v.Version(), mostRecentVersion)
|
||||
|
||||
require.NoError(t, v.Apply())
|
||||
dat, err := ioutil.ReadFile(testFilePath)
|
||||
dat, err := os.ReadFile(testFilePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, string(dat), mostRecentVersion)
|
||||
|
|
@ -309,7 +308,7 @@ func TestUpdateSpecificVersionService(t *testing.T) {
|
|||
require.Equal(t, reqVersion, v.Version())
|
||||
|
||||
require.NoError(t, v.Apply())
|
||||
dat, err := ioutil.ReadFile(testFilePath)
|
||||
dat, err := os.ReadFile(testFilePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, reqVersion, string(dat))
|
||||
|
|
@ -328,7 +327,7 @@ func TestCompressedUpdateService(t *testing.T) {
|
|||
require.Equal(t, "2020.09.02", v.Version())
|
||||
|
||||
require.NoError(t, v.Apply())
|
||||
dat, err := ioutil.ReadFile(testFilePath)
|
||||
dat, err := os.ReadFile(testFilePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, "2020.09.02", string(dat))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
#!/usr/bin/env python
|
||||
import requests
|
||||
from conftest import CfdModes
|
||||
from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
|
||||
from retrying import retry
|
||||
from cli import CloudflaredCli
|
||||
from util import LOGGER, write_config, start_cloudflared, wait_tunnel_ready, send_requests
|
||||
import platform
|
||||
|
||||
"""
|
||||
Each test in TestManagement will:
|
||||
1. Acquire a management token from Cloudflare public API
|
||||
2. Make a request against the management service for the running tunnel
|
||||
"""
|
||||
class TestManagement:
|
||||
"""
|
||||
test_get_host_details does the following:
|
||||
1. It gets a management token from Tunnelstore using cloudflared tail token <tunnel_id>
|
||||
2. It gets the connector_id after starting a cloudflare tunnel
|
||||
3. It sends a request to the management host with the connector_id and management token
|
||||
4. Asserts that the response has a hostname and ip.
|
||||
"""
|
||||
def test_get_host_details(self, tmp_path, component_tests_config):
|
||||
# TUN-7377 : wait_tunnel_ready does not work properly in windows.
|
||||
# Skipping this test for windows for now and will address it as part of tun-7377
|
||||
if platform.system() == "Windows":
|
||||
return
|
||||
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
|
||||
LOGGER.debug(config)
|
||||
headers = {}
|
||||
headers["Content-Type"] = "application/json"
|
||||
config_path = write_config(tmp_path, config.full_config)
|
||||
with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--label" , "test"], cfd_args=["run", "--hello-world"], new_process=True):
|
||||
wait_tunnel_ready(tunnel_url=config.get_url(),
|
||||
require_min_connections=1)
|
||||
cfd_cli = CloudflaredCli(config, config_path, LOGGER)
|
||||
connector_id = cfd_cli.get_connector_id(config)[0]
|
||||
url = cfd_cli.get_management_url("host_details", config, config_path)
|
||||
resp = send_request(url, headers=headers)
|
||||
|
||||
# Assert response json.
|
||||
assert resp.status_code == 200, "Expected cloudflared to return 200 for host details"
|
||||
assert resp.json()["hostname"] == "custom:test", "Expected cloudflared to return hostname"
|
||||
assert resp.json()["ip"] != "", "Expected cloudflared to return ip"
|
||||
assert resp.json()["connector_id"] == connector_id, "Expected cloudflared to return connector_id"
|
||||
|
||||
"""
|
||||
test_get_metrics will verify that the /metrics endpoint returns the prometheus metrics dump
|
||||
"""
|
||||
def test_get_metrics(self, tmp_path, component_tests_config):
|
||||
# TUN-7377 : wait_tunnel_ready does not work properly in windows.
|
||||
# Skipping this test for windows for now and will address it as part of tun-7377
|
||||
if platform.system() == "Windows":
|
||||
return
|
||||
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
|
||||
LOGGER.debug(config)
|
||||
config_path = write_config(tmp_path, config.full_config)
|
||||
with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
|
||||
wait_tunnel_ready(require_min_connections=1)
|
||||
cfd_cli = CloudflaredCli(config, config_path, LOGGER)
|
||||
url = cfd_cli.get_management_url("metrics", config, config_path)
|
||||
resp = send_request(url)
|
||||
|
||||
# Assert response.
|
||||
assert resp.status_code == 200, "Expected cloudflared to return 200 for /metrics"
|
||||
assert "# HELP build_info Build and version information" in resp.text, "Expected /metrics to have with the build_info details"
|
||||
|
||||
"""
|
||||
test_get_pprof_heap will verify that the /debug/pprof/heap endpoint returns a pprof/heap dump response
|
||||
"""
|
||||
def test_get_pprof_heap(self, tmp_path, component_tests_config):
|
||||
# TUN-7377 : wait_tunnel_ready does not work properly in windows.
|
||||
# Skipping this test for windows for now and will address it as part of tun-7377
|
||||
if platform.system() == "Windows":
|
||||
return
|
||||
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
|
||||
LOGGER.debug(config)
|
||||
config_path = write_config(tmp_path, config.full_config)
|
||||
with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
|
||||
wait_tunnel_ready(require_min_connections=1)
|
||||
cfd_cli = CloudflaredCli(config, config_path, LOGGER)
|
||||
url = cfd_cli.get_management_url("debug/pprof/heap", config, config_path)
|
||||
resp = send_request(url)
|
||||
|
||||
# Assert response.
|
||||
assert resp.status_code == 200, "Expected cloudflared to return 200 for /debug/pprof/heap"
|
||||
assert resp.headers["Content-Type"] == "application/octet-stream", "Expected /debug/pprof/heap to have return a binary response"
|
||||
|
||||
"""
|
||||
test_get_metrics_when_disabled will verify that diagnostic endpoints (such as /metrics) return 404 and are unmounted.
|
||||
"""
|
||||
def test_get_metrics_when_disabled(self, tmp_path, component_tests_config):
|
||||
# TUN-7377 : wait_tunnel_ready does not work properly in windows.
|
||||
# Skipping this test for windows for now and will address it as part of tun-7377
|
||||
if platform.system() == "Windows":
|
||||
return
|
||||
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
|
||||
LOGGER.debug(config)
|
||||
config_path = write_config(tmp_path, config.full_config)
|
||||
with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
|
||||
wait_tunnel_ready(require_min_connections=1)
|
||||
cfd_cli = CloudflaredCli(config, config_path, LOGGER)
|
||||
url = cfd_cli.get_management_url("metrics", config, config_path)
|
||||
resp = send_request(url)
|
||||
|
||||
# Assert response.
|
||||
assert resp.status_code == 404, "Expected cloudflared to return 404 for /metrics"
|
||||
|
||||
|
||||
@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
|
||||
def send_request(url, headers={}):
|
||||
with requests.Session() as s:
|
||||
return s.get(url, timeout=BACKOFF_SECS, headers=headers)
|
||||
|
|
@ -16,38 +16,6 @@ class TestTunnel:
|
|||
with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], cfd_args=["run", "--hello-world"], new_process=True):
|
||||
wait_tunnel_ready(tunnel_url=config.get_url(),
|
||||
require_min_connections=1)
|
||||
|
||||
"""
|
||||
test_get_host_details does the following:
|
||||
1. It gets a management token from Tunnelstore using cloudflared tail token <tunnel_id>
|
||||
2. It gets the connector_id after starting a cloudflare tunnel
|
||||
3. It sends a request to the management host with the connector_id and management token
|
||||
4. Asserts that the response has a hostname and ip.
|
||||
"""
|
||||
def test_get_host_details(self, tmp_path, component_tests_config):
|
||||
# TUN-7377 : wait_tunnel_ready does not work properly in windows.
|
||||
# Skipping this test for windows for now and will address it as part of tun-7377
|
||||
if platform.system() == "Windows":
|
||||
return
|
||||
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
|
||||
LOGGER.debug(config)
|
||||
headers = {}
|
||||
headers["Content-Type"] = "application/json"
|
||||
config_path = write_config(tmp_path, config.full_config)
|
||||
with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--label" , "test"], cfd_args=["run", "--hello-world"], new_process=True):
|
||||
wait_tunnel_ready(tunnel_url=config.get_url(),
|
||||
require_min_connections=1)
|
||||
cfd_cli = CloudflaredCli(config, config_path, LOGGER)
|
||||
connector_id = cfd_cli.get_connector_id(config)[0]
|
||||
url = cfd_cli.get_management_url("host_details", config, config_path)
|
||||
resp = send_request(url, headers=headers)
|
||||
|
||||
# Assert response json.
|
||||
assert resp.status_code == 200, "Expected cloudflared to return 200 for host details"
|
||||
assert resp.json()["hostname"] == "custom:test", "Expected cloudflared to return hostname"
|
||||
assert resp.json()["ip"] != "", "Expected cloudflared to return ip"
|
||||
assert resp.json()["connector_id"] == connector_id, "Expected cloudflared to return connector_id"
|
||||
|
||||
|
||||
def test_tunnel_url(self, tmp_path, component_tests_config):
|
||||
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
|
||||
|
|
|
|||
|
|
@ -157,14 +157,16 @@ type ReadWriteAcker interface {
|
|||
type HTTPResponseReadWriteAcker struct {
|
||||
r io.Reader
|
||||
w ResponseWriter
|
||||
f http.Flusher
|
||||
req *http.Request
|
||||
}
|
||||
|
||||
// NewHTTPResponseReadWriterAcker returns a new instance of HTTPResponseReadWriteAcker.
|
||||
func NewHTTPResponseReadWriterAcker(w ResponseWriter, req *http.Request) *HTTPResponseReadWriteAcker {
|
||||
func NewHTTPResponseReadWriterAcker(w ResponseWriter, flusher http.Flusher, req *http.Request) *HTTPResponseReadWriteAcker {
|
||||
return &HTTPResponseReadWriteAcker{
|
||||
r: req.Body,
|
||||
w: w,
|
||||
f: flusher,
|
||||
req: req,
|
||||
}
|
||||
}
|
||||
|
|
@ -174,7 +176,11 @@ func (h *HTTPResponseReadWriteAcker) Read(p []byte) (int, error) {
|
|||
}
|
||||
|
||||
func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
|
||||
return h.w.Write(p)
|
||||
n, err := h.w.Write(p)
|
||||
if n > 0 {
|
||||
h.f.Flush()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
|
||||
|
|
|
|||
|
|
@ -130,7 +130,8 @@ func wsEchoEndpoint(w ResponseWriter, r *http.Request) error {
|
|||
}
|
||||
wsCtx, cancel := context.WithCancel(r.Context())
|
||||
readPipe, writePipe := io.Pipe()
|
||||
wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
|
||||
|
||||
wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, w.(http.Flusher), r), &log)
|
||||
go func() {
|
||||
select {
|
||||
case <-wsCtx.Done():
|
||||
|
|
@ -175,7 +176,7 @@ func wsFlakyEndpoint(w ResponseWriter, r *http.Request) error {
|
|||
}
|
||||
wsCtx, cancel := context.WithCancel(r.Context())
|
||||
|
||||
wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
|
||||
wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, w.(http.Flusher), r), &log)
|
||||
|
||||
closedAfter := time.Millisecond * time.Duration(rand.Intn(50))
|
||||
originConn := &flakyConn{closeAt: time.Now().Add(closedAfter)}
|
||||
|
|
|
|||
|
|
@ -142,7 +142,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||
break
|
||||
}
|
||||
|
||||
rws := NewHTTPResponseReadWriterAcker(respWriter, r)
|
||||
rws := NewHTTPResponseReadWriterAcker(respWriter, respWriter, r)
|
||||
requestErr = originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
|
||||
Dest: host,
|
||||
CFRay: FindCfRayHeader(r),
|
||||
|
|
@ -289,6 +289,10 @@ func (rp *http2RespWriter) Header() http.Header {
|
|||
return rp.respHeaders
|
||||
}
|
||||
|
||||
func (rp *http2RespWriter) Flush() {
|
||||
rp.flusher.Flush()
|
||||
}
|
||||
|
||||
func (rp *http2RespWriter) WriteHeader(status int) {
|
||||
if rp.hijacked() {
|
||||
rp.log.Warn().Msg("WriteHeader after hijack")
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -84,7 +83,7 @@ func TestHTTP2ConfigurationSet(t *testing.T) {
|
|||
resp, err := edgeHTTP2Conn.RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
bdy, err := ioutil.ReadAll(resp.Body)
|
||||
bdy, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, `{"lastAppliedVersion":2,"err":null}`, string(bdy))
|
||||
cancel()
|
||||
|
|
@ -149,7 +148,7 @@ func TestServeHTTP(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.Equal(t, test.expectedStatus, resp.StatusCode)
|
||||
if test.expectedBody != nil {
|
||||
respBody, err := ioutil.ReadAll(resp.Body)
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.expectedBody, respBody)
|
||||
}
|
||||
|
|
@ -546,7 +545,7 @@ func benchmarkServeHTTP(b *testing.B, test testRequest) {
|
|||
require.NoError(b, err)
|
||||
require.Equal(b, test.expectedStatus, resp.StatusCode)
|
||||
if test.expectedBody != nil {
|
||||
respBody, err := ioutil.ReadAll(resp.Body)
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
require.NoError(b, err)
|
||||
require.Equal(b, test.expectedBody, respBody)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -449,10 +449,21 @@ func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header)
|
|||
return hrw.WriteConnectResponseData(nil, metadata...)
|
||||
}
|
||||
|
||||
func (hrw *httpResponseAdapter) Write(p []byte) (int, error) {
|
||||
// Make sure to send WriteHeader response if not called yet
|
||||
if !hrw.connectResponseSent {
|
||||
hrw.WriteRespHeaders(http.StatusOK, hrw.headers)
|
||||
}
|
||||
return hrw.RequestServerStream.Write(p)
|
||||
}
|
||||
|
||||
func (hrw *httpResponseAdapter) Header() http.Header {
|
||||
return hrw.headers
|
||||
}
|
||||
|
||||
// This is a no-op Flush because this adapter is over a quic.Stream and we don't need Flush here.
|
||||
func (hrw *httpResponseAdapter) Flush() {}
|
||||
|
||||
func (hrw *httpResponseAdapter) WriteHeader(status int) {
|
||||
hrw.WriteRespHeaders(status, hrw.headers)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -92,7 +92,10 @@ func (m *manager) shutdownSessions(err error) {
|
|||
byRemote: true,
|
||||
}
|
||||
for _, s := range m.sessions {
|
||||
s.close(closeSessionErr)
|
||||
m.unregisterSession(&unregisterSessionEvent{
|
||||
sessionID: s.ID,
|
||||
err: closeSessionErr,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
14
go.mod
14
go.mod
|
|
@ -3,16 +3,15 @@ module github.com/cloudflare/cloudflared
|
|||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
|
||||
github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
|
||||
github.com/coredns/coredns v1.10.0
|
||||
github.com/coreos/go-oidc/v3 v3.4.0
|
||||
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
|
||||
github.com/coreos/go-oidc/v3 v3.6.0
|
||||
github.com/coreos/go-systemd/v22 v22.5.0
|
||||
github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
|
||||
github.com/fsnotify/fsnotify v1.4.9
|
||||
github.com/getsentry/raven-go v0.2.0
|
||||
github.com/getsentry/sentry-go v0.16.0
|
||||
github.com/go-chi/chi/v5 v5.0.8
|
||||
github.com/go-chi/cors v1.2.1
|
||||
github.com/go-jose/go-jose/v3 v3.0.0
|
||||
github.com/gobwas/ws v1.0.4
|
||||
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
|
||||
|
|
@ -43,9 +42,7 @@ require (
|
|||
golang.org/x/sys v0.7.0
|
||||
golang.org/x/term v0.7.0
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/coreos/go-oidc.v2 v2.2.1
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0
|
||||
gopkg.in/square/go-jose.v2 v2.6.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
nhooyr.io/websocket v1.8.7
|
||||
zombiezen.com/go/capnproto2 v2.18.0+incompatible
|
||||
|
|
@ -55,7 +52,6 @@ require (
|
|||
github.com/BurntSushi/toml v1.2.0 // indirect
|
||||
github.com/apparentlymart/go-cidr v1.1.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 // indirect
|
||||
github.com/coredns/caddy v1.1.1 // indirect
|
||||
|
|
@ -66,7 +62,6 @@ require (
|
|||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
|
||||
github.com/go-chi/cors v1.2.1 // indirect
|
||||
github.com/go-logr/logr v1.2.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
|
||||
|
|
@ -88,7 +83,6 @@ require (
|
|||
github.com/onsi/gomega v1.23.0 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/procfs v0.8.0 // indirect
|
||||
github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
|
||||
|
|
@ -96,7 +90,7 @@ require (
|
|||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
|
||||
golang.org/x/mod v0.8.0 // indirect
|
||||
golang.org/x/oauth2 v0.4.0 // indirect
|
||||
golang.org/x/oauth2 v0.6.0 // indirect
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
golang.org/x/tools v0.6.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
|
|
|
|||
237
go.sum
237
go.sum
|
|
@ -13,36 +13,14 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
|
|||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
|
||||
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
|
||||
cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
|
||||
cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
|
||||
cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
|
||||
cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY=
|
||||
cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
|
||||
cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
|
||||
cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
|
||||
cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
|
||||
cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
|
||||
cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
|
||||
cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
|
||||
cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
|
||||
cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
|
||||
cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
|
||||
cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
|
||||
cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
|
||||
cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
|
||||
cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
|
|
@ -52,7 +30,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
|
|||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
|
||||
|
|
@ -74,8 +51,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
|
|||
github.com/bwesterb/go-ristretto v1.2.2/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s=
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
|
|
@ -84,8 +59,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
|
|||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93 h1:QrGfkZDnMxcWHaYDdB7CmqS9i26OAnUj/xcus/abYkY=
|
||||
github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93/go.mod h1:QiTe66jFdP7cUKMCCf/WrvDyYdtdmdZfVcdoLbzaKVY=
|
||||
github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 h1:YzpECHxZ9TzO7LpnKmPxItSd79lLgrR5heIlnqU4dTU=
|
||||
github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47/go.mod h1:qhx8gBILsYlbam7h09SvHDSkjpe3TfLA7b/z4rxJvkE=
|
||||
github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
|
||||
|
|
@ -95,23 +68,21 @@ github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633/go.mod h1:j/igS
|
|||
github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2 h1:0/KuLjh9lBMiXlooAdwoo+FbLVD5DABtquB0ImEFOK0=
|
||||
github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2/go.mod h1:XzuZIjv4mF5cM205RHHW1d60PQtWGwMR6jx38YKuYHs=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
|
||||
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
|
||||
github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
|
||||
github.com/coredns/coredns v1.10.0 h1:jCfuWsBjTs0dapkkhISfPCzn5LqvSRtrFtaf/Tjj4DI=
|
||||
github.com/coredns/coredns v1.10.0/go.mod h1:CIfRU5TgpuoIiJBJ4XrofQzfFQpPFh32ERpUevrSlaw=
|
||||
github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g=
|
||||
github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw=
|
||||
github.com/coreos/go-oidc/v3 v3.6.0 h1:AKVxfYw1Gmkn/w96z0DbT/B/xFnzTd3MkZvWLjF4n/o=
|
||||
github.com/coreos/go-oidc/v3 v3.6.0/go.mod h1:ZpHUsHBucTUj6WOkrP4E20UPynbLZzhTQ1XKCXkxyPc=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
|
||||
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
|
|
@ -124,12 +95,9 @@ github.com/devincarr/quic-go v0.0.0-20230502200822-d1f4edacbee7/go.mod h1:+4CVgV
|
|||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
|
||||
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
|
||||
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
|
||||
|
|
@ -145,8 +113,6 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjr
|
|||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
|
||||
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
|
||||
github.com/getsentry/sentry-go v0.16.0 h1:owk+S+5XcgJLlGR/3+3s6N4d+uKwqYvh/eS0AIMjPWo=
|
||||
github.com/getsentry/sentry-go v0.16.0/go.mod h1:ZXCloQLj0pG7mja5NK6NPf2V4A88YJ4pNlc2mOHwh6Y=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
|
|
@ -197,6 +163,7 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E
|
|||
github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
|
||||
github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
|
||||
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
|
||||
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
|
||||
|
|
@ -213,7 +180,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
|
|||
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
|
||||
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
|
|
@ -231,10 +197,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
|
|||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
|
|
@ -244,21 +208,16 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
|||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
|
||||
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
|
|
@ -266,12 +225,6 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
|
|||
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
|
|
@ -279,15 +232,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
|
|||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
|
||||
github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
|
||||
github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
|
||||
github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
|
||||
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
|
|
@ -366,8 +312,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
|||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
|
||||
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
|
|
@ -428,15 +372,12 @@ github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0
|
|||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
|
||||
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
|
||||
go.opentelemetry.io/contrib/propagators v0.22.0 h1:KGdv58M2//veiYLIhb31mofaI2LgkIPXXAZVeYVyfd8=
|
||||
go.opentelemetry.io/contrib/propagators v0.22.0/go.mod h1:xGOuXr6lLIF9BXipA4pm6UuOSI0M98U6tsI3khbOiwU=
|
||||
go.opentelemetry.io/otel v1.0.0-RC2/go.mod h1:w1thVQ7qbAy8MHb0IFj8a5Q2QU0l2ksf8u/CN8m3NOM=
|
||||
|
|
@ -489,8 +430,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
|
|||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
|
|
@ -499,8 +438,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
|
|||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
|
|
@ -532,26 +469,12 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
|
|||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
|
|
@ -559,24 +482,11 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
|
|||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
|
||||
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
|
||||
golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
|
||||
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
|
||||
golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
|
||||
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
|
@ -585,10 +495,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -624,44 +532,17 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
|
||||
|
|
@ -675,7 +556,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
|
|
@ -725,17 +605,7 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY
|
|||
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
|
||||
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
|
|
@ -743,9 +613,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
|
|||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
|
|
@ -762,29 +629,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
|
|||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
|
||||
google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
|
||||
google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
|
||||
google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
|
||||
google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
|
||||
google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo=
|
||||
google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4=
|
||||
google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
|
||||
google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
|
||||
google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
|
||||
google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
|
||||
google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
|
||||
google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
|
||||
google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
|
||||
google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
|
||||
google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
|
||||
google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
|
||||
google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
|
||||
google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
|
||||
google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
|
||||
google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
|
||||
google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
|
||||
google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
|
@ -823,54 +667,7 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
|
|||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
|
||||
google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
|
||||
google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
|
||||
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
|
||||
google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
|
||||
google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
|
||||
google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
|
||||
google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
|
||||
google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
|
||||
google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
|
||||
google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
|
||||
google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
|
||||
google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
|
||||
google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
|
||||
google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
|
||||
google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
|
||||
google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
|
||||
google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
|
||||
google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
|
||||
google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
|
||||
google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
|
||||
google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
|
||||
google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6fs45z9AxiewiKyLKYpA45W5Kpkks=
|
||||
google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
|
|
@ -885,29 +682,13 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
|
|||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
|
||||
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
|
||||
google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
|
||||
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
|
||||
google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
|
||||
google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
|
||||
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
|
||||
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
|
||||
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
|
||||
google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
|
||||
google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
|
||||
google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
|
||||
google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
|
@ -930,13 +711,9 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
|
|||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/coreos/go-oidc.v2 v2.2.1 h1:MY5SZClJ7vhjKfr64a4nHAOV/c3WH2gB9BMrR64J1Mc=
|
||||
gopkg.in/coreos/go-oidc.v2 v2.2.1/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
|
||||
gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
|
||||
gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
//go:build !cgo
|
||||
// +build !cgo
|
||||
|
||||
package h2mux
|
||||
|
||||
import (
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
//go:build cgo
|
||||
// +build cgo
|
||||
|
||||
package h2mux
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/cloudflare/brotli-go"
|
||||
)
|
||||
|
||||
func CompressionIsSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func newDecompressor(src io.Reader) *brotli.Reader {
|
||||
return brotli.NewReader(src)
|
||||
}
|
||||
|
||||
func newCompressor(dst io.Writer, quality, lgwin int) *brotli.Writer {
|
||||
return brotli.NewWriter(dst, brotli.WriterOptions{Quality: quality, LGWin: lgwin})
|
||||
}
|
||||
|
|
@ -135,14 +135,7 @@ func Handshake(
|
|||
m.f.ReadMetaHeaders = hpack.NewDecoder(4096, func(hpack.HeaderField) {})
|
||||
// Initialise the settings to identify this connection and confirm the other end is sane.
|
||||
handshakeSetting := http2.Setting{ID: SettingMuxerMagic, Val: MuxerMagicEdge}
|
||||
compressionSetting := http2.Setting{ID: SettingCompression, Val: config.CompressionQuality.toH2Setting()}
|
||||
if CompressionIsSupported() {
|
||||
config.Log.Debug().Msg("muxer: Compression is supported")
|
||||
m.compressionQuality = config.CompressionQuality.getPreset()
|
||||
} else {
|
||||
config.Log.Debug().Msg("muxer: Compression is not supported")
|
||||
compressionSetting = http2.Setting{ID: SettingCompression, Val: 0}
|
||||
}
|
||||
compressionSetting := http2.Setting{ID: SettingCompression, Val: 0}
|
||||
|
||||
expectedMagic := MuxerMagicOrigin
|
||||
if config.IsClient {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net"
|
||||
"os"
|
||||
|
|
@ -652,7 +651,7 @@ func TestHPACK(t *testing.T) {
|
|||
if stream.Headers[0].Value != "200" {
|
||||
t.Fatalf("expected status 200, got %s", stream.Headers[0].Value)
|
||||
}
|
||||
_, _ = ioutil.ReadAll(stream)
|
||||
_, _ = io.ReadAll(stream)
|
||||
_ = stream.Close()
|
||||
}
|
||||
}
|
||||
|
|
@ -678,157 +677,6 @@ func AssertIfPipeReadable(t *testing.T, pipe io.ReadCloser) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestMultipleStreamsWithDictionaries(t *testing.T) {
|
||||
l := zerolog.Nop()
|
||||
|
||||
for q := CompressionNone; q <= CompressionMax; q++ {
|
||||
htmlBody := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"` +
|
||||
`"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">` +
|
||||
`<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">` +
|
||||
`<head>` +
|
||||
` <title>Your page title here</title>` +
|
||||
`</head>` +
|
||||
`<body>` +
|
||||
`<h1>Your major heading here</h1>` +
|
||||
`<p>` +
|
||||
`This is a regular text paragraph.` +
|
||||
`</p>` +
|
||||
`<ul>` +
|
||||
` <li>` +
|
||||
` First bullet of a bullet list.` +
|
||||
` </li>` +
|
||||
` <li>` +
|
||||
` This is the <em>second</em> bullet.` +
|
||||
` </li>` +
|
||||
`</ul>` +
|
||||
`</body>` +
|
||||
`</html>`
|
||||
|
||||
f := MuxedStreamFunc(func(stream *MuxedStream) error {
|
||||
var contentType string
|
||||
var pathHeader Header
|
||||
|
||||
for _, h := range stream.Headers {
|
||||
if h.Name == ":path" {
|
||||
pathHeader = h
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if pathHeader.Name != ":path" {
|
||||
panic("Couldn't find :path header in test")
|
||||
}
|
||||
|
||||
if strings.Contains(pathHeader.Value, "html") {
|
||||
contentType = "text/html; charset=utf-8"
|
||||
} else if strings.Contains(pathHeader.Value, "js") {
|
||||
contentType = "application/javascript"
|
||||
} else if strings.Contains(pathHeader.Value, "css") {
|
||||
contentType = "text/css"
|
||||
} else {
|
||||
contentType = "img/gif"
|
||||
}
|
||||
|
||||
_ = stream.WriteHeaders([]Header{
|
||||
{Name: "content-type", Value: contentType},
|
||||
})
|
||||
_, _ = stream.Write([]byte(strings.Replace(htmlBody, "paragraph", pathHeader.Value, 1) + stream.Headers[5].Value))
|
||||
|
||||
return nil
|
||||
})
|
||||
muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, f)
|
||||
muxPair.Serve(t)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
paths := []string{
|
||||
"/html1",
|
||||
"/html2?sa:ds",
|
||||
"/html3",
|
||||
"/css1",
|
||||
"/html1",
|
||||
"/html2?sa:ds",
|
||||
"/html3",
|
||||
"/css1",
|
||||
"/css2",
|
||||
"/css3",
|
||||
"/js",
|
||||
"/js",
|
||||
"/js",
|
||||
"/js2",
|
||||
"/img2",
|
||||
"/html1",
|
||||
"/html2?sa:ds",
|
||||
"/html3",
|
||||
"/css1",
|
||||
"/css2",
|
||||
"/css3",
|
||||
"/js",
|
||||
"/js",
|
||||
"/js",
|
||||
"/js2",
|
||||
"/img1",
|
||||
}
|
||||
|
||||
wg.Add(len(paths))
|
||||
errorsC := make(chan error, len(paths))
|
||||
|
||||
for i, s := range paths {
|
||||
go func(index int, path string) {
|
||||
defer wg.Done()
|
||||
stream, err := muxPair.OpenEdgeMuxStream(
|
||||
[]Header{
|
||||
{Name: ":method", Value: "GET"},
|
||||
{Name: ":scheme", Value: "https"},
|
||||
{Name: ":authority", Value: "tunnel.otterlyadorable.co.uk"},
|
||||
{Name: ":path", Value: path},
|
||||
{Name: "cf-ray", Value: "378948953f044408-SFO-DOG"},
|
||||
{Name: "idx", Value: strconv.Itoa(index)},
|
||||
{Name: "accept-encoding", Value: "gzip, br"},
|
||||
},
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
errorsC <- fmt.Errorf("error in OpenStream: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
expectBody := strings.Replace(htmlBody, "paragraph", path, 1) + strconv.Itoa(index)
|
||||
responseBody := make([]byte, len(expectBody)*2)
|
||||
n, err := stream.Read(responseBody)
|
||||
if err != nil {
|
||||
errorsC <- fmt.Errorf("stream %d error from (*MuxedStream).Read: %s", stream.streamID, err)
|
||||
return
|
||||
}
|
||||
if n != len(expectBody) {
|
||||
errorsC <- fmt.Errorf("stream %d expected response body to have %d bytes, got %d", stream.streamID, len(expectBody), n)
|
||||
return
|
||||
}
|
||||
if string(responseBody[:n]) != expectBody {
|
||||
errorsC <- fmt.Errorf("stream %d expected response body %s, got %s", stream.streamID, expectBody, responseBody[:n])
|
||||
return
|
||||
}
|
||||
}(i, s)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errorsC)
|
||||
testFail := false
|
||||
for err := range errorsC {
|
||||
testFail = true
|
||||
l.Error().Msgf("%s", err)
|
||||
}
|
||||
if testFail {
|
||||
t.Fatalf("TestMultipleStreams failed")
|
||||
}
|
||||
|
||||
originMuxMetrics := muxPair.OriginMux.Metrics()
|
||||
if q > CompressionNone && originMuxMetrics.CompBytesBefore.Value() <= 10*originMuxMetrics.CompBytesAfter.Value() {
|
||||
t.Fatalf("Cross-stream compression is expected to give a better compression ratio")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sampleSiteHandler(files map[string][]byte) MuxedStreamFunc {
|
||||
return func(stream *MuxedStream) error {
|
||||
var contentType string
|
||||
|
|
@ -905,7 +753,7 @@ func loadSampleFiles(paths []string) (map[string][]byte, error) {
|
|||
files := make(map[string][]byte)
|
||||
for _, path := range paths {
|
||||
if _, ok := files[path]; !ok {
|
||||
expectBody, err := ioutil.ReadFile(path)
|
||||
expectBody, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -915,82 +763,6 @@ func loadSampleFiles(paths []string) (map[string][]byte, error) {
|
|||
return files, nil
|
||||
}
|
||||
|
||||
func TestSampleSiteWithDictionaries(t *testing.T) {
|
||||
paths := []string{
|
||||
"./sample/index.html",
|
||||
"./sample/index2.html",
|
||||
"./sample/index1.html",
|
||||
"./sample/ghost-url.min.js",
|
||||
"./sample/jquery.fitvids.js",
|
||||
"./sample/index1.html",
|
||||
"./sample/index2.html",
|
||||
"./sample/index.html",
|
||||
}
|
||||
files, err := loadSampleFiles(paths)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for q := CompressionNone; q <= CompressionMax; q++ {
|
||||
muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, sampleSiteHandler(files))
|
||||
muxPair.Serve(t)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errC := make(chan error, len(paths))
|
||||
|
||||
wg.Add(len(paths))
|
||||
for _, s := range paths {
|
||||
go func(path string) {
|
||||
defer wg.Done()
|
||||
errC <- sampleSiteTest(muxPair, path, files)
|
||||
}(s)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errC)
|
||||
|
||||
for err := range errC {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
originMuxMetrics := muxPair.OriginMux.Metrics()
|
||||
if q > CompressionNone && originMuxMetrics.CompBytesBefore.Value() <= 10*originMuxMetrics.CompBytesAfter.Value() {
|
||||
t.Fatalf("Cross-stream compression is expected to give a better compression ratio")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongSiteWithDictionaries(t *testing.T) {
|
||||
paths := []string{
|
||||
"./sample/index.html",
|
||||
"./sample/index1.html",
|
||||
"./sample/index2.html",
|
||||
"./sample/ghost-url.min.js",
|
||||
"./sample/jquery.fitvids.js",
|
||||
}
|
||||
files, err := loadSampleFiles(paths)
|
||||
assert.NoError(t, err)
|
||||
for q := CompressionNone; q <= CompressionMedium; q++ {
|
||||
muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, sampleSiteHandler(files))
|
||||
muxPair.Serve(t)
|
||||
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
tstLen := 500
|
||||
errGroup, _ := errgroup.WithContext(context.Background())
|
||||
for i := 0; i < tstLen; i++ {
|
||||
errGroup.Go(func() error {
|
||||
path := paths[rand.Int()%len(paths)]
|
||||
return sampleSiteTest(muxPair, path, files)
|
||||
})
|
||||
}
|
||||
assert.NoError(t, errGroup.Wait())
|
||||
|
||||
originMuxMetrics := muxPair.OriginMux.Metrics()
|
||||
if q > CompressionNone && originMuxMetrics.CompBytesBefore.Value() <= 10*originMuxMetrics.CompBytesAfter.Value() {
|
||||
t.Fatalf("Cross-stream compression is expected to give a better compression ratio")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkOpenStream(b *testing.B) {
|
||||
const streams = 5000
|
||||
for i := 0; i < b.N; i++ {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
|
|
@ -234,7 +234,7 @@ func rootHandler(serverName string) http.HandlerFunc {
|
|||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
var buffer bytes.Buffer
|
||||
var body string
|
||||
rawBody, err := ioutil.ReadAll(r.Body)
|
||||
rawBody, err := io.ReadAll(r.Body)
|
||||
if err == nil {
|
||||
body = string(rawBody)
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -118,7 +117,7 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
|
|||
}
|
||||
for _, status := range statusCodes {
|
||||
handler := func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte(sendMessage), body)
|
||||
|
||||
|
|
@ -180,7 +179,7 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
|
|||
assert.NoError(t, err)
|
||||
assert.Equal(t, status, resp.StatusCode)
|
||||
require.Equal(t, echoHeaderReturnValue, resp.Header.Get(echoHeaderName))
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte(echoMessage), body)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package ingress
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -141,7 +141,7 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
respBody, err := ioutil.ReadAll(resp.Body)
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, respBody, []byte(originURL.Host))
|
||||
}
|
||||
|
|
@ -180,7 +180,7 @@ func TestHTTPServiceUsesIngressRuleScheme(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
respBody, err := ioutil.ReadAll(resp.Body)
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, respBody, []byte(p))
|
||||
}
|
||||
|
|
|
|||
9
jet.yaml
9
jet.yaml
|
|
@ -1,9 +0,0 @@
|
|||
---
|
||||
basename: "cloudflared"
|
||||
comment: "Cloudflare Tunnel"
|
||||
copyright: "Cloudflare, Inc"
|
||||
arch: "x86"
|
||||
abi: "64"
|
||||
files:
|
||||
- source: cloudflared
|
||||
destination: /var/db/scripts/jet/cloudflared
|
||||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -30,6 +31,17 @@ const (
|
|||
reasonIdleLimitExceeded = "session was idle for too long"
|
||||
)
|
||||
|
||||
var (
|
||||
// CORS middleware required to allow dash to access management.argotunnel.com requests
|
||||
corsHandler = cors.Handler(cors.Options{
|
||||
// Allows for any subdomain of cloudflare.com
|
||||
AllowedOrigins: []string{"https://*.cloudflare.com"},
|
||||
// Required to present cookies or other authentication across origin boundries
|
||||
AllowCredentials: true,
|
||||
MaxAge: 300, // Maximum value not ignored by any of major browsers
|
||||
})
|
||||
)
|
||||
|
||||
type ManagementService struct {
|
||||
// The management tunnel hostname
|
||||
Hostname string
|
||||
|
|
@ -53,6 +65,7 @@ type ManagementService struct {
|
|||
}
|
||||
|
||||
func New(managementHostname string,
|
||||
enableDiagServices bool,
|
||||
serviceIP string,
|
||||
clientID uuid.UUID,
|
||||
label string,
|
||||
|
|
@ -70,21 +83,21 @@ func New(managementHostname string,
|
|||
}
|
||||
r := chi.NewRouter()
|
||||
r.Use(ValidateAccessTokenQueryMiddleware)
|
||||
r.Get("/ping", ping)
|
||||
r.Head("/ping", ping)
|
||||
|
||||
// Default management services
|
||||
r.With(corsHandler).Get("/ping", ping)
|
||||
r.With(corsHandler).Head("/ping", ping)
|
||||
r.Get("/logs", s.logs)
|
||||
r.Get("/metrics", s.metricsHandler.ServeHTTP)
|
||||
r.Route("/host_details", func(r chi.Router) {
|
||||
// CORS middleware required to allow dash to access management.argotunnel.com requests
|
||||
r.Use(cors.Handler(cors.Options{
|
||||
// Allows for any subdomain of cloudflare.com
|
||||
AllowedOrigins: []string{"https://*.cloudflare.com"},
|
||||
// Required to present cookies or other authentication across origin boundries
|
||||
AllowCredentials: true,
|
||||
MaxAge: 300, // Maximum value not ignored by any of major browsers
|
||||
}))
|
||||
r.Get("/", s.getHostDetails)
|
||||
})
|
||||
r.With(corsHandler).Get("/host_details", s.getHostDetails)
|
||||
|
||||
// Diagnostic management services
|
||||
if enableDiagServices {
|
||||
// Prometheus endpoint
|
||||
r.With(corsHandler).Get("/metrics", s.metricsHandler.ServeHTTP)
|
||||
// Supports only heap and goroutine
|
||||
r.With(corsHandler).Get("/debug/pprof/{profile:heap|goroutine}", pprof.Index)
|
||||
}
|
||||
|
||||
s.router = r
|
||||
return s
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,13 @@ package management
|
|||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
|
@ -15,9 +19,23 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
noopLogger = zerolog.New(io.Discard)
|
||||
noopLogger = zerolog.New(io.Discard)
|
||||
managementHostname = "https://management.argotunnel.com"
|
||||
)
|
||||
|
||||
func TestDisableDiagnosticRoutes(t *testing.T) {
|
||||
mgmt := New("management.argotunnel.com", false, "1.1.1.1:80", uuid.Nil, "", &noopLogger, nil)
|
||||
for _, path := range []string{"/metrics", "/debug/pprof/goroutine", "/debug/pprof/heap"} {
|
||||
t.Run(strings.Replace(path, "/", "_", -1), func(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", managementHostname+path+"?access_token="+validToken, nil)
|
||||
recorder := httptest.NewRecorder()
|
||||
mgmt.ServeHTTP(recorder, req)
|
||||
resp := recorder.Result()
|
||||
require.Equal(t, http.StatusNotFound, resp.StatusCode)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadEventsLoop(t *testing.T) {
|
||||
sentEvent := EventStartStreaming{
|
||||
ClientEvent: ClientEvent{Type: StartStreaming},
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ import (
|
|||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/go-jose/go-jose/v3"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/square/go-jose.v2"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -51,7 +50,7 @@ func TestUpdateConfiguration(t *testing.T) {
|
|||
initConfig := &Config{
|
||||
Ingress: &ingress.Ingress{},
|
||||
}
|
||||
orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, []ingress.Rule{ingress.NewManagementRule(management.New("management.argotunnel.com", "1.1.1.1:80", uuid.Nil, "", &testLogger, nil))}, &testLogger)
|
||||
orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, []ingress.Rule{ingress.NewManagementRule(management.New("management.argotunnel.com", false, "1.1.1.1:80", uuid.Nil, "", &testLogger, nil))}, &testLogger)
|
||||
require.NoError(t, err)
|
||||
initOriginProxy, err := orchestrator.GetOriginProxy()
|
||||
require.NoError(t, err)
|
||||
|
|
@ -338,7 +337,7 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
|
|||
switch resp.StatusCode {
|
||||
// v1 proxy, warp enabled
|
||||
case 200:
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, t.Name(), string(body))
|
||||
warpRoutingDisabled = false
|
||||
|
|
@ -450,7 +449,7 @@ func proxyTCP(ctx context.Context, originProxy connection.OriginProxy, originAdd
|
|||
CFRay: "123",
|
||||
LBProbe: false,
|
||||
}
|
||||
rws := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
|
||||
rws := connection.NewHTTPResponseReadWriterAcker(respWriter, w.(http.Flusher), req)
|
||||
|
||||
return originProxy.ProxyTCP(ctx, rws, tcpReq)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -136,8 +136,11 @@ func (p *Proxy) ProxyHTTP(
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rws := connection.NewHTTPResponseReadWriterAcker(w, req)
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
return fmt.Errorf("response writer is not a flusher")
|
||||
}
|
||||
rws := connection.NewHTTPResponseReadWriterAcker(w, flusher, req)
|
||||
if err := p.proxyStream(tr.ToTracedContext(), rws, dest, originProxy); err != nil {
|
||||
rule, srv := ruleField(p.ingressRules, ruleNum)
|
||||
p.logRequestError(err, cfRay, "", rule, srv)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
package proxy
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -17,7 +16,7 @@ import (
|
|||
)
|
||||
|
||||
func TestUnixSocketOrigin(t *testing.T) {
|
||||
file, err := ioutil.TempFile("", "unix.sock")
|
||||
file, err := os.CreateTemp("", "unix.sock")
|
||||
require.NoError(t, err)
|
||||
os.Remove(file.Name()) // remove the file since binding the socket expects to create it
|
||||
|
||||
|
|
|
|||
|
|
@ -698,7 +698,7 @@ func TestConnections(t *testing.T) {
|
|||
}()
|
||||
}
|
||||
if test.args.connectionType == connection.TypeTCP {
|
||||
rwa := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
|
||||
rwa := connection.NewHTTPResponseReadWriterAcker(respWriter, respWriter.(http.Flusher), req)
|
||||
err = proxy.ProxyTCP(ctx, rwa, &connection.TCPRequest{Dest: dest})
|
||||
} else {
|
||||
log := zerolog.Nop()
|
||||
|
|
@ -834,6 +834,8 @@ func (w *wsRespWriter) WriteRespHeaders(status int, header http.Header) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (w *wsRespWriter) Flush() {}
|
||||
|
||||
func (w *wsRespWriter) AddTrailer(trailerName, trailerValue string) {
|
||||
// do nothing
|
||||
}
|
||||
|
|
@ -873,6 +875,8 @@ func (m *mockTCPRespWriter) Write(p []byte) (n int, err error) {
|
|||
return m.w.Write(p)
|
||||
}
|
||||
|
||||
func (m *mockTCPRespWriter) Flush() {}
|
||||
|
||||
func (m *mockTCPRespWriter) AddTrailer(trailerName, trailerValue string) {
|
||||
// do nothing
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package socks
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
|
@ -32,7 +32,7 @@ func sendSocksRequest(t *testing.T) []byte {
|
|||
assert.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
b, err := ioutil.ReadAll(resp.Body)
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
assert.NoError(t, err)
|
||||
|
||||
return b
|
||||
|
|
|
|||
|
|
@ -10,15 +10,15 @@ import (
|
|||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/go-jose/go-jose/v3/jwt"
|
||||
homedir "github.com/mitchellh/go-homedir"
|
||||
"github.com/pkg/errors"
|
||||
gossh "golang.org/x/crypto/ssh"
|
||||
"gopkg.in/square/go-jose.v2/jwt"
|
||||
|
||||
"github.com/cloudflare/cloudflared/config"
|
||||
cfpath "github.com/cloudflare/cloudflared/token"
|
||||
|
|
@ -148,7 +148,7 @@ func generateKeyPair(fullName string) ([]byte, error) {
|
|||
return nil, err
|
||||
}
|
||||
if exist {
|
||||
return ioutil.ReadFile(pubKeyName)
|
||||
return os.ReadFile(pubKeyName)
|
||||
}
|
||||
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
|
|
@ -187,5 +187,5 @@ func writeKey(filename string, data []byte) error {
|
|||
return err
|
||||
}
|
||||
|
||||
return ioutil.WriteFile(filepath, data, 0600)
|
||||
return os.WriteFile(filepath, data, 0600)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
|
|
@ -16,9 +15,9 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-jose/go-jose/v3"
|
||||
"github.com/go-jose/go-jose/v3/jwt"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"gopkg.in/square/go-jose.v2"
|
||||
"gopkg.in/square/go-jose.v2/jwt"
|
||||
|
||||
"github.com/cloudflare/cloudflared/config"
|
||||
cfpath "github.com/cloudflare/cloudflared/token"
|
||||
|
|
@ -64,7 +63,7 @@ func TestCertGenSuccess(t *testing.T) {
|
|||
mockRequest = func(url, contentType string, body io.Reader) (*http.Response, error) {
|
||||
assert.Contains(t, "/cdn-cgi/access/cert_sign", url)
|
||||
assert.Equal(t, "application/json", contentType)
|
||||
buf, err := ioutil.ReadAll(body)
|
||||
buf, err := io.ReadAll(body)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, buf)
|
||||
return w.Result(), nil
|
||||
|
|
|
|||
|
|
@ -8,13 +8,43 @@ import (
|
|||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/raven-go"
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/cloudflare/cloudflared/cfio"
|
||||
)
|
||||
|
||||
type Stream interface {
|
||||
Reader
|
||||
WriterCloser
|
||||
}
|
||||
|
||||
type Reader interface {
|
||||
io.Reader
|
||||
}
|
||||
|
||||
type WriterCloser interface {
|
||||
io.Writer
|
||||
WriteCloser
|
||||
}
|
||||
|
||||
type WriteCloser interface {
|
||||
CloseWrite() error
|
||||
}
|
||||
|
||||
type nopCloseWriterAdapter struct {
|
||||
io.ReadWriter
|
||||
}
|
||||
|
||||
func NopCloseWriterAdapter(stream io.ReadWriter) *nopCloseWriterAdapter {
|
||||
return &nopCloseWriterAdapter{stream}
|
||||
}
|
||||
|
||||
func (n *nopCloseWriterAdapter) CloseWrite() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type bidirectionalStreamStatus struct {
|
||||
doneChan chan struct{}
|
||||
anyDone uint32
|
||||
|
|
@ -32,8 +62,24 @@ func (s *bidirectionalStreamStatus) markUniStreamDone() {
|
|||
s.doneChan <- struct{}{}
|
||||
}
|
||||
|
||||
func (s *bidirectionalStreamStatus) waitAnyDone() {
|
||||
func (s *bidirectionalStreamStatus) wait(maxWaitForSecondStream time.Duration) error {
|
||||
<-s.doneChan
|
||||
|
||||
// Only wait for second stream to finish if maxWait is greater than zero
|
||||
if maxWaitForSecondStream > 0 {
|
||||
|
||||
timer := time.NewTimer(maxWaitForSecondStream)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-timer.C:
|
||||
return fmt.Errorf("timeout waiting for second stream to finish")
|
||||
case <-s.doneChan:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
func (s *bidirectionalStreamStatus) isAnyDone() bool {
|
||||
return atomic.LoadUint32(&s.anyDone) > 0
|
||||
|
|
@ -41,16 +87,28 @@ func (s *bidirectionalStreamStatus) isAnyDone() bool {
|
|||
|
||||
// Pipe copies copy data to & from provided io.ReadWriters.
|
||||
func Pipe(tunnelConn, originConn io.ReadWriter, log *zerolog.Logger) {
|
||||
status := newBiStreamStatus()
|
||||
|
||||
go unidirectionalStream(tunnelConn, originConn, "origin->tunnel", status, log)
|
||||
go unidirectionalStream(originConn, tunnelConn, "tunnel->origin", status, log)
|
||||
|
||||
// If one side is done, we are done.
|
||||
status.waitAnyDone()
|
||||
PipeBidirectional(NopCloseWriterAdapter(tunnelConn), NopCloseWriterAdapter(originConn), 0, log)
|
||||
}
|
||||
|
||||
func unidirectionalStream(dst io.Writer, src io.Reader, dir string, status *bidirectionalStreamStatus, log *zerolog.Logger) {
|
||||
// PipeBidirectional copies data two BidirectionStreams. It is a special case of Pipe where it receives a concept that allows for Read and Write side to be closed independently.
|
||||
// The main difference is that when piping data from a reader to a writer, if EOF is read, then this implementation propagates the EOF signal to the destination/writer by closing the write side of the
|
||||
// Bidirectional Stream.
|
||||
// Finally, depending on once EOF is ready from one of the provided streams, the other direction of streaming data will have a configured time period to also finish, otherwise,
|
||||
// the method will return immediately with a timeout error. It is however, the responsability of the caller to close the associated streams in both ends in order to free all the resources/go-routines.
|
||||
func PipeBidirectional(downstream, upstream Stream, maxWaitForSecondStream time.Duration, log *zerolog.Logger) error {
|
||||
status := newBiStreamStatus()
|
||||
|
||||
go unidirectionalStream(downstream, upstream, "upstream->downstream", status, log)
|
||||
go unidirectionalStream(upstream, downstream, "downstream->upstream", status, log)
|
||||
|
||||
if err := status.wait(maxWaitForSecondStream); err != nil {
|
||||
return errors.Wrap(err, "unable to wait for both streams while proxying")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func unidirectionalStream(dst WriterCloser, src Reader, dir string, status *bidirectionalStreamStatus, log *zerolog.Logger) {
|
||||
defer func() {
|
||||
// The bidirectional streaming spawns 2 goroutines to stream each direction.
|
||||
// If any ends, the callstack returns, meaning the Tunnel request/stream (depending on http2 vs quic) will
|
||||
|
|
@ -58,28 +116,21 @@ func unidirectionalStream(dst io.Writer, src io.Reader, dir string, status *bidi
|
|||
// server/origin listens forever until closure), it may read/write from the underlying ReadWriter (backed by
|
||||
// the Edge<->cloudflared transport) in an unexpected state.
|
||||
// Because of this, we set this recover() logic.
|
||||
if r := recover(); r != nil {
|
||||
if err := recover(); err != nil {
|
||||
if status.isAnyDone() {
|
||||
// We handle such unexpected errors only when we detect that one side of the streaming is done.
|
||||
log.Debug().Msgf("Gracefully handled error %v in Streaming for %s, error %s", r, dir, debug.Stack())
|
||||
log.Debug().Msgf("recovered from panic in stream.Pipe for %s, error %s, %s", dir, err, debug.Stack())
|
||||
} else {
|
||||
// Otherwise, this is unexpected, but we prevent the program from crashing anyway.
|
||||
log.Warn().Msgf("Gracefully handled unexpected error %v in Streaming for %s, error %s", r, dir, debug.Stack())
|
||||
|
||||
tags := make(map[string]string)
|
||||
tags["root"] = "websocket.stream"
|
||||
tags["dir"] = dir
|
||||
switch rval := r.(type) {
|
||||
case error:
|
||||
raven.CaptureError(rval, tags)
|
||||
default:
|
||||
rvalStr := fmt.Sprint(rval)
|
||||
raven.CaptureMessage(rvalStr, tags)
|
||||
}
|
||||
log.Warn().Msgf("recovered from panic in stream.Pipe for %s, error %s, %s", dir, err, debug.Stack())
|
||||
sentry.CurrentHub().Recover(err)
|
||||
sentry.Flush(time.Second * 5)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
defer dst.CloseWrite()
|
||||
|
||||
_, err := copyData(dst, src, dir)
|
||||
if err != nil {
|
||||
log.Debug().Msgf("%s copy: %v", dir, err)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,122 @@
|
|||
package stream
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPipeBidirectionalFinishBothSides(t *testing.T) {
|
||||
fun := func(upstream, downstream *mockedStream) {
|
||||
downstream.closeReader()
|
||||
upstream.closeReader()
|
||||
}
|
||||
|
||||
testPipeBidirectionalUnblocking(t, fun, time.Millisecond*200, false)
|
||||
}
|
||||
|
||||
func TestPipeBidirectionalFinishOneSideTimeout(t *testing.T) {
|
||||
fun := func(upstream, downstream *mockedStream) {
|
||||
downstream.closeReader()
|
||||
}
|
||||
|
||||
testPipeBidirectionalUnblocking(t, fun, time.Millisecond*200, true)
|
||||
}
|
||||
|
||||
func TestPipeBidirectionalClosingWriteBothSidesAlsoExists(t *testing.T) {
|
||||
fun := func(upstream, downstream *mockedStream) {
|
||||
downstream.CloseWrite()
|
||||
upstream.CloseWrite()
|
||||
|
||||
downstream.writeToReader("abc")
|
||||
upstream.writeToReader("abc")
|
||||
}
|
||||
|
||||
testPipeBidirectionalUnblocking(t, fun, time.Millisecond*200, false)
|
||||
}
|
||||
|
||||
func TestPipeBidirectionalClosingWriteSingleSideAlsoExists(t *testing.T) {
|
||||
fun := func(upstream, downstream *mockedStream) {
|
||||
downstream.CloseWrite()
|
||||
|
||||
downstream.writeToReader("abc")
|
||||
upstream.writeToReader("abc")
|
||||
}
|
||||
|
||||
testPipeBidirectionalUnblocking(t, fun, time.Millisecond*200, true)
|
||||
}
|
||||
|
||||
func testPipeBidirectionalUnblocking(t *testing.T, afterFun func(*mockedStream, *mockedStream), timeout time.Duration, expectTimeout bool) {
|
||||
logger := zerolog.Nop()
|
||||
|
||||
downstream := newMockedStream()
|
||||
upstream := newMockedStream()
|
||||
|
||||
resultCh := make(chan error)
|
||||
go func() {
|
||||
resultCh <- PipeBidirectional(downstream, upstream, timeout, &logger)
|
||||
}()
|
||||
|
||||
afterFun(upstream, downstream)
|
||||
|
||||
select {
|
||||
case err := <-resultCh:
|
||||
if expectTimeout {
|
||||
require.NotNil(t, err)
|
||||
} else {
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
case <-time.After(timeout * 2):
|
||||
require.Fail(t, "test timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func newMockedStream() *mockedStream {
|
||||
return &mockedStream{
|
||||
readCh: make(chan *string),
|
||||
writeCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
type mockedStream struct {
|
||||
readCh chan *string
|
||||
writeCh chan struct{}
|
||||
|
||||
writeCloseOnce sync.Once
|
||||
}
|
||||
|
||||
func (m *mockedStream) Read(p []byte) (n int, err error) {
|
||||
result := <-m.readCh
|
||||
if result == nil {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
return len(*result), nil
|
||||
}
|
||||
|
||||
func (m *mockedStream) Write(p []byte) (n int, err error) {
|
||||
<-m.writeCh
|
||||
|
||||
return 0, fmt.Errorf("closed")
|
||||
}
|
||||
|
||||
func (m *mockedStream) CloseWrite() error {
|
||||
m.writeCloseOnce.Do(func() {
|
||||
close(m.writeCh)
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockedStream) closeReader() {
|
||||
close(m.readCh)
|
||||
}
|
||||
func (m *mockedStream) writeToReader(content string) {
|
||||
m.readCh <- &content
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ package supervisor
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -72,7 +72,7 @@ func fetchActiveIncidents() (incidents []Incident) {
|
|||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,6 +70,8 @@ type TunnelConfig struct {
|
|||
PacketConfig *ingress.GlobalRouterConfig
|
||||
|
||||
UDPUnregisterSessionTimeout time.Duration
|
||||
|
||||
DisableQUICPathMTUDiscovery bool
|
||||
}
|
||||
|
||||
func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
|
||||
|
|
@ -596,14 +598,15 @@ func (e *EdgeTunnelServer) serveQUIC(
|
|||
}
|
||||
|
||||
quicConfig := &quic.Config{
|
||||
HandshakeIdleTimeout: quicpogs.HandshakeIdleTimeout,
|
||||
MaxIdleTimeout: quicpogs.MaxIdleTimeout,
|
||||
KeepAlivePeriod: quicpogs.MaxIdlePingPeriod,
|
||||
MaxIncomingStreams: quicpogs.MaxIncomingStreams,
|
||||
MaxIncomingUniStreams: quicpogs.MaxIncomingStreams,
|
||||
EnableDatagrams: true,
|
||||
MaxDatagramFrameSize: quicpogs.MaxDatagramFrameSize,
|
||||
Tracer: quicpogs.NewClientTracer(connLogger.Logger(), connIndex),
|
||||
HandshakeIdleTimeout: quicpogs.HandshakeIdleTimeout,
|
||||
MaxIdleTimeout: quicpogs.MaxIdleTimeout,
|
||||
KeepAlivePeriod: quicpogs.MaxIdlePingPeriod,
|
||||
MaxIncomingStreams: quicpogs.MaxIncomingStreams,
|
||||
MaxIncomingUniStreams: quicpogs.MaxIncomingStreams,
|
||||
EnableDatagrams: true,
|
||||
MaxDatagramFrameSize: quicpogs.MaxDatagramFrameSize,
|
||||
Tracer: quicpogs.NewClientTracer(connLogger.Logger(), connIndex),
|
||||
DisablePathMTUDiscovery: e.config.DisableQUICPathMTUDiscovery,
|
||||
}
|
||||
|
||||
quicConn, err := connection.NewQUICConnection(
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
|
|
@ -77,7 +77,7 @@ func LoadOriginCA(originCAPoolFilename string, log *zerolog.Logger) (*x509.CertP
|
|||
|
||||
if originCAPoolFilename != "" {
|
||||
var err error
|
||||
originCustomCAPool, err = ioutil.ReadFile(originCAPoolFilename)
|
||||
originCustomCAPool, err = os.ReadFile(originCAPoolFilename)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, fmt.Sprintf("unable to read the file %s for --%s", originCAPoolFilename, OriginCAPoolFlag))
|
||||
}
|
||||
|
|
@ -116,7 +116,7 @@ func LoadCustomOriginCA(originCAFilename string) (*x509.CertPool, error) {
|
|||
return certPool, nil
|
||||
}
|
||||
|
||||
customOriginCA, err := ioutil.ReadFile(originCAFilename)
|
||||
customOriginCA, err := os.ReadFile(originCAFilename)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, fmt.Sprintf("unable to read the file %s", originCAFilename))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ package tlsconfig
|
|||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
|
@ -90,7 +90,7 @@ func GetConfig(p *TLSParameters) (*tls.Config, error) {
|
|||
func LoadCert(certPaths []string) (*x509.CertPool, error) {
|
||||
ca := x509.NewCertPool()
|
||||
for _, certPath := range certPaths {
|
||||
caCert, err := ioutil.ReadFile(certPath)
|
||||
caCert, err := os.ReadFile(certPath)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Error reading certificate %s", certPath)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
|
|
@ -13,9 +12,9 @@ import (
|
|||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/go-jose/go-jose/v3"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
"gopkg.in/square/go-jose.v2"
|
||||
|
||||
"github.com/cloudflare/cloudflared/config"
|
||||
"github.com/cloudflare/cloudflared/retry"
|
||||
|
|
@ -124,7 +123,7 @@ func (l *lock) Acquire() error {
|
|||
|
||||
// Create a lock file so other processes won't also try to get the token at
|
||||
// the same time
|
||||
if err := ioutil.WriteFile(l.lockFilePath, []byte{}, 0600); err != nil {
|
||||
if err := os.WriteFile(l.lockFilePath, []byte{}, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
|
@ -208,7 +207,7 @@ func getToken(appURL *url.URL, appInfo *AppInfo, useHostOnly bool, log *zerolog.
|
|||
log.Debug().Msgf("failed to exchange org token for app token: %s", err)
|
||||
} else {
|
||||
// generate app path
|
||||
if err := ioutil.WriteFile(appTokenPath, []byte(appToken), 0600); err != nil {
|
||||
if err := os.WriteFile(appTokenPath, []byte(appToken), 0600); err != nil {
|
||||
return "", errors.Wrap(err, "failed to write app token to disk")
|
||||
}
|
||||
return appToken, nil
|
||||
|
|
@ -237,12 +236,12 @@ func getTokensFromEdge(appURL *url.URL, appAUD, appTokenPath, orgTokenPath strin
|
|||
|
||||
// If we were able to get the auth domain and generate an org token path, lets write it to disk.
|
||||
if orgTokenPath != "" {
|
||||
if err := ioutil.WriteFile(orgTokenPath, []byte(resp.OrgToken), 0600); err != nil {
|
||||
if err := os.WriteFile(orgTokenPath, []byte(resp.OrgToken), 0600); err != nil {
|
||||
return "", errors.Wrap(err, "failed to write org token to disk")
|
||||
}
|
||||
}
|
||||
|
||||
if err := ioutil.WriteFile(appTokenPath, []byte(resp.AppToken), 0600); err != nil {
|
||||
if err := os.WriteFile(appTokenPath, []byte(resp.AppToken), 0600); err != nil {
|
||||
return "", errors.Wrap(err, "failed to write app token to disk")
|
||||
}
|
||||
|
||||
|
|
@ -389,7 +388,7 @@ func GetAppTokenIfExists(appInfo *AppInfo) (string, error) {
|
|||
|
||||
// GetTokenIfExists will return the token from local storage if it exists and not expired
|
||||
func getTokenIfExists(path string) (*jose.JSONWebSignature, error) {
|
||||
content, err := ioutil.ReadFile(path)
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import (
|
|||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
|
@ -105,7 +105,7 @@ func exchangeWireformat(msg []byte, endpoint *url.URL, client *http.Client) ([]b
|
|||
}
|
||||
|
||||
// Read wireformat response from the body
|
||||
buf, err := ioutil.ReadAll(resp.Body)
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read the response body")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,9 +9,9 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/net/idna"
|
||||
"gopkg.in/coreos/go-oidc.v2"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package validation
|
|||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
|
@ -329,7 +329,7 @@ func (f testRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
|
|||
func emptyResponse(statusCode int) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: statusCode,
|
||||
Body: ioutil.NopCloser(bytes.NewReader(nil)),
|
||||
Body: io.NopCloser(bytes.NewReader(nil)),
|
||||
Header: make(http.Header),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,373 +0,0 @@
|
|||
Mozilla Public License Version 2.0
|
||||
==================================
|
||||
|
||||
1. Definitions
|
||||
--------------
|
||||
|
||||
1.1. "Contributor"
|
||||
means each individual or legal entity that creates, contributes to
|
||||
the creation of, or owns Covered Software.
|
||||
|
||||
1.2. "Contributor Version"
|
||||
means the combination of the Contributions of others (if any) used
|
||||
by a Contributor and that particular Contributor's Contribution.
|
||||
|
||||
1.3. "Contribution"
|
||||
means Covered Software of a particular Contributor.
|
||||
|
||||
1.4. "Covered Software"
|
||||
means Source Code Form to which the initial Contributor has attached
|
||||
the notice in Exhibit A, the Executable Form of such Source Code
|
||||
Form, and Modifications of such Source Code Form, in each case
|
||||
including portions thereof.
|
||||
|
||||
1.5. "Incompatible With Secondary Licenses"
|
||||
means
|
||||
|
||||
(a) that the initial Contributor has attached the notice described
|
||||
in Exhibit B to the Covered Software; or
|
||||
|
||||
(b) that the Covered Software was made available under the terms of
|
||||
version 1.1 or earlier of the License, but not also under the
|
||||
terms of a Secondary License.
|
||||
|
||||
1.6. "Executable Form"
|
||||
means any form of the work other than Source Code Form.
|
||||
|
||||
1.7. "Larger Work"
|
||||
means a work that combines Covered Software with other material, in
|
||||
a separate file or files, that is not Covered Software.
|
||||
|
||||
1.8. "License"
|
||||
means this document.
|
||||
|
||||
1.9. "Licensable"
|
||||
means having the right to grant, to the maximum extent possible,
|
||||
whether at the time of the initial grant or subsequently, any and
|
||||
all of the rights conveyed by this License.
|
||||
|
||||
1.10. "Modifications"
|
||||
means any of the following:
|
||||
|
||||
(a) any file in Source Code Form that results from an addition to,
|
||||
deletion from, or modification of the contents of Covered
|
||||
Software; or
|
||||
|
||||
(b) any new file in Source Code Form that contains any Covered
|
||||
Software.
|
||||
|
||||
1.11. "Patent Claims" of a Contributor
|
||||
means any patent claim(s), including without limitation, method,
|
||||
process, and apparatus claims, in any patent Licensable by such
|
||||
Contributor that would be infringed, but for the grant of the
|
||||
License, by the making, using, selling, offering for sale, having
|
||||
made, import, or transfer of either its Contributions or its
|
||||
Contributor Version.
|
||||
|
||||
1.12. "Secondary License"
|
||||
means either the GNU General Public License, Version 2.0, the GNU
|
||||
Lesser General Public License, Version 2.1, the GNU Affero General
|
||||
Public License, Version 3.0, or any later versions of those
|
||||
licenses.
|
||||
|
||||
1.13. "Source Code Form"
|
||||
means the form of the work preferred for making modifications.
|
||||
|
||||
1.14. "You" (or "Your")
|
||||
means an individual or a legal entity exercising rights under this
|
||||
License. For legal entities, "You" includes any entity that
|
||||
controls, is controlled by, or is under common control with You. For
|
||||
purposes of this definition, "control" means (a) the power, direct
|
||||
or indirect, to cause the direction or management of such entity,
|
||||
whether by contract or otherwise, or (b) ownership of more than
|
||||
fifty percent (50%) of the outstanding shares or beneficial
|
||||
ownership of such entity.
|
||||
|
||||
2. License Grants and Conditions
|
||||
--------------------------------
|
||||
|
||||
2.1. Grants
|
||||
|
||||
Each Contributor hereby grants You a world-wide, royalty-free,
|
||||
non-exclusive license:
|
||||
|
||||
(a) under intellectual property rights (other than patent or trademark)
|
||||
Licensable by such Contributor to use, reproduce, make available,
|
||||
modify, display, perform, distribute, and otherwise exploit its
|
||||
Contributions, either on an unmodified basis, with Modifications, or
|
||||
as part of a Larger Work; and
|
||||
|
||||
(b) under Patent Claims of such Contributor to make, use, sell, offer
|
||||
for sale, have made, import, and otherwise transfer either its
|
||||
Contributions or its Contributor Version.
|
||||
|
||||
2.2. Effective Date
|
||||
|
||||
The licenses granted in Section 2.1 with respect to any Contribution
|
||||
become effective for each Contribution on the date the Contributor first
|
||||
distributes such Contribution.
|
||||
|
||||
2.3. Limitations on Grant Scope
|
||||
|
||||
The licenses granted in this Section 2 are the only rights granted under
|
||||
this License. No additional rights or licenses will be implied from the
|
||||
distribution or licensing of Covered Software under this License.
|
||||
Notwithstanding Section 2.1(b) above, no patent license is granted by a
|
||||
Contributor:
|
||||
|
||||
(a) for any code that a Contributor has removed from Covered Software;
|
||||
or
|
||||
|
||||
(b) for infringements caused by: (i) Your and any other third party's
|
||||
modifications of Covered Software, or (ii) the combination of its
|
||||
Contributions with other software (except as part of its Contributor
|
||||
Version); or
|
||||
|
||||
(c) under Patent Claims infringed by Covered Software in the absence of
|
||||
its Contributions.
|
||||
|
||||
This License does not grant any rights in the trademarks, service marks,
|
||||
or logos of any Contributor (except as may be necessary to comply with
|
||||
the notice requirements in Section 3.4).
|
||||
|
||||
2.4. Subsequent Licenses
|
||||
|
||||
No Contributor makes additional grants as a result of Your choice to
|
||||
distribute the Covered Software under a subsequent version of this
|
||||
License (see Section 10.2) or under the terms of a Secondary License (if
|
||||
permitted under the terms of Section 3.3).
|
||||
|
||||
2.5. Representation
|
||||
|
||||
Each Contributor represents that the Contributor believes its
|
||||
Contributions are its original creation(s) or it has sufficient rights
|
||||
to grant the rights to its Contributions conveyed by this License.
|
||||
|
||||
2.6. Fair Use
|
||||
|
||||
This License is not intended to limit any rights You have under
|
||||
applicable copyright doctrines of fair use, fair dealing, or other
|
||||
equivalents.
|
||||
|
||||
2.7. Conditions
|
||||
|
||||
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
|
||||
in Section 2.1.
|
||||
|
||||
3. Responsibilities
|
||||
-------------------
|
||||
|
||||
3.1. Distribution of Source Form
|
||||
|
||||
All distribution of Covered Software in Source Code Form, including any
|
||||
Modifications that You create or to which You contribute, must be under
|
||||
the terms of this License. You must inform recipients that the Source
|
||||
Code Form of the Covered Software is governed by the terms of this
|
||||
License, and how they can obtain a copy of this License. You may not
|
||||
attempt to alter or restrict the recipients' rights in the Source Code
|
||||
Form.
|
||||
|
||||
3.2. Distribution of Executable Form
|
||||
|
||||
If You distribute Covered Software in Executable Form then:
|
||||
|
||||
(a) such Covered Software must also be made available in Source Code
|
||||
Form, as described in Section 3.1, and You must inform recipients of
|
||||
the Executable Form how they can obtain a copy of such Source Code
|
||||
Form by reasonable means in a timely manner, at a charge no more
|
||||
than the cost of distribution to the recipient; and
|
||||
|
||||
(b) You may distribute such Executable Form under the terms of this
|
||||
License, or sublicense it under different terms, provided that the
|
||||
license for the Executable Form does not attempt to limit or alter
|
||||
the recipients' rights in the Source Code Form under this License.
|
||||
|
||||
3.3. Distribution of a Larger Work
|
||||
|
||||
You may create and distribute a Larger Work under terms of Your choice,
|
||||
provided that You also comply with the requirements of this License for
|
||||
the Covered Software. If the Larger Work is a combination of Covered
|
||||
Software with a work governed by one or more Secondary Licenses, and the
|
||||
Covered Software is not Incompatible With Secondary Licenses, this
|
||||
License permits You to additionally distribute such Covered Software
|
||||
under the terms of such Secondary License(s), so that the recipient of
|
||||
the Larger Work may, at their option, further distribute the Covered
|
||||
Software under the terms of either this License or such Secondary
|
||||
License(s).
|
||||
|
||||
3.4. Notices
|
||||
|
||||
You may not remove or alter the substance of any license notices
|
||||
(including copyright notices, patent notices, disclaimers of warranty,
|
||||
or limitations of liability) contained within the Source Code Form of
|
||||
the Covered Software, except that You may alter any license notices to
|
||||
the extent required to remedy known factual inaccuracies.
|
||||
|
||||
3.5. Application of Additional Terms
|
||||
|
||||
You may choose to offer, and to charge a fee for, warranty, support,
|
||||
indemnity or liability obligations to one or more recipients of Covered
|
||||
Software. However, You may do so only on Your own behalf, and not on
|
||||
behalf of any Contributor. You must make it absolutely clear that any
|
||||
such warranty, support, indemnity, or liability obligation is offered by
|
||||
You alone, and You hereby agree to indemnify every Contributor for any
|
||||
liability incurred by such Contributor as a result of warranty, support,
|
||||
indemnity or liability terms You offer. You may include additional
|
||||
disclaimers of warranty and limitations of liability specific to any
|
||||
jurisdiction.
|
||||
|
||||
4. Inability to Comply Due to Statute or Regulation
|
||||
---------------------------------------------------
|
||||
|
||||
If it is impossible for You to comply with any of the terms of this
|
||||
License with respect to some or all of the Covered Software due to
|
||||
statute, judicial order, or regulation then You must: (a) comply with
|
||||
the terms of this License to the maximum extent possible; and (b)
|
||||
describe the limitations and the code they affect. Such description must
|
||||
be placed in a text file included with all distributions of the Covered
|
||||
Software under this License. Except to the extent prohibited by statute
|
||||
or regulation, such description must be sufficiently detailed for a
|
||||
recipient of ordinary skill to be able to understand it.
|
||||
|
||||
5. Termination
|
||||
--------------
|
||||
|
||||
5.1. The rights granted under this License will terminate automatically
|
||||
if You fail to comply with any of its terms. However, if You become
|
||||
compliant, then the rights granted under this License from a particular
|
||||
Contributor are reinstated (a) provisionally, unless and until such
|
||||
Contributor explicitly and finally terminates Your grants, and (b) on an
|
||||
ongoing basis, if such Contributor fails to notify You of the
|
||||
non-compliance by some reasonable means prior to 60 days after You have
|
||||
come back into compliance. Moreover, Your grants from a particular
|
||||
Contributor are reinstated on an ongoing basis if such Contributor
|
||||
notifies You of the non-compliance by some reasonable means, this is the
|
||||
first time You have received notice of non-compliance with this License
|
||||
from such Contributor, and You become compliant prior to 30 days after
|
||||
Your receipt of the notice.
|
||||
|
||||
5.2. If You initiate litigation against any entity by asserting a patent
|
||||
infringement claim (excluding declaratory judgment actions,
|
||||
counter-claims, and cross-claims) alleging that a Contributor Version
|
||||
directly or indirectly infringes any patent, then the rights granted to
|
||||
You by any and all Contributors for the Covered Software under Section
|
||||
2.1 of this License shall terminate.
|
||||
|
||||
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
|
||||
end user license agreements (excluding distributors and resellers) which
|
||||
have been validly granted by You or Your distributors under this License
|
||||
prior to termination shall survive termination.
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 6. Disclaimer of Warranty *
|
||||
* ------------------------- *
|
||||
* *
|
||||
* Covered Software is provided under this License on an "as is" *
|
||||
* basis, without warranty of any kind, either expressed, implied, or *
|
||||
* statutory, including, without limitation, warranties that the *
|
||||
* Covered Software is free of defects, merchantable, fit for a *
|
||||
* particular purpose or non-infringing. The entire risk as to the *
|
||||
* quality and performance of the Covered Software is with You. *
|
||||
* Should any Covered Software prove defective in any respect, You *
|
||||
* (not any Contributor) assume the cost of any necessary servicing, *
|
||||
* repair, or correction. This disclaimer of warranty constitutes an *
|
||||
* essential part of this License. No use of any Covered Software is *
|
||||
* authorized under this License except under this disclaimer. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 7. Limitation of Liability *
|
||||
* -------------------------- *
|
||||
* *
|
||||
* Under no circumstances and under no legal theory, whether tort *
|
||||
* (including negligence), contract, or otherwise, shall any *
|
||||
* Contributor, or anyone who distributes Covered Software as *
|
||||
* permitted above, be liable to You for any direct, indirect, *
|
||||
* special, incidental, or consequential damages of any character *
|
||||
* including, without limitation, damages for lost profits, loss of *
|
||||
* goodwill, work stoppage, computer failure or malfunction, or any *
|
||||
* and all other commercial damages or losses, even if such party *
|
||||
* shall have been informed of the possibility of such damages. This *
|
||||
* limitation of liability shall not apply to liability for death or *
|
||||
* personal injury resulting from such party's negligence to the *
|
||||
* extent applicable law prohibits such limitation. Some *
|
||||
* jurisdictions do not allow the exclusion or limitation of *
|
||||
* incidental or consequential damages, so this exclusion and *
|
||||
* limitation may not apply to You. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
8. Litigation
|
||||
-------------
|
||||
|
||||
Any litigation relating to this License may be brought only in the
|
||||
courts of a jurisdiction where the defendant maintains its principal
|
||||
place of business and such litigation shall be governed by laws of that
|
||||
jurisdiction, without reference to its conflict-of-law provisions.
|
||||
Nothing in this Section shall prevent a party's ability to bring
|
||||
cross-claims or counter-claims.
|
||||
|
||||
9. Miscellaneous
|
||||
----------------
|
||||
|
||||
This License represents the complete agreement concerning the subject
|
||||
matter hereof. If any provision of this License is held to be
|
||||
unenforceable, such provision shall be reformed only to the extent
|
||||
necessary to make it enforceable. Any law or regulation which provides
|
||||
that the language of a contract shall be construed against the drafter
|
||||
shall not be used to construe this License against a Contributor.
|
||||
|
||||
10. Versions of the License
|
||||
---------------------------
|
||||
|
||||
10.1. New Versions
|
||||
|
||||
Mozilla Foundation is the license steward. Except as provided in Section
|
||||
10.3, no one other than the license steward has the right to modify or
|
||||
publish new versions of this License. Each version will be given a
|
||||
distinguishing version number.
|
||||
|
||||
10.2. Effect of New Versions
|
||||
|
||||
You may distribute the Covered Software under the terms of the version
|
||||
of the License under which You originally received the Covered Software,
|
||||
or under the terms of any subsequent version published by the license
|
||||
steward.
|
||||
|
||||
10.3. Modified Versions
|
||||
|
||||
If you create software not governed by this License, and you want to
|
||||
create a new license for such software, you may create and use a
|
||||
modified version of this License if you rename the license and remove
|
||||
any references to the name of the license steward (except to note that
|
||||
such modified license differs from this License).
|
||||
|
||||
10.4. Distributing Source Code Form that is Incompatible With Secondary
|
||||
Licenses
|
||||
|
||||
If You choose to distribute Source Code Form that is Incompatible With
|
||||
Secondary Licenses under the terms of this version of the License, the
|
||||
notice described in Exhibit B of this License must be attached.
|
||||
|
||||
Exhibit A - Source Code Form License Notice
|
||||
-------------------------------------------
|
||||
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
If it is not possible or desirable to put the notice in a particular
|
||||
file, then You may include the notice in a location (such as a LICENSE
|
||||
file in a relevant directory) where a recipient would be likely to look
|
||||
for such a notice.
|
||||
|
||||
You may add additional accurate notices of copyright ownership.
|
||||
|
||||
Exhibit B - "Incompatible With Secondary Licenses" Notice
|
||||
---------------------------------------------------------
|
||||
|
||||
This Source Code Form is "Incompatible With Secondary Licenses", as
|
||||
defined by the Mozilla Public License, v. 2.0.
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
# GoCertifi: SSL Certificates for Golang
|
||||
|
||||
This Go package contains a CA bundle that you can reference in your Go code.
|
||||
This is useful for systems that do not have CA bundles that Golang can find
|
||||
itself, or where a uniform set of CAs is valuable.
|
||||
|
||||
This is the same CA bundle that ships with the
|
||||
[Python Requests](https://github.com/kennethreitz/requests) library, and is a
|
||||
Golang specific port of [certifi](https://github.com/kennethreitz/certifi). The
|
||||
CA bundle is derived from Mozilla's canonical set.
|
||||
|
||||
## Usage
|
||||
|
||||
You can use the `gocertifi` package as follows:
|
||||
|
||||
```go
|
||||
import "github.com/certifi/gocertifi"
|
||||
|
||||
certPool, err := gocertifi.CACerts()
|
||||
```
|
||||
|
||||
You can use the returned `*x509.CertPool` as part of an HTTP transport, for example:
|
||||
|
||||
```go
|
||||
import (
|
||||
"net/http"
|
||||
"crypto/tls"
|
||||
)
|
||||
|
||||
// Setup an HTTP client with a custom transport
|
||||
transport := &http.Transport{
|
||||
Proxy: ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
DualStack: true,
|
||||
}).DialContext,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 100,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
}
|
||||
// or, starting with go1.13 simply use:
|
||||
// transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
|
||||
transport.TLSClientConfig = &tls.Config{RootCAs: certPool}
|
||||
client := &http.Client{Transport: transport}
|
||||
|
||||
// Make an HTTP request using our custom transport
|
||||
resp, err := client.Get("https://example.com")
|
||||
```
|
||||
|
||||
## Detailed Documentation
|
||||
|
||||
Import as follows:
|
||||
|
||||
```go
|
||||
import "github.com/certifi/gocertifi"
|
||||
```
|
||||
|
||||
### Functions
|
||||
|
||||
```go
|
||||
func CACerts() (*x509.CertPool, error)
|
||||
```
|
||||
CACerts builds an X.509 certificate pool containing the Mozilla CA Certificate
|
||||
bundle. This can't actually error and always returns successfully with `nil`
|
||||
as the error. This will be replaced in `v2` to only return the `CertPool`.
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,130 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function to find backward reference copies. */
|
||||
|
||||
#include "./enc/backward_references.h"
|
||||
|
||||
#include "./common/constants.h"
|
||||
#include "./common/dictionary.h"
|
||||
#include <brotli/types.h>
|
||||
#include "./enc/command.h"
|
||||
#include "./enc/dictionary_hash.h"
|
||||
#include "./enc/memory.h"
|
||||
#include "./enc/port.h"
|
||||
#include "./enc/quality.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
|
||||
size_t max_distance,
|
||||
const int* dist_cache) {
|
||||
if (distance <= max_distance) {
|
||||
size_t distance_plus_3 = distance + 3;
|
||||
size_t offset0 = distance_plus_3 - (size_t)dist_cache[0];
|
||||
size_t offset1 = distance_plus_3 - (size_t)dist_cache[1];
|
||||
if (distance == (size_t)dist_cache[0]) {
|
||||
return 0;
|
||||
} else if (distance == (size_t)dist_cache[1]) {
|
||||
return 1;
|
||||
} else if (offset0 < 7) {
|
||||
return (0x9750468 >> (4 * offset0)) & 0xF;
|
||||
} else if (offset1 < 7) {
|
||||
return (0xFDB1ACE >> (4 * offset1)) & 0xF;
|
||||
} else if (distance == (size_t)dist_cache[2]) {
|
||||
return 2;
|
||||
} else if (distance == (size_t)dist_cache[3]) {
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
return distance + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
|
||||
}
|
||||
|
||||
#define EXPAND_CAT(a, b) CAT(a, b)
|
||||
#define CAT(a, b) a ## b
|
||||
#define FN(X) EXPAND_CAT(X, HASHER())
|
||||
|
||||
#define HASHER() H2
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/backward_references_inc.h"
|
||||
#undef HASHER
|
||||
|
||||
#define HASHER() H3
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/backward_references_inc.h"
|
||||
#undef HASHER
|
||||
|
||||
#define HASHER() H4
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/backward_references_inc.h"
|
||||
#undef HASHER
|
||||
|
||||
#define HASHER() H5
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/backward_references_inc.h"
|
||||
#undef HASHER
|
||||
|
||||
#define HASHER() H6
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/backward_references_inc.h"
|
||||
#undef HASHER
|
||||
|
||||
#define HASHER() H40
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/backward_references_inc.h"
|
||||
#undef HASHER
|
||||
|
||||
#define HASHER() H41
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/backward_references_inc.h"
|
||||
#undef HASHER
|
||||
|
||||
#define HASHER() H42
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/backward_references_inc.h"
|
||||
#undef HASHER
|
||||
|
||||
#define HASHER() H54
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/backward_references_inc.h"
|
||||
#undef HASHER
|
||||
|
||||
#undef FN
|
||||
#undef CAT
|
||||
#undef EXPAND_CAT
|
||||
|
||||
void BrotliCreateBackwardReferences(const BrotliDictionary* dictionary,
|
||||
size_t num_bytes,
|
||||
size_t position,
|
||||
const uint8_t* ringbuffer,
|
||||
size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params,
|
||||
HasherHandle hasher,
|
||||
int* dist_cache,
|
||||
size_t* last_insert_len,
|
||||
Command* commands,
|
||||
size_t* num_commands,
|
||||
size_t* num_literals) {
|
||||
switch (params->hasher.type) {
|
||||
#define CASE_(N) \
|
||||
case N: \
|
||||
CreateBackwardReferencesH ## N(dictionary, \
|
||||
kStaticDictionaryHash, num_bytes, position, ringbuffer, \
|
||||
ringbuffer_mask, params, hasher, dist_cache, \
|
||||
last_insert_len, commands, num_commands, num_literals); \
|
||||
break;
|
||||
FOR_GENERIC_HASHERS(CASE_)
|
||||
#undef CASE_
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
|
@ -1,790 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function to find backward reference copies. */
|
||||
|
||||
#include "./enc/backward_references_hq.h"
|
||||
|
||||
#include <string.h> /* memcpy, memset */
|
||||
|
||||
#include "./common/constants.h"
|
||||
#include <brotli/types.h>
|
||||
#include "./enc/command.h"
|
||||
#include "./enc/fast_log.h"
|
||||
#include "./enc/find_match_length.h"
|
||||
#include "./enc/literal_cost.h"
|
||||
#include "./enc/memory.h"
|
||||
#include "./enc/port.h"
|
||||
#include "./enc/prefix.h"
|
||||
#include "./enc/quality.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static const float kInfinity = 1.7e38f; /* ~= 2 ^ 127 */
|
||||
|
||||
static const uint32_t kDistanceCacheIndex[] = {
|
||||
0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1,
|
||||
};
|
||||
static const int kDistanceCacheOffset[] = {
|
||||
0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3
|
||||
};
|
||||
|
||||
void BrotliInitZopfliNodes(ZopfliNode* array, size_t length) {
|
||||
ZopfliNode stub;
|
||||
size_t i;
|
||||
stub.length = 1;
|
||||
stub.distance = 0;
|
||||
stub.insert_length = 0;
|
||||
stub.u.cost = kInfinity;
|
||||
for (i = 0; i < length; ++i) array[i] = stub;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t ZopfliNodeCopyLength(const ZopfliNode* self) {
|
||||
return self->length & 0xffffff;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t ZopfliNodeLengthCode(const ZopfliNode* self) {
|
||||
const uint32_t modifier = self->length >> 24;
|
||||
return ZopfliNodeCopyLength(self) + 9u - modifier;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t ZopfliNodeCopyDistance(const ZopfliNode* self) {
|
||||
return self->distance & 0x1ffffff;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t ZopfliNodeDistanceCode(const ZopfliNode* self) {
|
||||
const uint32_t short_code = self->distance >> 25;
|
||||
return short_code == 0 ?
|
||||
ZopfliNodeCopyDistance(self) + BROTLI_NUM_DISTANCE_SHORT_CODES - 1 :
|
||||
short_code - 1;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t ZopfliNodeCommandLength(const ZopfliNode* self) {
|
||||
return ZopfliNodeCopyLength(self) + self->insert_length;
|
||||
}
|
||||
|
||||
/* Histogram based cost model for zopflification. */
|
||||
typedef struct ZopfliCostModel {
|
||||
/* The insert and copy length symbols. */
|
||||
float cost_cmd_[BROTLI_NUM_COMMAND_SYMBOLS];
|
||||
float cost_dist_[BROTLI_NUM_DISTANCE_SYMBOLS];
|
||||
/* Cumulative costs of literals per position in the stream. */
|
||||
float* literal_costs_;
|
||||
float min_cost_cmd_;
|
||||
size_t num_bytes_;
|
||||
} ZopfliCostModel;
|
||||
|
||||
static void InitZopfliCostModel(
|
||||
MemoryManager* m, ZopfliCostModel* self, size_t num_bytes) {
|
||||
self->num_bytes_ = num_bytes;
|
||||
self->literal_costs_ = BROTLI_ALLOC(m, float, num_bytes + 2);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
}
|
||||
|
||||
static void CleanupZopfliCostModel(MemoryManager* m, ZopfliCostModel* self) {
|
||||
BROTLI_FREE(m, self->literal_costs_);
|
||||
}
|
||||
|
||||
static void SetCost(const uint32_t* histogram, size_t histogram_size,
|
||||
float* cost) {
|
||||
size_t sum = 0;
|
||||
float log2sum;
|
||||
size_t i;
|
||||
for (i = 0; i < histogram_size; i++) {
|
||||
sum += histogram[i];
|
||||
}
|
||||
log2sum = (float)FastLog2(sum);
|
||||
for (i = 0; i < histogram_size; i++) {
|
||||
if (histogram[i] == 0) {
|
||||
cost[i] = log2sum + 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Shannon bits for this symbol. */
|
||||
cost[i] = log2sum - (float)FastLog2(histogram[i]);
|
||||
|
||||
/* Cannot be coded with less than 1 bit */
|
||||
if (cost[i] < 1) cost[i] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
static void ZopfliCostModelSetFromCommands(ZopfliCostModel* self,
|
||||
size_t position,
|
||||
const uint8_t* ringbuffer,
|
||||
size_t ringbuffer_mask,
|
||||
const Command* commands,
|
||||
size_t num_commands,
|
||||
size_t last_insert_len) {
|
||||
uint32_t histogram_literal[BROTLI_NUM_LITERAL_SYMBOLS];
|
||||
uint32_t histogram_cmd[BROTLI_NUM_COMMAND_SYMBOLS];
|
||||
uint32_t histogram_dist[BROTLI_NUM_DISTANCE_SYMBOLS];
|
||||
float cost_literal[BROTLI_NUM_LITERAL_SYMBOLS];
|
||||
size_t pos = position - last_insert_len;
|
||||
float min_cost_cmd = kInfinity;
|
||||
size_t i;
|
||||
float* cost_cmd = self->cost_cmd_;
|
||||
|
||||
memset(histogram_literal, 0, sizeof(histogram_literal));
|
||||
memset(histogram_cmd, 0, sizeof(histogram_cmd));
|
||||
memset(histogram_dist, 0, sizeof(histogram_dist));
|
||||
|
||||
for (i = 0; i < num_commands; i++) {
|
||||
size_t inslength = commands[i].insert_len_;
|
||||
size_t copylength = CommandCopyLen(&commands[i]);
|
||||
size_t distcode = commands[i].dist_prefix_;
|
||||
size_t cmdcode = commands[i].cmd_prefix_;
|
||||
size_t j;
|
||||
|
||||
histogram_cmd[cmdcode]++;
|
||||
if (cmdcode >= 128) histogram_dist[distcode]++;
|
||||
|
||||
for (j = 0; j < inslength; j++) {
|
||||
histogram_literal[ringbuffer[(pos + j) & ringbuffer_mask]]++;
|
||||
}
|
||||
|
||||
pos += inslength + copylength;
|
||||
}
|
||||
|
||||
SetCost(histogram_literal, BROTLI_NUM_LITERAL_SYMBOLS, cost_literal);
|
||||
SetCost(histogram_cmd, BROTLI_NUM_COMMAND_SYMBOLS, cost_cmd);
|
||||
SetCost(histogram_dist, BROTLI_NUM_DISTANCE_SYMBOLS, self->cost_dist_);
|
||||
|
||||
for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i) {
|
||||
min_cost_cmd = BROTLI_MIN(float, min_cost_cmd, cost_cmd[i]);
|
||||
}
|
||||
self->min_cost_cmd_ = min_cost_cmd;
|
||||
|
||||
{
|
||||
float* literal_costs = self->literal_costs_;
|
||||
size_t num_bytes = self->num_bytes_;
|
||||
literal_costs[0] = 0.0;
|
||||
for (i = 0; i < num_bytes; ++i) {
|
||||
literal_costs[i + 1] = literal_costs[i] +
|
||||
cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void ZopfliCostModelSetFromLiteralCosts(ZopfliCostModel* self,
|
||||
size_t position,
|
||||
const uint8_t* ringbuffer,
|
||||
size_t ringbuffer_mask) {
|
||||
float* literal_costs = self->literal_costs_;
|
||||
float* cost_dist = self->cost_dist_;
|
||||
float* cost_cmd = self->cost_cmd_;
|
||||
size_t num_bytes = self->num_bytes_;
|
||||
size_t i;
|
||||
BrotliEstimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask,
|
||||
ringbuffer, &literal_costs[1]);
|
||||
literal_costs[0] = 0.0;
|
||||
for (i = 0; i < num_bytes; ++i) {
|
||||
literal_costs[i + 1] += literal_costs[i];
|
||||
}
|
||||
for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i) {
|
||||
cost_cmd[i] = (float)FastLog2(11 + (uint32_t)i);
|
||||
}
|
||||
for (i = 0; i < BROTLI_NUM_DISTANCE_SYMBOLS; ++i) {
|
||||
cost_dist[i] = (float)FastLog2(20 + (uint32_t)i);
|
||||
}
|
||||
self->min_cost_cmd_ = (float)FastLog2(11);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE float ZopfliCostModelGetCommandCost(
|
||||
const ZopfliCostModel* self, uint16_t cmdcode) {
|
||||
return self->cost_cmd_[cmdcode];
|
||||
}
|
||||
|
||||
static BROTLI_INLINE float ZopfliCostModelGetDistanceCost(
|
||||
const ZopfliCostModel* self, size_t distcode) {
|
||||
return self->cost_dist_[distcode];
|
||||
}
|
||||
|
||||
static BROTLI_INLINE float ZopfliCostModelGetLiteralCosts(
|
||||
const ZopfliCostModel* self, size_t from, size_t to) {
|
||||
return self->literal_costs_[to] - self->literal_costs_[from];
|
||||
}
|
||||
|
||||
static BROTLI_INLINE float ZopfliCostModelGetMinCostCmd(
|
||||
const ZopfliCostModel* self) {
|
||||
return self->min_cost_cmd_;
|
||||
}
|
||||
|
||||
/* REQUIRES: len >= 2, start_pos <= pos */
|
||||
/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */
|
||||
/* Maintains the "ZopfliNode array invariant". */
|
||||
static BROTLI_INLINE void UpdateZopfliNode(ZopfliNode* nodes, size_t pos,
|
||||
size_t start_pos, size_t len, size_t len_code, size_t dist,
|
||||
size_t short_code, float cost) {
|
||||
ZopfliNode* next = &nodes[pos + len];
|
||||
next->length = (uint32_t)(len | ((len + 9u - len_code) << 24));
|
||||
next->distance = (uint32_t)(dist | (short_code << 25));
|
||||
next->insert_length = (uint32_t)(pos - start_pos);
|
||||
next->u.cost = cost;
|
||||
}
|
||||
|
||||
typedef struct PosData {
|
||||
size_t pos;
|
||||
int distance_cache[4];
|
||||
float costdiff;
|
||||
float cost;
|
||||
} PosData;
|
||||
|
||||
/* Maintains the smallest 8 cost difference together with their positions */
|
||||
typedef struct StartPosQueue {
|
||||
PosData q_[8];
|
||||
size_t idx_;
|
||||
} StartPosQueue;
|
||||
|
||||
static BROTLI_INLINE void InitStartPosQueue(StartPosQueue* self) {
|
||||
self->idx_ = 0;
|
||||
}
|
||||
|
||||
static size_t StartPosQueueSize(const StartPosQueue* self) {
|
||||
return BROTLI_MIN(size_t, self->idx_, 8);
|
||||
}
|
||||
|
||||
static void StartPosQueuePush(StartPosQueue* self, const PosData* posdata) {
|
||||
size_t offset = ~(self->idx_++) & 7;
|
||||
size_t len = StartPosQueueSize(self);
|
||||
size_t i;
|
||||
PosData* q = self->q_;
|
||||
q[offset] = *posdata;
|
||||
/* Restore the sorted order. In the list of |len| items at most |len - 1|
|
||||
adjacent element comparisons / swaps are required. */
|
||||
for (i = 1; i < len; ++i) {
|
||||
if (q[offset & 7].costdiff > q[(offset + 1) & 7].costdiff) {
|
||||
BROTLI_SWAP(PosData, q, offset & 7, (offset + 1) & 7);
|
||||
}
|
||||
++offset;
|
||||
}
|
||||
}
|
||||
|
||||
static const PosData* StartPosQueueAt(const StartPosQueue* self, size_t k) {
|
||||
return &self->q_[(k - self->idx_) & 7];
|
||||
}
|
||||
|
||||
/* Returns the minimum possible copy length that can improve the cost of any */
|
||||
/* future position. */
|
||||
static size_t ComputeMinimumCopyLength(const float start_cost,
|
||||
const ZopfliNode* nodes,
|
||||
const size_t num_bytes,
|
||||
const size_t pos) {
|
||||
/* Compute the minimum possible cost of reaching any future position. */
|
||||
float min_cost = start_cost;
|
||||
size_t len = 2;
|
||||
size_t next_len_bucket = 4;
|
||||
size_t next_len_offset = 10;
|
||||
while (pos + len <= num_bytes && nodes[pos + len].u.cost <= min_cost) {
|
||||
/* We already reached (pos + len) with no more cost than the minimum
|
||||
possible cost of reaching anything from this pos, so there is no point in
|
||||
looking for lengths <= len. */
|
||||
++len;
|
||||
if (len == next_len_offset) {
|
||||
/* We reached the next copy length code bucket, so we add one more
|
||||
extra bit to the minimum cost. */
|
||||
min_cost += 1.0f;
|
||||
next_len_offset += next_len_bucket;
|
||||
next_len_bucket *= 2;
|
||||
}
|
||||
}
|
||||
return len;
|
||||
}
|
||||
|
||||
/* REQUIRES: nodes[pos].cost < kInfinity
|
||||
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
|
||||
static uint32_t ComputeDistanceShortcut(const size_t block_start,
|
||||
const size_t pos,
|
||||
const size_t max_backward,
|
||||
const ZopfliNode* nodes) {
|
||||
const size_t clen = ZopfliNodeCopyLength(&nodes[pos]);
|
||||
const size_t ilen = nodes[pos].insert_length;
|
||||
const size_t dist = ZopfliNodeCopyDistance(&nodes[pos]);
|
||||
/* Since |block_start + pos| is the end position of the command, the copy part
|
||||
starts from |block_start + pos - clen|. Distances that are greater than
|
||||
this or greater than |max_backward| are static dictionary references, and
|
||||
do not update the last distances. Also distance code 0 (last distance)
|
||||
does not update the last distances. */
|
||||
if (pos == 0) {
|
||||
return 0;
|
||||
} else if (dist + clen <= block_start + pos &&
|
||||
dist <= max_backward &&
|
||||
ZopfliNodeDistanceCode(&nodes[pos]) > 0) {
|
||||
return (uint32_t)pos;
|
||||
} else {
|
||||
return nodes[pos - clen - ilen].u.shortcut;
|
||||
}
|
||||
}
|
||||
|
||||
/* Fills in dist_cache[0..3] with the last four distances (as defined by
|
||||
Section 4. of the Spec) that would be used at (block_start + pos) if we
|
||||
used the shortest path of commands from block_start, computed from
|
||||
nodes[0..pos]. The last four distances at block_start are in
|
||||
starting_dist_cache[0..3].
|
||||
REQUIRES: nodes[pos].cost < kInfinity
|
||||
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
|
||||
static void ComputeDistanceCache(const size_t pos,
|
||||
const int* starting_dist_cache,
|
||||
const ZopfliNode* nodes,
|
||||
int* dist_cache) {
|
||||
int idx = 0;
|
||||
size_t p = nodes[pos].u.shortcut;
|
||||
while (idx < 4 && p > 0) {
|
||||
const size_t ilen = nodes[p].insert_length;
|
||||
const size_t clen = ZopfliNodeCopyLength(&nodes[p]);
|
||||
const size_t dist = ZopfliNodeCopyDistance(&nodes[p]);
|
||||
dist_cache[idx++] = (int)dist;
|
||||
/* Because of prerequisite, p >= clen + ilen >= 2. */
|
||||
p = nodes[p - clen - ilen].u.shortcut;
|
||||
}
|
||||
for (; idx < 4; ++idx) {
|
||||
dist_cache[idx] = *starting_dist_cache++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it
|
||||
is eligible. */
|
||||
static void EvaluateNode(
|
||||
const size_t block_start, const size_t pos, const size_t max_backward_limit,
|
||||
const int* starting_dist_cache, const ZopfliCostModel* model,
|
||||
StartPosQueue* queue, ZopfliNode* nodes) {
|
||||
/* Save cost, because ComputeDistanceCache invalidates it. */
|
||||
float node_cost = nodes[pos].u.cost;
|
||||
nodes[pos].u.shortcut = ComputeDistanceShortcut(
|
||||
block_start, pos, max_backward_limit, nodes);
|
||||
if (node_cost <= ZopfliCostModelGetLiteralCosts(model, 0, pos)) {
|
||||
PosData posdata;
|
||||
posdata.pos = pos;
|
||||
posdata.cost = node_cost;
|
||||
posdata.costdiff = node_cost -
|
||||
ZopfliCostModelGetLiteralCosts(model, 0, pos);
|
||||
ComputeDistanceCache(
|
||||
pos, starting_dist_cache, nodes, posdata.distance_cache);
|
||||
StartPosQueuePush(queue, &posdata);
|
||||
}
|
||||
}
|
||||
|
||||
/* Returns longest copy length. */
|
||||
static size_t UpdateNodes(
|
||||
const size_t num_bytes, const size_t block_start, const size_t pos,
|
||||
const uint8_t* ringbuffer, const size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params, const size_t max_backward_limit,
|
||||
const int* starting_dist_cache, const size_t num_matches,
|
||||
const BackwardMatch* matches, const ZopfliCostModel* model,
|
||||
StartPosQueue* queue, ZopfliNode* nodes) {
|
||||
const size_t cur_ix = block_start + pos;
|
||||
const size_t cur_ix_masked = cur_ix & ringbuffer_mask;
|
||||
const size_t max_distance = BROTLI_MIN(size_t, cur_ix, max_backward_limit);
|
||||
const size_t max_len = num_bytes - pos;
|
||||
const size_t max_zopfli_len = MaxZopfliLen(params);
|
||||
const size_t max_iters = MaxZopfliCandidates(params);
|
||||
size_t min_len;
|
||||
size_t result = 0;
|
||||
size_t k;
|
||||
|
||||
EvaluateNode(block_start, pos, max_backward_limit, starting_dist_cache, model,
|
||||
queue, nodes);
|
||||
|
||||
{
|
||||
const PosData* posdata = StartPosQueueAt(queue, 0);
|
||||
float min_cost = (posdata->cost + ZopfliCostModelGetMinCostCmd(model) +
|
||||
ZopfliCostModelGetLiteralCosts(model, posdata->pos, pos));
|
||||
min_len = ComputeMinimumCopyLength(min_cost, nodes, num_bytes, pos);
|
||||
}
|
||||
|
||||
/* Go over the command starting positions in order of increasing cost
|
||||
difference. */
|
||||
for (k = 0; k < max_iters && k < StartPosQueueSize(queue); ++k) {
|
||||
const PosData* posdata = StartPosQueueAt(queue, k);
|
||||
const size_t start = posdata->pos;
|
||||
const uint16_t inscode = GetInsertLengthCode(pos - start);
|
||||
const float start_costdiff = posdata->costdiff;
|
||||
const float base_cost = start_costdiff + (float)GetInsertExtra(inscode) +
|
||||
ZopfliCostModelGetLiteralCosts(model, 0, pos);
|
||||
|
||||
/* Look for last distance matches using the distance cache from this
|
||||
starting position. */
|
||||
size_t best_len = min_len - 1;
|
||||
size_t j = 0;
|
||||
for (; j < BROTLI_NUM_DISTANCE_SHORT_CODES && best_len < max_len; ++j) {
|
||||
const size_t idx = kDistanceCacheIndex[j];
|
||||
const size_t backward =
|
||||
(size_t)(posdata->distance_cache[idx] + kDistanceCacheOffset[j]);
|
||||
size_t prev_ix = cur_ix - backward;
|
||||
if (prev_ix >= cur_ix) {
|
||||
continue;
|
||||
}
|
||||
if (BROTLI_PREDICT_FALSE(backward > max_distance)) {
|
||||
continue;
|
||||
}
|
||||
prev_ix &= ringbuffer_mask;
|
||||
|
||||
if (cur_ix_masked + best_len > ringbuffer_mask ||
|
||||
prev_ix + best_len > ringbuffer_mask ||
|
||||
ringbuffer[cur_ix_masked + best_len] !=
|
||||
ringbuffer[prev_ix + best_len]) {
|
||||
continue;
|
||||
}
|
||||
{
|
||||
const size_t len =
|
||||
FindMatchLengthWithLimit(&ringbuffer[prev_ix],
|
||||
&ringbuffer[cur_ix_masked],
|
||||
max_len);
|
||||
const float dist_cost = base_cost +
|
||||
ZopfliCostModelGetDistanceCost(model, j);
|
||||
size_t l;
|
||||
for (l = best_len + 1; l <= len; ++l) {
|
||||
const uint16_t copycode = GetCopyLengthCode(l);
|
||||
const uint16_t cmdcode =
|
||||
CombineLengthCodes(inscode, copycode, j == 0);
|
||||
const float cost = (cmdcode < 128 ? base_cost : dist_cost) +
|
||||
(float)GetCopyExtra(copycode) +
|
||||
ZopfliCostModelGetCommandCost(model, cmdcode);
|
||||
if (cost < nodes[pos + l].u.cost) {
|
||||
UpdateZopfliNode(nodes, pos, start, l, l, backward, j + 1, cost);
|
||||
result = BROTLI_MAX(size_t, result, l);
|
||||
}
|
||||
best_len = l;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* At higher iterations look only for new last distance matches, since
|
||||
looking only for new command start positions with the same distances
|
||||
does not help much. */
|
||||
if (k >= 2) continue;
|
||||
|
||||
{
|
||||
/* Loop through all possible copy lengths at this position. */
|
||||
size_t len = min_len;
|
||||
for (j = 0; j < num_matches; ++j) {
|
||||
BackwardMatch match = matches[j];
|
||||
size_t dist = match.distance;
|
||||
BROTLI_BOOL is_dictionary_match = TO_BROTLI_BOOL(dist > max_distance);
|
||||
/* We already tried all possible last distance matches, so we can use
|
||||
normal distance code here. */
|
||||
size_t dist_code = dist + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
|
||||
uint16_t dist_symbol;
|
||||
uint32_t distextra;
|
||||
uint32_t distnumextra;
|
||||
float dist_cost;
|
||||
size_t max_match_len;
|
||||
PrefixEncodeCopyDistance(dist_code, 0, 0, &dist_symbol, &distextra);
|
||||
distnumextra = distextra >> 24;
|
||||
dist_cost = base_cost + (float)distnumextra +
|
||||
ZopfliCostModelGetDistanceCost(model, dist_symbol);
|
||||
|
||||
/* Try all copy lengths up until the maximum copy length corresponding
|
||||
to this distance. If the distance refers to the static dictionary, or
|
||||
the maximum length is long enough, try only one maximum length. */
|
||||
max_match_len = BackwardMatchLength(&match);
|
||||
if (len < max_match_len &&
|
||||
(is_dictionary_match || max_match_len > max_zopfli_len)) {
|
||||
len = max_match_len;
|
||||
}
|
||||
for (; len <= max_match_len; ++len) {
|
||||
const size_t len_code =
|
||||
is_dictionary_match ? BackwardMatchLengthCode(&match) : len;
|
||||
const uint16_t copycode = GetCopyLengthCode(len_code);
|
||||
const uint16_t cmdcode = CombineLengthCodes(inscode, copycode, 0);
|
||||
const float cost = dist_cost + (float)GetCopyExtra(copycode) +
|
||||
ZopfliCostModelGetCommandCost(model, cmdcode);
|
||||
if (cost < nodes[pos + len].u.cost) {
|
||||
UpdateZopfliNode(nodes, pos, start, len, len_code, dist, 0, cost);
|
||||
result = BROTLI_MAX(size_t, result, len);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static size_t ComputeShortestPathFromNodes(size_t num_bytes,
|
||||
ZopfliNode* nodes) {
|
||||
size_t index = num_bytes;
|
||||
size_t num_commands = 0;
|
||||
while (nodes[index].insert_length == 0 && nodes[index].length == 1) --index;
|
||||
nodes[index].u.next = BROTLI_UINT32_MAX;
|
||||
while (index != 0) {
|
||||
size_t len = ZopfliNodeCommandLength(&nodes[index]);
|
||||
index -= len;
|
||||
nodes[index].u.next = (uint32_t)len;
|
||||
num_commands++;
|
||||
}
|
||||
return num_commands;
|
||||
}
|
||||
|
||||
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
|
||||
void BrotliZopfliCreateCommands(const size_t num_bytes,
|
||||
const size_t block_start,
|
||||
const size_t max_backward_limit,
|
||||
const ZopfliNode* nodes,
|
||||
int* dist_cache,
|
||||
size_t* last_insert_len,
|
||||
Command* commands,
|
||||
size_t* num_literals) {
|
||||
size_t pos = 0;
|
||||
uint32_t offset = nodes[0].u.next;
|
||||
size_t i;
|
||||
for (i = 0; offset != BROTLI_UINT32_MAX; i++) {
|
||||
const ZopfliNode* next = &nodes[pos + offset];
|
||||
size_t copy_length = ZopfliNodeCopyLength(next);
|
||||
size_t insert_length = next->insert_length;
|
||||
pos += insert_length;
|
||||
offset = next->u.next;
|
||||
if (i == 0) {
|
||||
insert_length += *last_insert_len;
|
||||
*last_insert_len = 0;
|
||||
}
|
||||
{
|
||||
size_t distance = ZopfliNodeCopyDistance(next);
|
||||
size_t len_code = ZopfliNodeLengthCode(next);
|
||||
size_t max_distance =
|
||||
BROTLI_MIN(size_t, block_start + pos, max_backward_limit);
|
||||
BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance);
|
||||
size_t dist_code = ZopfliNodeDistanceCode(next);
|
||||
|
||||
InitCommand(&commands[i], insert_length,
|
||||
copy_length, (int)len_code - (int)copy_length, dist_code);
|
||||
|
||||
if (!is_dictionary && dist_code > 0) {
|
||||
dist_cache[3] = dist_cache[2];
|
||||
dist_cache[2] = dist_cache[1];
|
||||
dist_cache[1] = dist_cache[0];
|
||||
dist_cache[0] = (int)distance;
|
||||
}
|
||||
}
|
||||
|
||||
*num_literals += insert_length;
|
||||
pos += copy_length;
|
||||
}
|
||||
*last_insert_len += num_bytes - pos;
|
||||
}
|
||||
|
||||
static size_t ZopfliIterate(size_t num_bytes,
|
||||
size_t position,
|
||||
const uint8_t* ringbuffer,
|
||||
size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params,
|
||||
const size_t max_backward_limit,
|
||||
const int* dist_cache,
|
||||
const ZopfliCostModel* model,
|
||||
const uint32_t* num_matches,
|
||||
const BackwardMatch* matches,
|
||||
ZopfliNode* nodes) {
|
||||
const size_t max_zopfli_len = MaxZopfliLen(params);
|
||||
StartPosQueue queue;
|
||||
size_t cur_match_pos = 0;
|
||||
size_t i;
|
||||
nodes[0].length = 0;
|
||||
nodes[0].u.cost = 0;
|
||||
InitStartPosQueue(&queue);
|
||||
for (i = 0; i + 3 < num_bytes; i++) {
|
||||
size_t skip = UpdateNodes(num_bytes, position, i, ringbuffer,
|
||||
ringbuffer_mask, params, max_backward_limit, dist_cache,
|
||||
num_matches[i], &matches[cur_match_pos], model, &queue, nodes);
|
||||
if (skip < BROTLI_LONG_COPY_QUICK_STEP) skip = 0;
|
||||
cur_match_pos += num_matches[i];
|
||||
if (num_matches[i] == 1 &&
|
||||
BackwardMatchLength(&matches[cur_match_pos - 1]) > max_zopfli_len) {
|
||||
skip = BROTLI_MAX(size_t,
|
||||
BackwardMatchLength(&matches[cur_match_pos - 1]), skip);
|
||||
}
|
||||
if (skip > 1) {
|
||||
skip--;
|
||||
while (skip) {
|
||||
i++;
|
||||
if (i + 3 >= num_bytes) break;
|
||||
EvaluateNode(
|
||||
position, i, max_backward_limit, dist_cache, model, &queue, nodes);
|
||||
cur_match_pos += num_matches[i];
|
||||
skip--;
|
||||
}
|
||||
}
|
||||
}
|
||||
return ComputeShortestPathFromNodes(num_bytes, nodes);
|
||||
}
|
||||
|
||||
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
|
||||
size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
|
||||
const BrotliDictionary* dictionary,
|
||||
size_t num_bytes,
|
||||
size_t position,
|
||||
const uint8_t* ringbuffer,
|
||||
size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params,
|
||||
const size_t max_backward_limit,
|
||||
const int* dist_cache,
|
||||
HasherHandle hasher,
|
||||
ZopfliNode* nodes) {
|
||||
const size_t max_zopfli_len = MaxZopfliLen(params);
|
||||
ZopfliCostModel model;
|
||||
StartPosQueue queue;
|
||||
BackwardMatch matches[MAX_NUM_MATCHES_H10];
|
||||
const size_t store_end = num_bytes >= StoreLookaheadH10() ?
|
||||
position + num_bytes - StoreLookaheadH10() + 1 : position;
|
||||
size_t i;
|
||||
nodes[0].length = 0;
|
||||
nodes[0].u.cost = 0;
|
||||
InitZopfliCostModel(m, &model, num_bytes);
|
||||
if (BROTLI_IS_OOM(m)) return 0;
|
||||
ZopfliCostModelSetFromLiteralCosts(
|
||||
&model, position, ringbuffer, ringbuffer_mask);
|
||||
InitStartPosQueue(&queue);
|
||||
for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; i++) {
|
||||
const size_t pos = position + i;
|
||||
const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
|
||||
size_t num_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
|
||||
ringbuffer_mask, pos, num_bytes - i, max_distance, params, matches);
|
||||
size_t skip;
|
||||
if (num_matches > 0 &&
|
||||
BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
|
||||
matches[0] = matches[num_matches - 1];
|
||||
num_matches = 1;
|
||||
}
|
||||
skip = UpdateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask,
|
||||
params, max_backward_limit, dist_cache, num_matches, matches, &model,
|
||||
&queue, nodes);
|
||||
if (skip < BROTLI_LONG_COPY_QUICK_STEP) skip = 0;
|
||||
if (num_matches == 1 && BackwardMatchLength(&matches[0]) > max_zopfli_len) {
|
||||
skip = BROTLI_MAX(size_t, BackwardMatchLength(&matches[0]), skip);
|
||||
}
|
||||
if (skip > 1) {
|
||||
/* Add the tail of the copy to the hasher. */
|
||||
StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1, BROTLI_MIN(
|
||||
size_t, pos + skip, store_end));
|
||||
skip--;
|
||||
while (skip) {
|
||||
i++;
|
||||
if (i + HashTypeLengthH10() - 1 >= num_bytes) break;
|
||||
EvaluateNode(
|
||||
position, i, max_backward_limit, dist_cache, &model, &queue, nodes);
|
||||
skip--;
|
||||
}
|
||||
}
|
||||
}
|
||||
CleanupZopfliCostModel(m, &model);
|
||||
return ComputeShortestPathFromNodes(num_bytes, nodes);
|
||||
}
|
||||
|
||||
void BrotliCreateZopfliBackwardReferences(
|
||||
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
|
||||
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
|
||||
size_t* last_insert_len, Command* commands, size_t* num_commands,
|
||||
size_t* num_literals) {
|
||||
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
|
||||
ZopfliNode* nodes;
|
||||
nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
BrotliInitZopfliNodes(nodes, num_bytes + 1);
|
||||
*num_commands += BrotliZopfliComputeShortestPath(m, dictionary, num_bytes,
|
||||
position, ringbuffer, ringbuffer_mask, params, max_backward_limit,
|
||||
dist_cache, hasher, nodes);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, nodes,
|
||||
dist_cache, last_insert_len, commands, num_literals);
|
||||
BROTLI_FREE(m, nodes);
|
||||
}
|
||||
|
||||
void BrotliCreateHqZopfliBackwardReferences(
|
||||
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
|
||||
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
|
||||
size_t* last_insert_len, Command* commands, size_t* num_commands,
|
||||
size_t* num_literals) {
|
||||
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
|
||||
uint32_t* num_matches = BROTLI_ALLOC(m, uint32_t, num_bytes);
|
||||
size_t matches_size = 4 * num_bytes;
|
||||
const size_t store_end = num_bytes >= StoreLookaheadH10() ?
|
||||
position + num_bytes - StoreLookaheadH10() + 1 : position;
|
||||
size_t cur_match_pos = 0;
|
||||
size_t i;
|
||||
size_t orig_num_literals;
|
||||
size_t orig_last_insert_len;
|
||||
int orig_dist_cache[4];
|
||||
size_t orig_num_commands;
|
||||
ZopfliCostModel model;
|
||||
ZopfliNode* nodes;
|
||||
BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) {
|
||||
const size_t pos = position + i;
|
||||
size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
|
||||
size_t max_length = num_bytes - i;
|
||||
size_t num_found_matches;
|
||||
size_t cur_match_end;
|
||||
size_t j;
|
||||
/* Ensure that we have enough free slots. */
|
||||
BROTLI_ENSURE_CAPACITY(m, BackwardMatch, matches, matches_size,
|
||||
cur_match_pos + MAX_NUM_MATCHES_H10);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
num_found_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
|
||||
ringbuffer_mask, pos, max_length, max_distance, params,
|
||||
&matches[cur_match_pos]);
|
||||
cur_match_end = cur_match_pos + num_found_matches;
|
||||
for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
|
||||
assert(BackwardMatchLength(&matches[j]) <
|
||||
BackwardMatchLength(&matches[j + 1]));
|
||||
assert(matches[j].distance > max_distance ||
|
||||
matches[j].distance <= matches[j + 1].distance);
|
||||
}
|
||||
num_matches[i] = (uint32_t)num_found_matches;
|
||||
if (num_found_matches > 0) {
|
||||
const size_t match_len = BackwardMatchLength(&matches[cur_match_end - 1]);
|
||||
if (match_len > MAX_ZOPFLI_LEN_QUALITY_11) {
|
||||
const size_t skip = match_len - 1;
|
||||
matches[cur_match_pos++] = matches[cur_match_end - 1];
|
||||
num_matches[i] = 1;
|
||||
/* Add the tail of the copy to the hasher. */
|
||||
StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1,
|
||||
BROTLI_MIN(size_t, pos + match_len, store_end));
|
||||
memset(&num_matches[i + 1], 0, skip * sizeof(num_matches[0]));
|
||||
i += skip;
|
||||
} else {
|
||||
cur_match_pos = cur_match_end;
|
||||
}
|
||||
}
|
||||
}
|
||||
orig_num_literals = *num_literals;
|
||||
orig_last_insert_len = *last_insert_len;
|
||||
memcpy(orig_dist_cache, dist_cache, 4 * sizeof(dist_cache[0]));
|
||||
orig_num_commands = *num_commands;
|
||||
nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
InitZopfliCostModel(m, &model, num_bytes);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
for (i = 0; i < 2; i++) {
|
||||
BrotliInitZopfliNodes(nodes, num_bytes + 1);
|
||||
if (i == 0) {
|
||||
ZopfliCostModelSetFromLiteralCosts(
|
||||
&model, position, ringbuffer, ringbuffer_mask);
|
||||
} else {
|
||||
ZopfliCostModelSetFromCommands(&model, position, ringbuffer,
|
||||
ringbuffer_mask, commands, *num_commands - orig_num_commands,
|
||||
orig_last_insert_len);
|
||||
}
|
||||
*num_commands = orig_num_commands;
|
||||
*num_literals = orig_num_literals;
|
||||
*last_insert_len = orig_last_insert_len;
|
||||
memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
|
||||
*num_commands += ZopfliIterate(num_bytes, position, ringbuffer,
|
||||
ringbuffer_mask, params, max_backward_limit, dist_cache,
|
||||
&model, num_matches, matches, nodes);
|
||||
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit,
|
||||
nodes, dist_cache, last_insert_len, commands, num_literals);
|
||||
}
|
||||
CleanupZopfliCostModel(m, &model);
|
||||
BROTLI_FREE(m, nodes);
|
||||
BROTLI_FREE(m, matches);
|
||||
BROTLI_FREE(m, num_matches);
|
||||
}
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Functions to estimate the bit cost of Huffman trees. */
|
||||
|
||||
#include "./enc/bit_cost.h"
|
||||
|
||||
#include "./common/constants.h"
|
||||
#include <brotli/types.h>
|
||||
#include "./enc/fast_log.h"
|
||||
#include "./enc/histogram.h"
|
||||
#include "./enc/port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define FN(X) X ## Literal
|
||||
#include "./enc/bit_cost_inc.h" /* NOLINT(build/include) */
|
||||
#undef FN
|
||||
|
||||
#define FN(X) X ## Command
|
||||
#include "./enc/bit_cost_inc.h" /* NOLINT(build/include) */
|
||||
#undef FN
|
||||
|
||||
#define FN(X) X ## Distance
|
||||
#include "./enc/bit_cost_inc.h" /* NOLINT(build/include) */
|
||||
#undef FN
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Bit reading helpers */
|
||||
|
||||
#include "./dec/bit_reader.h"
|
||||
|
||||
#include <brotli/types.h>
|
||||
#include "./dec/port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void BrotliInitBitReader(BrotliBitReader* const br) {
|
||||
br->val_ = 0;
|
||||
br->bit_pos_ = sizeof(br->val_) << 3;
|
||||
}
|
||||
|
||||
BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* const br) {
|
||||
size_t aligned_read_mask = (sizeof(br->val_) >> 1) - 1;
|
||||
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
|
||||
overflow. If unalignment is caused by BrotliSafeReadBits, then there is
|
||||
enough space in accumulator to fix alignment. */
|
||||
if (!BROTLI_ALIGNED_READ) {
|
||||
aligned_read_mask = 0;
|
||||
}
|
||||
if (BrotliGetAvailableBits(br) == 0) {
|
||||
if (!BrotliPullByte(br)) {
|
||||
return BROTLI_FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
while ((((size_t)br->next_in) & aligned_read_mask) != 0) {
|
||||
if (!BrotliPullByte(br)) {
|
||||
/* If we consumed all the input, we don't care about the alignment. */
|
||||
return BROTLI_TRUE;
|
||||
}
|
||||
}
|
||||
return BROTLI_TRUE;
|
||||
}
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Block split point selection utilities. */
|
||||
|
||||
#include "./enc/block_splitter.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <string.h> /* memcpy, memset */
|
||||
|
||||
#include "./enc/bit_cost.h"
|
||||
#include "./enc/cluster.h"
|
||||
#include "./enc/command.h"
|
||||
#include "./enc/fast_log.h"
|
||||
#include "./enc/histogram.h"
|
||||
#include "./enc/memory.h"
|
||||
#include "./enc/port.h"
|
||||
#include "./enc/quality.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static const size_t kMaxLiteralHistograms = 100;
|
||||
static const size_t kMaxCommandHistograms = 50;
|
||||
static const double kLiteralBlockSwitchCost = 28.1;
|
||||
static const double kCommandBlockSwitchCost = 13.5;
|
||||
static const double kDistanceBlockSwitchCost = 14.6;
|
||||
static const size_t kLiteralStrideLength = 70;
|
||||
static const size_t kCommandStrideLength = 40;
|
||||
static const size_t kSymbolsPerLiteralHistogram = 544;
|
||||
static const size_t kSymbolsPerCommandHistogram = 530;
|
||||
static const size_t kSymbolsPerDistanceHistogram = 544;
|
||||
static const size_t kMinLengthForBlockSplitting = 128;
|
||||
static const size_t kIterMulForRefining = 2;
|
||||
static const size_t kMinItersForRefining = 100;
|
||||
|
||||
static size_t CountLiterals(const Command* cmds, const size_t num_commands) {
|
||||
/* Count how many we have. */
|
||||
size_t total_length = 0;
|
||||
size_t i;
|
||||
for (i = 0; i < num_commands; ++i) {
|
||||
total_length += cmds[i].insert_len_;
|
||||
}
|
||||
return total_length;
|
||||
}
|
||||
|
||||
static void CopyLiteralsToByteArray(const Command* cmds,
|
||||
const size_t num_commands,
|
||||
const uint8_t* data,
|
||||
const size_t offset,
|
||||
const size_t mask,
|
||||
uint8_t* literals) {
|
||||
size_t pos = 0;
|
||||
size_t from_pos = offset & mask;
|
||||
size_t i;
|
||||
for (i = 0; i < num_commands; ++i) {
|
||||
size_t insert_len = cmds[i].insert_len_;
|
||||
if (from_pos + insert_len > mask) {
|
||||
size_t head_size = mask + 1 - from_pos;
|
||||
memcpy(literals + pos, data + from_pos, head_size);
|
||||
from_pos = 0;
|
||||
pos += head_size;
|
||||
insert_len -= head_size;
|
||||
}
|
||||
if (insert_len > 0) {
|
||||
memcpy(literals + pos, data + from_pos, insert_len);
|
||||
pos += insert_len;
|
||||
}
|
||||
from_pos = (from_pos + insert_len + CommandCopyLen(&cmds[i])) & mask;
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE unsigned int MyRand(unsigned int* seed) {
|
||||
*seed *= 16807U;
|
||||
if (*seed == 0) {
|
||||
*seed = 1;
|
||||
}
|
||||
return *seed;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE double BitCost(size_t count) {
|
||||
return count == 0 ? -2.0 : FastLog2(count);
|
||||
}
|
||||
|
||||
#define HISTOGRAMS_PER_BATCH 64
|
||||
#define CLUSTERS_PER_BATCH 16
|
||||
|
||||
#define FN(X) X ## Literal
|
||||
#define DataType uint8_t
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/block_splitter_inc.h"
|
||||
#undef DataType
|
||||
#undef FN
|
||||
|
||||
#define FN(X) X ## Command
|
||||
#define DataType uint16_t
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/block_splitter_inc.h"
|
||||
#undef FN
|
||||
|
||||
#define FN(X) X ## Distance
|
||||
/* NOLINTNEXTLINE(build/include) */
|
||||
#include "./enc/block_splitter_inc.h"
|
||||
#undef DataType
|
||||
#undef FN
|
||||
|
||||
void BrotliInitBlockSplit(BlockSplit* self) {
|
||||
self->num_types = 0;
|
||||
self->num_blocks = 0;
|
||||
self->types = 0;
|
||||
self->lengths = 0;
|
||||
self->types_alloc_size = 0;
|
||||
self->lengths_alloc_size = 0;
|
||||
}
|
||||
|
||||
void BrotliDestroyBlockSplit(MemoryManager* m, BlockSplit* self) {
|
||||
BROTLI_FREE(m, self->types);
|
||||
BROTLI_FREE(m, self->lengths);
|
||||
}
|
||||
|
||||
void BrotliSplitBlock(MemoryManager* m,
|
||||
const Command* cmds,
|
||||
const size_t num_commands,
|
||||
const uint8_t* data,
|
||||
const size_t pos,
|
||||
const size_t mask,
|
||||
const BrotliEncoderParams* params,
|
||||
BlockSplit* literal_split,
|
||||
BlockSplit* insert_and_copy_split,
|
||||
BlockSplit* dist_split) {
|
||||
{
|
||||
size_t literals_count = CountLiterals(cmds, num_commands);
|
||||
uint8_t* literals = BROTLI_ALLOC(m, uint8_t, literals_count);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
/* Create a continuous array of literals. */
|
||||
CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals);
|
||||
/* Create the block split on the array of literals.
|
||||
Literal histograms have alphabet size 256. */
|
||||
SplitByteVectorLiteral(
|
||||
m, literals, literals_count,
|
||||
kSymbolsPerLiteralHistogram, kMaxLiteralHistograms,
|
||||
kLiteralStrideLength, kLiteralBlockSwitchCost, params,
|
||||
literal_split);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
BROTLI_FREE(m, literals);
|
||||
}
|
||||
|
||||
{
|
||||
/* Compute prefix codes for commands. */
|
||||
uint16_t* insert_and_copy_codes = BROTLI_ALLOC(m, uint16_t, num_commands);
|
||||
size_t i;
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
for (i = 0; i < num_commands; ++i) {
|
||||
insert_and_copy_codes[i] = cmds[i].cmd_prefix_;
|
||||
}
|
||||
/* Create the block split on the array of command prefixes. */
|
||||
SplitByteVectorCommand(
|
||||
m, insert_and_copy_codes, num_commands,
|
||||
kSymbolsPerCommandHistogram, kMaxCommandHistograms,
|
||||
kCommandStrideLength, kCommandBlockSwitchCost, params,
|
||||
insert_and_copy_split);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
/* TODO: reuse for distances? */
|
||||
BROTLI_FREE(m, insert_and_copy_codes);
|
||||
}
|
||||
|
||||
{
|
||||
/* Create a continuous array of distance prefixes. */
|
||||
uint16_t* distance_prefixes = BROTLI_ALLOC(m, uint16_t, num_commands);
|
||||
size_t j = 0;
|
||||
size_t i;
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
for (i = 0; i < num_commands; ++i) {
|
||||
const Command* cmd = &cmds[i];
|
||||
if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
|
||||
distance_prefixes[j++] = cmd->dist_prefix_;
|
||||
}
|
||||
}
|
||||
/* Create the block split on the array of distance prefixes. */
|
||||
SplitByteVectorDistance(
|
||||
m, distance_prefixes, j,
|
||||
kSymbolsPerDistanceHistogram, kMaxCommandHistograms,
|
||||
kCommandStrideLength, kDistanceBlockSwitchCost, params,
|
||||
dist_split);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
BROTLI_FREE(m, distance_prefixes);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
package brotli
|
||||
|
||||
import (
|
||||
"C"
|
||||
)
|
||||
|
|
@ -1,362 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/**
|
||||
* @file
|
||||
* API for Brotli decompression.
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_DEC_DECODE_H_
|
||||
#define BROTLI_DEC_DECODE_H_
|
||||
|
||||
#include <brotli/port.h>
|
||||
#include <brotli/types.h>
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Opaque structure that holds decoder state.
|
||||
*
|
||||
* Allocated and initialized with ::BrotliDecoderCreateInstance.
|
||||
* Cleaned up and deallocated with ::BrotliDecoderDestroyInstance.
|
||||
*/
|
||||
typedef struct BrotliDecoderStateStruct BrotliDecoderState;
|
||||
|
||||
/**
|
||||
* Result type for ::BrotliDecoderDecompress and
|
||||
* ::BrotliDecoderDecompressStream functions.
|
||||
*/
|
||||
typedef enum {
|
||||
/** Decoding error, e.g. corrupted input or memory allocation problem. */
|
||||
BROTLI_DECODER_RESULT_ERROR = 0,
|
||||
/** Decoding successfully completed */
|
||||
BROTLI_DECODER_RESULT_SUCCESS = 1,
|
||||
/** Partially done; should be called again with more input */
|
||||
BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT = 2,
|
||||
/** Partially done; should be called again with more output */
|
||||
BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT = 3
|
||||
} BrotliDecoderResult;
|
||||
|
||||
/**
|
||||
* Template that evaluates items of ::BrotliDecoderErrorCode.
|
||||
*
|
||||
* Example: @code {.cpp}
|
||||
* // Log Brotli error code.
|
||||
* switch (brotliDecoderErrorCode) {
|
||||
* #define CASE_(PREFIX, NAME, CODE) \
|
||||
* case BROTLI_DECODER ## PREFIX ## NAME: \
|
||||
* LOG(INFO) << "error code:" << #NAME; \
|
||||
* break;
|
||||
* #define NEWLINE_
|
||||
* BROTLI_DECODER_ERROR_CODES_LIST(CASE_, NEWLINE_)
|
||||
* #undef CASE_
|
||||
* #undef NEWLINE_
|
||||
* default: LOG(FATAL) << "unknown brotli error code";
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
#define BROTLI_DECODER_ERROR_CODES_LIST(BROTLI_ERROR_CODE, SEPARATOR) \
|
||||
BROTLI_ERROR_CODE(_, NO_ERROR, 0) SEPARATOR \
|
||||
/* Same as BrotliDecoderResult values */ \
|
||||
BROTLI_ERROR_CODE(_, SUCCESS, 1) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_, NEEDS_MORE_INPUT, 2) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_, NEEDS_MORE_OUTPUT, 3) SEPARATOR \
|
||||
\
|
||||
/* Errors caused by invalid input */ \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, EXUBERANT_NIBBLE, -1) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, RESERVED, -2) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, EXUBERANT_META_NIBBLE, -3) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, SIMPLE_HUFFMAN_ALPHABET, -4) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, SIMPLE_HUFFMAN_SAME, -5) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, CL_SPACE, -6) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, HUFFMAN_SPACE, -7) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, CONTEXT_MAP_REPEAT, -8) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, BLOCK_LENGTH_1, -9) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, BLOCK_LENGTH_2, -10) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, TRANSFORM, -11) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, DICTIONARY, -12) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, WINDOW_BITS, -13) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_1, -14) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR \
|
||||
\
|
||||
/* -16..-18 codes are reserved */ \
|
||||
\
|
||||
BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_, INVALID_ARGUMENTS, -20) SEPARATOR \
|
||||
\
|
||||
/* Memory allocation problems */ \
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, CONTEXT_MODES, -21) SEPARATOR \
|
||||
/* Literal, insert and distance trees together */ \
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, TREE_GROUPS, -22) SEPARATOR \
|
||||
/* -23..-24 codes are reserved for distinct tree groups */ \
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, CONTEXT_MAP, -25) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_1, -26) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_2, -27) SEPARATOR \
|
||||
/* -28..-29 codes are reserved for dynamic ring-buffer allocation */ \
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, BLOCK_TYPE_TREES, -30) SEPARATOR \
|
||||
\
|
||||
/* "Impossible" states */ \
|
||||
BROTLI_ERROR_CODE(_ERROR_, UNREACHABLE, -31)
|
||||
|
||||
/**
|
||||
* Error code for detailed logging / production debugging.
|
||||
*
|
||||
* See ::BrotliDecoderGetErrorCode and ::BROTLI_LAST_ERROR_CODE.
|
||||
*/
|
||||
typedef enum {
|
||||
#define BROTLI_COMMA_ ,
|
||||
#define BROTLI_ERROR_CODE_ENUM_ITEM_(PREFIX, NAME, CODE) \
|
||||
BROTLI_DECODER ## PREFIX ## NAME = CODE
|
||||
BROTLI_DECODER_ERROR_CODES_LIST(BROTLI_ERROR_CODE_ENUM_ITEM_, BROTLI_COMMA_)
|
||||
} BrotliDecoderErrorCode;
|
||||
#undef BROTLI_ERROR_CODE_ENUM_ITEM_
|
||||
#undef BROTLI_COMMA_
|
||||
|
||||
/**
|
||||
* The value of the last error code, negative integer.
|
||||
*
|
||||
* All other error code values are in the range from ::BROTLI_LAST_ERROR_CODE
|
||||
* to @c -1. There are also 4 other possible non-error codes @c 0 .. @c 3 in
|
||||
* ::BrotliDecoderErrorCode enumeration.
|
||||
*/
|
||||
#define BROTLI_LAST_ERROR_CODE BROTLI_DECODER_ERROR_UNREACHABLE
|
||||
|
||||
/** Options to be used with ::BrotliDecoderSetParameter. */
|
||||
typedef enum BrotliDecoderParameter {
|
||||
/**
|
||||
* Disable "canny" ring buffer allocation strategy.
|
||||
*
|
||||
* Ring buffer is allocated according to window size, despite the real size of
|
||||
* the content.
|
||||
*/
|
||||
BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION = 0
|
||||
} BrotliDecoderParameter;
|
||||
|
||||
/**
|
||||
* Sets the specified parameter to the given decoder instance.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @param param parameter to set
|
||||
* @param value new parameter value
|
||||
* @returns ::BROTLI_FALSE if parameter is unrecognized, or value is invalid
|
||||
* @returns ::BROTLI_TRUE if value is accepted
|
||||
*/
|
||||
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter(
|
||||
BrotliDecoderState* state, BrotliDecoderParameter param, uint32_t value);
|
||||
|
||||
/**
|
||||
* Creates an instance of ::BrotliDecoderState and initializes it.
|
||||
*
|
||||
* The instance can be used once for decoding and should then be destroyed with
|
||||
* ::BrotliDecoderDestroyInstance, it cannot be reused for a new decoding
|
||||
* session.
|
||||
*
|
||||
* @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
|
||||
* case they are both zero, default memory allocators are used. @p opaque is
|
||||
* passed to @p alloc_func and @p free_func when they are called.
|
||||
*
|
||||
* @param alloc_func custom memory allocation function
|
||||
* @param free_func custom memory fee function
|
||||
* @param opaque custom memory manager handle
|
||||
* @returns @c 0 if instance can not be allocated or initialized
|
||||
* @returns pointer to initialized ::BrotliDecoderState otherwise
|
||||
*/
|
||||
BROTLI_DEC_API BrotliDecoderState* BrotliDecoderCreateInstance(
|
||||
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
|
||||
|
||||
/**
|
||||
* Deinitializes and frees ::BrotliDecoderState instance.
|
||||
*
|
||||
* @param state decoder instance to be cleaned up and deallocated
|
||||
*/
|
||||
BROTLI_DEC_API void BrotliDecoderDestroyInstance(BrotliDecoderState* state);
|
||||
|
||||
/**
|
||||
* Performs one-shot memory-to-memory decompression.
|
||||
*
|
||||
* Decompresses the data in @p encoded_buffer into @p decoded_buffer, and sets
|
||||
* @p *decoded_size to the decompressed length.
|
||||
*
|
||||
* @param encoded_size size of @p encoded_buffer
|
||||
* @param encoded_buffer compressed data buffer with at least @p encoded_size
|
||||
* addressable bytes
|
||||
* @param[in, out] decoded_size @b in: size of @p decoded_buffer; \n
|
||||
* @b out: length of decompressed data written to
|
||||
* @p decoded_buffer
|
||||
* @param decoded_buffer decompressed data destination buffer
|
||||
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
|
||||
* allocation failed, or @p decoded_buffer is not large enough;
|
||||
* @returns ::BROTLI_DECODER_RESULT_SUCCESS otherwise
|
||||
*/
|
||||
BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompress(
|
||||
size_t encoded_size,
|
||||
const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)],
|
||||
size_t* decoded_size,
|
||||
uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(*decoded_size)]);
|
||||
|
||||
/**
|
||||
* Decompresses the input stream to the output stream.
|
||||
*
|
||||
* The values @p *available_in and @p *available_out must specify the number of
|
||||
* bytes addressable at @p *next_in and @p *next_out respectively.
|
||||
* When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
|
||||
*
|
||||
* After each call, @p *available_in will be decremented by the amount of input
|
||||
* bytes consumed, and the @p *next_in pointer will be incremented by that
|
||||
* amount. Similarly, @p *available_out will be decremented by the amount of
|
||||
* output bytes written, and the @p *next_out pointer will be incremented by
|
||||
* that amount.
|
||||
*
|
||||
* @p total_out, if it is not a null-pointer, will be set to the number
|
||||
* of bytes decompressed since the last @p state initialization.
|
||||
*
|
||||
* @note Input is never overconsumed, so @p next_in and @p available_in could be
|
||||
* passed to the next consumer after decoding is complete.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @param[in, out] available_in @b in: amount of available input; \n
|
||||
* @b out: amount of unused input
|
||||
* @param[in, out] next_in pointer to the next compressed byte
|
||||
* @param[in, out] available_out @b in: length of output buffer; \n
|
||||
* @b out: remaining size of output buffer
|
||||
* @param[in, out] next_out output buffer cursor;
|
||||
* can be @c NULL if @p available_out is @c 0
|
||||
* @param[out] total_out number of bytes decompressed so far; can be @c NULL
|
||||
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
|
||||
* allocation failed, arguments were invalid, etc.;
|
||||
* use ::BrotliDecoderGetErrorCode to get detailed error code
|
||||
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT decoding is blocked until
|
||||
* more input data is provided
|
||||
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
|
||||
* more output space is provided
|
||||
* @returns ::BROTLI_DECODER_RESULT_SUCCESS decoding is finished, no more
|
||||
* input might be consumed and no more output will be produced
|
||||
*/
|
||||
BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream(
|
||||
BrotliDecoderState* state, size_t* available_in, const uint8_t** next_in,
|
||||
size_t* available_out, uint8_t** next_out, size_t* total_out);
|
||||
|
||||
/**
|
||||
* Prepends LZ77 dictionary.
|
||||
*
|
||||
* Fills the fresh ::BrotliDecoderState with additional data corpus for LZ77
|
||||
* backward references.
|
||||
*
|
||||
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
|
||||
* @warning The dictionary must exist in memory until decoding is done and
|
||||
* is owned by the caller.
|
||||
*
|
||||
* Workflow:
|
||||
* -# Allocate and initialize state with ::BrotliDecoderCreateInstance
|
||||
* -# Invoke ::BrotliDecoderSetCustomDictionary
|
||||
* -# Use ::BrotliDecoderDecompressStream
|
||||
* -# Clean up and free state with ::BrotliDecoderDestroyInstance
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @param size length of @p dict; should be less or equal to 2^24 (16MiB),
|
||||
* otherwise the dictionary will be ignored
|
||||
* @param dict "dictionary"; @b MUST be the same as used during compression
|
||||
*/
|
||||
BROTLI_DEC_API void BrotliDecoderSetCustomDictionary(
|
||||
BrotliDecoderState* state, size_t size,
|
||||
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
|
||||
|
||||
/**
|
||||
* Checks if decoder has more output.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @returns ::BROTLI_TRUE, if decoder has some unconsumed output
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderHasMoreOutput(
|
||||
const BrotliDecoderState* state);
|
||||
|
||||
/**
|
||||
* Acquires pointer to internal output buffer.
|
||||
*
|
||||
* This method is used to make language bindings easier and more efficient:
|
||||
* -# push data to ::BrotliDecoderDecompressStream,
|
||||
* until ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT is reported
|
||||
* -# use ::BrotliDecoderTakeOutput to peek bytes and copy to language-specific
|
||||
* entity
|
||||
*
|
||||
* Also this could be useful if there is an output stream that is able to
|
||||
* consume all the provided data (e.g. when data is saved to file system).
|
||||
*
|
||||
* @attention After every call to ::BrotliDecoderTakeOutput @p *size bytes of
|
||||
* output are considered consumed for all consecutive calls to the
|
||||
* instance methods; returned pointer becomes invalidated as well.
|
||||
*
|
||||
* @note Decoder output is not guaranteed to be contiguous. This means that
|
||||
* after the size-unrestricted call to ::BrotliDecoderTakeOutput,
|
||||
* immediate next call to ::BrotliDecoderTakeOutput may return more data.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
|
||||
* any amount could be handled; \n
|
||||
* @b out: amount of data pointed by returned pointer and
|
||||
* considered consumed; \n
|
||||
* out value is never greater than in value, unless it is @c 0
|
||||
* @returns pointer to output data
|
||||
*/
|
||||
BROTLI_DEC_API const uint8_t* BrotliDecoderTakeOutput(
|
||||
BrotliDecoderState* state, size_t* size);
|
||||
|
||||
/**
|
||||
* Checks if instance has already consumed input.
|
||||
*
|
||||
* Instance that returns ::BROTLI_FALSE is considered "fresh" and could be
|
||||
* reused.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @returns ::BROTLI_TRUE if decoder has already used some input bytes
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsUsed(const BrotliDecoderState* state);
|
||||
|
||||
/**
|
||||
* Checks if decoder instance reached the final state.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @returns ::BROTLI_TRUE if decoder is in a state where it reached the end of
|
||||
* the input and produced all of the output
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* state);
|
||||
|
||||
/**
|
||||
* Acquires a detailed error code.
|
||||
*
|
||||
* Should be used only after ::BrotliDecoderDecompressStream returns
|
||||
* ::BROTLI_DECODER_RESULT_ERROR.
|
||||
*
|
||||
* See also ::BrotliDecoderErrorString
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @returns last saved error code
|
||||
*/
|
||||
BROTLI_DEC_API BrotliDecoderErrorCode BrotliDecoderGetErrorCode(
|
||||
const BrotliDecoderState* state);
|
||||
|
||||
/**
|
||||
* Converts error code to a c-string.
|
||||
*/
|
||||
BROTLI_DEC_API const char* BrotliDecoderErrorString(BrotliDecoderErrorCode c);
|
||||
|
||||
/**
|
||||
* Gets a decoder library version.
|
||||
*
|
||||
* Look at BROTLI_VERSION for more information.
|
||||
*/
|
||||
BROTLI_DEC_API uint32_t BrotliDecoderVersion(void);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_DEC_DECODE_H_ */
|
||||
|
|
@ -1,421 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/**
|
||||
* @file
|
||||
* API for Brotli compression.
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_ENC_ENCODE_H_
|
||||
#define BROTLI_ENC_ENCODE_H_
|
||||
|
||||
#include <brotli/port.h>
|
||||
#include <brotli/types.h>
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
|
||||
#define BROTLI_MIN_WINDOW_BITS 10
|
||||
/**
|
||||
* Maximal value for ::BROTLI_PARAM_LGWIN parameter.
|
||||
*
|
||||
* @note equal to @c BROTLI_MAX_DISTANCE_BITS constant.
|
||||
*/
|
||||
#define BROTLI_MAX_WINDOW_BITS 24
|
||||
/** Minimal value for ::BROTLI_PARAM_LGBLOCK parameter. */
|
||||
#define BROTLI_MIN_INPUT_BLOCK_BITS 16
|
||||
/** Maximal value for ::BROTLI_PARAM_LGBLOCK parameter. */
|
||||
#define BROTLI_MAX_INPUT_BLOCK_BITS 24
|
||||
/** Minimal value for ::BROTLI_PARAM_QUALITY parameter. */
|
||||
#define BROTLI_MIN_QUALITY 0
|
||||
/** Maximal value for ::BROTLI_PARAM_QUALITY parameter. */
|
||||
#define BROTLI_MAX_QUALITY 11
|
||||
|
||||
/** Options for ::BROTLI_PARAM_MODE parameter. */
|
||||
typedef enum BrotliEncoderMode {
|
||||
/**
|
||||
* Default compression mode.
|
||||
*
|
||||
* In this mode compressor does not know anything in advance about the
|
||||
* properties of the input.
|
||||
*/
|
||||
BROTLI_MODE_GENERIC = 0,
|
||||
/** Compression mode for UTF-8 formatted text input. */
|
||||
BROTLI_MODE_TEXT = 1,
|
||||
/** Compression mode used in WOFF 2.0. */
|
||||
BROTLI_MODE_FONT = 2
|
||||
} BrotliEncoderMode;
|
||||
|
||||
/** Default value for ::BROTLI_PARAM_QUALITY parameter. */
|
||||
#define BROTLI_DEFAULT_QUALITY 11
|
||||
/** Default value for ::BROTLI_PARAM_LGWIN parameter. */
|
||||
#define BROTLI_DEFAULT_WINDOW 22
|
||||
/** Default value for ::BROTLI_PARAM_MODE parameter. */
|
||||
#define BROTLI_DEFAULT_MODE BROTLI_MODE_GENERIC
|
||||
|
||||
/** Operations that can be performed by streaming encoder. */
|
||||
typedef enum BrotliEncoderOperation {
|
||||
/**
|
||||
* Process input.
|
||||
*
|
||||
* Encoder may postpone producing output, until it has processed enough input.
|
||||
*/
|
||||
BROTLI_OPERATION_PROCESS = 0,
|
||||
/**
|
||||
* Produce output for all processed input.
|
||||
*
|
||||
* Actual flush is performed when input stream is depleted and there is enough
|
||||
* space in output stream. This means that client should repeat
|
||||
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
|
||||
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
|
||||
*
|
||||
* @warning Until flush is complete, client @b SHOULD @b NOT swap,
|
||||
* reduce or extend input stream.
|
||||
*
|
||||
* When flush is complete, output data will be sufficient for decoder to
|
||||
* reproduce all the given input.
|
||||
*/
|
||||
BROTLI_OPERATION_FLUSH = 1,
|
||||
/**
|
||||
* Finalize the stream.
|
||||
*
|
||||
* Actual finalization is performed when input stream is depleted and there is
|
||||
* enough space in output stream. This means that client should repeat
|
||||
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
|
||||
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
|
||||
*
|
||||
* @warning Until finalization is complete, client @b SHOULD @b NOT swap,
|
||||
* reduce or extend input stream.
|
||||
*
|
||||
* Helper function ::BrotliEncoderIsFinished checks if stream is finalized and
|
||||
* output fully dumped.
|
||||
*
|
||||
* Adding more input data to finalized stream is impossible.
|
||||
*/
|
||||
BROTLI_OPERATION_FINISH = 2,
|
||||
/**
|
||||
* Emit metadata block to stream.
|
||||
*
|
||||
* Metadata is opaque to Brotli: neither encoder, nor decoder processes this
|
||||
* data or relies on it. It may be used to pass some extra information from
|
||||
* encoder client to decoder client without interfering with main data stream.
|
||||
*
|
||||
* @note Encoder may emit empty metadata blocks internally, to pad encoded
|
||||
* stream to byte boundary.
|
||||
*
|
||||
* @warning Until emitting metadata is complete client @b SHOULD @b NOT swap,
|
||||
* reduce or extend input stream.
|
||||
*
|
||||
* @warning The whole content of input buffer is considered to be the content
|
||||
* of metadata block. Do @b NOT @e append metadata to input stream,
|
||||
* before it is depleted with other operations.
|
||||
*
|
||||
* Stream is soft-flushed before metadata block is emitted. Metadata block
|
||||
* @b MUST be no longer than than 16MiB.
|
||||
*/
|
||||
BROTLI_OPERATION_EMIT_METADATA = 3
|
||||
} BrotliEncoderOperation;
|
||||
|
||||
/** Options to be used with ::BrotliEncoderSetParameter. */
|
||||
typedef enum BrotliEncoderParameter {
|
||||
/**
|
||||
* Tune encoder for specific input.
|
||||
*
|
||||
* ::BrotliEncoderMode enumerates all available values.
|
||||
*/
|
||||
BROTLI_PARAM_MODE = 0,
|
||||
/**
|
||||
* The main compression speed-density lever.
|
||||
*
|
||||
* The higher the quality, the slower the compression. Range is
|
||||
* from ::BROTLI_MIN_QUALITY to ::BROTLI_MAX_QUALITY.
|
||||
*/
|
||||
BROTLI_PARAM_QUALITY = 1,
|
||||
/**
|
||||
* Recommended sliding LZ77 window size.
|
||||
*
|
||||
* Encoder may reduce this value, e.g. if input is much smaller than
|
||||
* window size.
|
||||
*
|
||||
* Window size is `(1 << value) - 16`.
|
||||
*
|
||||
* Range is from ::BROTLI_MIN_WINDOW_BITS to ::BROTLI_MAX_WINDOW_BITS.
|
||||
*/
|
||||
BROTLI_PARAM_LGWIN = 2,
|
||||
/**
|
||||
* Recommended input block size.
|
||||
*
|
||||
* Encoder may reduce this value, e.g. if input is much smaller than input
|
||||
* block size.
|
||||
*
|
||||
* Range is from ::BROTLI_MIN_INPUT_BLOCK_BITS to
|
||||
* ::BROTLI_MAX_INPUT_BLOCK_BITS.
|
||||
*
|
||||
* @note Bigger input block size allows better compression, but consumes more
|
||||
* memory. \n The rough formula of memory used for temporary input
|
||||
* storage is `3 << lgBlock`.
|
||||
*/
|
||||
BROTLI_PARAM_LGBLOCK = 3,
|
||||
/**
|
||||
* Flag that affects usage of "literal context modeling" format feature.
|
||||
*
|
||||
* This flag is a "decoding-speed vs compression ratio" trade-off.
|
||||
*/
|
||||
BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING = 4,
|
||||
/**
|
||||
* Estimated total input size for all ::BrotliEncoderCompressStream calls.
|
||||
*
|
||||
* The default value is 0, which means that the total input size is unknown.
|
||||
*/
|
||||
BROTLI_PARAM_SIZE_HINT = 5
|
||||
} BrotliEncoderParameter;
|
||||
|
||||
/**
|
||||
* Opaque structure that holds encoder state.
|
||||
*
|
||||
* Allocated and initialized with ::BrotliEncoderCreateInstance.
|
||||
* Cleaned up and deallocated with ::BrotliEncoderDestroyInstance.
|
||||
*/
|
||||
typedef struct BrotliEncoderStateStruct BrotliEncoderState;
|
||||
|
||||
/**
|
||||
* Sets the specified parameter to the given encoder instance.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @param param parameter to set
|
||||
* @param value new parameter value
|
||||
* @returns ::BROTLI_FALSE if parameter is unrecognized, or value is invalid
|
||||
* @returns ::BROTLI_FALSE if value of parameter can not be changed at current
|
||||
* encoder state (e.g. when encoding is started, window size might be
|
||||
* already encoded and therefore it is impossible to change it)
|
||||
* @returns ::BROTLI_TRUE if value is accepted
|
||||
* @warning invalid values might be accepted in case they would not break
|
||||
* encoding process.
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderSetParameter(
|
||||
BrotliEncoderState* state, BrotliEncoderParameter param, uint32_t value);
|
||||
|
||||
/**
|
||||
* Creates an instance of ::BrotliEncoderState and initializes it.
|
||||
*
|
||||
* @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
|
||||
* case they are both zero, default memory allocators are used. @p opaque is
|
||||
* passed to @p alloc_func and @p free_func when they are called.
|
||||
*
|
||||
* @param alloc_func custom memory allocation function
|
||||
* @param free_func custom memory fee function
|
||||
* @param opaque custom memory manager handle
|
||||
* @returns @c 0 if instance can not be allocated or initialized
|
||||
* @returns pointer to initialized ::BrotliEncoderState otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
|
||||
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
|
||||
|
||||
/**
|
||||
* Deinitializes and frees ::BrotliEncoderState instance.
|
||||
*
|
||||
* @param state decoder instance to be cleaned up and deallocated
|
||||
*/
|
||||
BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
|
||||
|
||||
/**
|
||||
* Prepends imaginary LZ77 dictionary.
|
||||
*
|
||||
* Fills the fresh ::BrotliEncoderState with additional data corpus for LZ77
|
||||
* backward references.
|
||||
*
|
||||
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
|
||||
*
|
||||
* Workflow:
|
||||
* -# Allocate and initialize state with ::BrotliEncoderCreateInstance
|
||||
* -# Set ::BROTLI_PARAM_LGWIN parameter
|
||||
* -# Invoke ::BrotliEncoderSetCustomDictionary
|
||||
* -# Use ::BrotliEncoderCompressStream
|
||||
* -# Clean up and free state with ::BrotliEncoderDestroyInstance
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @param size length of @p dict; at most "window size" bytes are used
|
||||
* @param dict "dictionary"; @b MUST use same dictionary during decompression
|
||||
*/
|
||||
BROTLI_ENC_API void BrotliEncoderSetCustomDictionary(
|
||||
BrotliEncoderState* state, size_t size,
|
||||
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
|
||||
|
||||
/**
|
||||
* Calculates the output size bound for the given @p input_size.
|
||||
*
|
||||
* @warning Result is not applicable to ::BrotliEncoderCompressStream output,
|
||||
* because every "flush" adds extra overhead bytes, and some encoder
|
||||
* settings (e.g. quality @c 0 and @c 1) might imply a "soft flush"
|
||||
* after every chunk of input.
|
||||
*
|
||||
* @param input_size size of projected input
|
||||
* @returns @c 0 if result does not fit @c size_t
|
||||
*/
|
||||
BROTLI_ENC_API size_t BrotliEncoderMaxCompressedSize(size_t input_size);
|
||||
|
||||
/**
|
||||
* Performs one-shot memory-to-memory compression.
|
||||
*
|
||||
* Compresses the data in @p input_buffer into @p encoded_buffer, and sets
|
||||
* @p *encoded_size to the compressed length.
|
||||
*
|
||||
* @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero
|
||||
* value, then output is guaranteed to be no longer than that.
|
||||
*
|
||||
* @param quality quality parameter value, e.g. ::BROTLI_DEFAULT_QUALITY
|
||||
* @param lgwin lgwin parameter value, e.g. ::BROTLI_DEFAULT_WINDOW
|
||||
* @param mode mode parameter value, e.g. ::BROTLI_DEFAULT_MODE
|
||||
* @param input_size size of @p input_buffer
|
||||
* @param input_buffer input data buffer with at least @p input_size
|
||||
* addressable bytes
|
||||
* @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n
|
||||
* @b out: length of compressed data written to
|
||||
* @p encoded_buffer, or @c 0 if compression fails
|
||||
* @param encoded_buffer compressed data destination buffer
|
||||
* @returns ::BROTLI_FALSE in case of compression error
|
||||
* @returns ::BROTLI_FALSE if output buffer is too small
|
||||
* @returns ::BROTLI_TRUE otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompress(
|
||||
int quality, int lgwin, BrotliEncoderMode mode, size_t input_size,
|
||||
const uint8_t input_buffer[BROTLI_ARRAY_PARAM(input_size)],
|
||||
size_t* encoded_size,
|
||||
uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(*encoded_size)]);
|
||||
|
||||
/**
|
||||
* Compresses input stream to output stream.
|
||||
*
|
||||
* The values @p *available_in and @p *available_out must specify the number of
|
||||
* bytes addressable at @p *next_in and @p *next_out respectively.
|
||||
* When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
|
||||
*
|
||||
* After each call, @p *available_in will be decremented by the amount of input
|
||||
* bytes consumed, and the @p *next_in pointer will be incremented by that
|
||||
* amount. Similarly, @p *available_out will be decremented by the amount of
|
||||
* output bytes written, and the @p *next_out pointer will be incremented by
|
||||
* that amount.
|
||||
*
|
||||
* @p total_out, if it is not a null-pointer, will be set to the number
|
||||
* of bytes decompressed since the last @p state initialization.
|
||||
*
|
||||
*
|
||||
*
|
||||
* Internally workflow consists of 3 tasks:
|
||||
* -# (optionally) copy input data to internal buffer
|
||||
* -# actually compress data and (optionally) store it to internal buffer
|
||||
* -# (optionally) copy compressed bytes from internal buffer to output stream
|
||||
*
|
||||
* Whenever all 3 tasks can't move forward anymore, or error occurs, this
|
||||
* method returns the control flow to caller.
|
||||
*
|
||||
* @p op is used to perform flush, finish the stream, or inject metadata block.
|
||||
* See ::BrotliEncoderOperation for more information.
|
||||
*
|
||||
* Flushing the stream means forcing encoding of all input passed to encoder and
|
||||
* completing the current output block, so it could be fully decoded by stream
|
||||
* decoder. To perform flush set @p op to ::BROTLI_OPERATION_FLUSH.
|
||||
* Under some circumstances (e.g. lack of output stream capacity) this operation
|
||||
* would require several calls to ::BrotliEncoderCompressStream. The method must
|
||||
* be called again until both input stream is depleted and encoder has no more
|
||||
* output (see ::BrotliEncoderHasMoreOutput) after the method is called.
|
||||
*
|
||||
* Finishing the stream means encoding of all input passed to encoder and
|
||||
* adding specific "final" marks, so stream decoder could determine that stream
|
||||
* is complete. To perform finish set @p op to ::BROTLI_OPERATION_FINISH.
|
||||
* Under some circumstances (e.g. lack of output stream capacity) this operation
|
||||
* would require several calls to ::BrotliEncoderCompressStream. The method must
|
||||
* be called again until both input stream is depleted and encoder has no more
|
||||
* output (see ::BrotliEncoderHasMoreOutput) after the method is called.
|
||||
*
|
||||
* @warning When flushing and finishing, @p op should not change until operation
|
||||
* is complete; input stream should not be swapped, reduced or
|
||||
* extended as well.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @param op requested operation
|
||||
* @param[in, out] available_in @b in: amount of available input; \n
|
||||
* @b out: amount of unused input
|
||||
* @param[in, out] next_in pointer to the next input byte
|
||||
* @param[in, out] available_out @b in: length of output buffer; \n
|
||||
* @b out: remaining size of output buffer
|
||||
* @param[in, out] next_out compressed output buffer cursor;
|
||||
* can be @c NULL if @p available_out is @c 0
|
||||
* @param[out] total_out number of bytes produced so far; can be @c NULL
|
||||
* @returns ::BROTLI_FALSE if there was an error
|
||||
* @returns ::BROTLI_TRUE otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompressStream(
|
||||
BrotliEncoderState* state, BrotliEncoderOperation op, size_t* available_in,
|
||||
const uint8_t** next_in, size_t* available_out, uint8_t** next_out,
|
||||
size_t* total_out);
|
||||
|
||||
/**
|
||||
* Checks if encoder instance reached the final state.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @returns ::BROTLI_TRUE if encoder is in a state where it reached the end of
|
||||
* the input and produced all of the output
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderIsFinished(BrotliEncoderState* state);
|
||||
|
||||
/**
|
||||
* Checks if encoder has more output.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @returns ::BROTLI_TRUE, if encoder has some unconsumed output
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
|
||||
BrotliEncoderState* state);
|
||||
|
||||
/**
|
||||
* Acquires pointer to internal output buffer.
|
||||
*
|
||||
* This method is used to make language bindings easier and more efficient:
|
||||
* -# push data to ::BrotliEncoderCompressStream,
|
||||
* until ::BrotliEncoderHasMoreOutput returns BROTL_TRUE
|
||||
* -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific
|
||||
* entity
|
||||
*
|
||||
* Also this could be useful if there is an output stream that is able to
|
||||
* consume all the provided data (e.g. when data is saved to file system).
|
||||
*
|
||||
* @attention After every call to ::BrotliEncoderTakeOutput @p *size bytes of
|
||||
* output are considered consumed for all consecutive calls to the
|
||||
* instance methods; returned pointer becomes invalidated as well.
|
||||
*
|
||||
* @note Encoder output is not guaranteed to be contiguous. This means that
|
||||
* after the size-unrestricted call to ::BrotliEncoderTakeOutput,
|
||||
* immediate next call to ::BrotliEncoderTakeOutput may return more data.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
|
||||
* any amount could be handled; \n
|
||||
* @b out: amount of data pointed by returned pointer and
|
||||
* considered consumed; \n
|
||||
* out value is never greater than in value, unless it is @c 0
|
||||
* @returns pointer to output data
|
||||
*/
|
||||
BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput(
|
||||
BrotliEncoderState* state, size_t* size);
|
||||
|
||||
|
||||
/**
|
||||
* Gets an encoder library version.
|
||||
*
|
||||
* Look at BROTLI_VERSION for more information.
|
||||
*/
|
||||
BROTLI_ENC_API uint32_t BrotliEncoderVersion(void);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_ENC_ENCODE_H_ */
|
||||
|
|
@ -1,146 +0,0 @@
|
|||
/* Copyright 2016 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Macros for compiler / platform specific features and build options. */
|
||||
|
||||
#ifndef BROTLI_COMMON_PORT_H_
|
||||
#define BROTLI_COMMON_PORT_H_
|
||||
|
||||
/* Compatibility with non-clang compilers. */
|
||||
#ifndef __has_builtin
|
||||
#define __has_builtin(x) 0
|
||||
#endif
|
||||
|
||||
#ifndef __has_attribute
|
||||
#define __has_attribute(x) 0
|
||||
#endif
|
||||
|
||||
#ifndef __has_feature
|
||||
#define __has_feature(x) 0
|
||||
#endif
|
||||
|
||||
#if defined(__GNUC__) && defined(__GNUC_MINOR__)
|
||||
#define BROTLI_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
#else
|
||||
#define BROTLI_GCC_VERSION 0
|
||||
#endif
|
||||
|
||||
#if defined(__ICC)
|
||||
#define BROTLI_ICC_VERSION __ICC
|
||||
#else
|
||||
#define BROTLI_ICC_VERSION 0
|
||||
#endif
|
||||
|
||||
#if defined(BROTLI_BUILD_MODERN_COMPILER)
|
||||
#define BROTLI_MODERN_COMPILER 1
|
||||
#elif BROTLI_GCC_VERSION >= 304 || BROTLI_ICC_VERSION >= 1600
|
||||
#define BROTLI_MODERN_COMPILER 1
|
||||
#else
|
||||
#define BROTLI_MODERN_COMPILER 0
|
||||
#endif
|
||||
|
||||
/* Define "BROTLI_PREDICT_TRUE" and "BROTLI_PREDICT_FALSE" macros for capable
|
||||
compilers.
|
||||
|
||||
To apply compiler hint, enclose the branching condition into macros, like this:
|
||||
|
||||
if (BROTLI_PREDICT_TRUE(zero == 0)) {
|
||||
// main execution path
|
||||
} else {
|
||||
// compiler should place this code outside of main execution path
|
||||
}
|
||||
|
||||
OR:
|
||||
|
||||
if (BROTLI_PREDICT_FALSE(something_rare_or_unexpected_happens)) {
|
||||
// compiler should place this code outside of main execution path
|
||||
}
|
||||
|
||||
*/
|
||||
#if BROTLI_MODERN_COMPILER || __has_builtin(__builtin_expect)
|
||||
#define BROTLI_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
|
||||
#define BROTLI_PREDICT_FALSE(x) (__builtin_expect(x, 0))
|
||||
#else
|
||||
#define BROTLI_PREDICT_FALSE(x) (x)
|
||||
#define BROTLI_PREDICT_TRUE(x) (x)
|
||||
#endif
|
||||
|
||||
#if BROTLI_MODERN_COMPILER || __has_attribute(always_inline)
|
||||
#define BROTLI_ATTRIBUTE_ALWAYS_INLINE __attribute__ ((always_inline))
|
||||
#else
|
||||
#define BROTLI_ATTRIBUTE_ALWAYS_INLINE
|
||||
#endif
|
||||
|
||||
#if defined(_WIN32) || defined(__CYGWIN__)
|
||||
#define BROTLI_ATTRIBUTE_VISIBILITY_HIDDEN
|
||||
#elif BROTLI_MODERN_COMPILER || __has_attribute(visibility)
|
||||
#define BROTLI_ATTRIBUTE_VISIBILITY_HIDDEN \
|
||||
__attribute__ ((visibility ("hidden")))
|
||||
#else
|
||||
#define BROTLI_ATTRIBUTE_VISIBILITY_HIDDEN
|
||||
#endif
|
||||
|
||||
#ifndef BROTLI_INTERNAL
|
||||
#define BROTLI_INTERNAL BROTLI_ATTRIBUTE_VISIBILITY_HIDDEN
|
||||
#endif
|
||||
|
||||
#if defined(BROTLI_SHARED_COMPILATION) && defined(_WIN32)
|
||||
#if defined(BROTLICOMMON_SHARED_COMPILATION)
|
||||
#define BROTLI_COMMON_API __declspec(dllexport)
|
||||
#else
|
||||
#define BROTLI_COMMON_API __declspec(dllimport)
|
||||
#endif /* BROTLICOMMON_SHARED_COMPILATION */
|
||||
#if defined(BROTLIDEC_SHARED_COMPILATION)
|
||||
#define BROTLI_DEC_API __declspec(dllexport)
|
||||
#else
|
||||
#define BROTLI_DEC_API __declspec(dllimport)
|
||||
#endif /* BROTLIDEC_SHARED_COMPILATION */
|
||||
#if defined(BROTLIENC_SHARED_COMPILATION)
|
||||
#define BROTLI_ENC_API __declspec(dllexport)
|
||||
#else
|
||||
#define BROTLI_ENC_API __declspec(dllimport)
|
||||
#endif /* BROTLIENC_SHARED_COMPILATION */
|
||||
#else /* BROTLI_SHARED_COMPILATION && _WIN32 */
|
||||
#define BROTLI_COMMON_API
|
||||
#define BROTLI_DEC_API
|
||||
#define BROTLI_ENC_API
|
||||
#endif
|
||||
|
||||
#ifndef _MSC_VER
|
||||
#if defined(__cplusplus) || !defined(__STRICT_ANSI__) || \
|
||||
(defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L)
|
||||
#define BROTLI_INLINE inline BROTLI_ATTRIBUTE_ALWAYS_INLINE
|
||||
#else
|
||||
#define BROTLI_INLINE
|
||||
#endif
|
||||
#else /* _MSC_VER */
|
||||
#define BROTLI_INLINE __forceinline
|
||||
#endif /* _MSC_VER */
|
||||
|
||||
#if !defined(__cplusplus) && !defined(c_plusplus) && \
|
||||
(defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L)
|
||||
#define BROTLI_RESTRICT restrict
|
||||
#elif BROTLI_GCC_VERSION > 295 || defined(__llvm__)
|
||||
#define BROTLI_RESTRICT __restrict
|
||||
#else
|
||||
#define BROTLI_RESTRICT
|
||||
#endif
|
||||
|
||||
#if BROTLI_MODERN_COMPILER || __has_attribute(noinline)
|
||||
#define BROTLI_NOINLINE __attribute__((noinline))
|
||||
#else
|
||||
#define BROTLI_NOINLINE
|
||||
#endif
|
||||
|
||||
#if BROTLI_MODERN_COMPILER || __has_attribute(deprecated)
|
||||
#define BROTLI_DEPRECATED __attribute__((deprecated))
|
||||
#else
|
||||
#define BROTLI_DEPRECATED
|
||||
#endif
|
||||
|
||||
#define BROTLI_UNUSED(X) (void)(X)
|
||||
|
||||
#endif /* BROTLI_COMMON_PORT_H_ */
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/**
|
||||
* @file
|
||||
* Common types used in decoder and encoder API.
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_COMMON_TYPES_H_
|
||||
#define BROTLI_COMMON_TYPES_H_
|
||||
|
||||
#include <stddef.h> /* for size_t */
|
||||
|
||||
#if defined(_MSC_VER) && (_MSC_VER < 1600)
|
||||
typedef __int8 int8_t;
|
||||
typedef unsigned __int8 uint8_t;
|
||||
typedef __int16 int16_t;
|
||||
typedef unsigned __int16 uint16_t;
|
||||
typedef __int32 int32_t;
|
||||
typedef unsigned __int32 uint32_t;
|
||||
typedef unsigned __int64 uint64_t;
|
||||
typedef __int64 int64_t;
|
||||
#else
|
||||
#include <stdint.h>
|
||||
#endif /* defined(_MSC_VER) && (_MSC_VER < 1600) */
|
||||
|
||||
/**
|
||||
* A portable @c bool replacement.
|
||||
*
|
||||
* ::BROTLI_BOOL is a "documentation" type: actually it is @c int, but in API it
|
||||
* denotes a type, whose only values are ::BROTLI_TRUE and ::BROTLI_FALSE.
|
||||
*
|
||||
* ::BROTLI_BOOL values passed to Brotli should either be ::BROTLI_TRUE or
|
||||
* ::BROTLI_FALSE, or be a result of ::TO_BROTLI_BOOL macros.
|
||||
*
|
||||
* ::BROTLI_BOOL values returned by Brotli should not be tested for equality
|
||||
* with @c true, @c false, ::BROTLI_TRUE, ::BROTLI_FALSE, but rather should be
|
||||
* evaluated, for example: @code{.cpp}
|
||||
* if (SomeBrotliFunction(encoder, BROTLI_TRUE) &&
|
||||
* !OtherBrotliFunction(decoder, BROTLI_FALSE)) {
|
||||
* bool x = !!YetAnotherBrotliFunction(encoder, TO_BROLTI_BOOL(2 * 2 == 4));
|
||||
* DoSomething(x);
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
#define BROTLI_BOOL int
|
||||
/** Portable @c true replacement. */
|
||||
#define BROTLI_TRUE 1
|
||||
/** Portable @c false replacement. */
|
||||
#define BROTLI_FALSE 0
|
||||
/** @c bool to ::BROTLI_BOOL conversion macros. */
|
||||
#define TO_BROTLI_BOOL(X) (!!(X) ? BROTLI_TRUE : BROTLI_FALSE)
|
||||
|
||||
#define BROTLI_MAKE_UINT64_T(high, low) ((((uint64_t)(high)) << 32) | low)
|
||||
|
||||
#define BROTLI_UINT32_MAX (~((uint32_t)0))
|
||||
#define BROTLI_SIZE_MAX (~((size_t)0))
|
||||
|
||||
/**
|
||||
* Allocating function pointer type.
|
||||
*
|
||||
* @param opaque custom memory manager handle provided by client
|
||||
* @param size requested memory region size; can not be @c 0
|
||||
* @returns @c 0 in the case of failure
|
||||
* @returns a valid pointer to a memory region of at least @p size bytes
|
||||
* long otherwise
|
||||
*/
|
||||
typedef void* (*brotli_alloc_func)(void* opaque, size_t size);
|
||||
|
||||
/**
|
||||
* Deallocating function pointer type.
|
||||
*
|
||||
* This function @b SHOULD do nothing if @p address is @c 0.
|
||||
*
|
||||
* @param opaque custom memory manager handle provided by client
|
||||
* @param address memory region pointer returned by ::brotli_alloc_func, or @c 0
|
||||
*/
|
||||
typedef void (*brotli_free_func)(void* opaque, void* address);
|
||||
|
||||
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
|
||||
!defined(__cplusplus) && !defined(__PGI)
|
||||
#define BROTLI_ARRAY_PARAM(L) L
|
||||
#else
|
||||
#define BROTLI_ARRAY_PARAM(L)
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_COMMON_TYPES_H_ */
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,19 +0,0 @@
|
|||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Distributed under MIT license.
|
||||
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
|
||||
package brotli
|
||||
|
||||
// Inform golang build system that it should link brotli libraries.
|
||||
|
||||
// #cgo CFLAGS: -O3
|
||||
// #cgo LDFLAGS: -lm
|
||||
import "C"
|
||||
|
||||
import (
|
||||
_ "github.com/cloudflare/brotli-go/brotli"
|
||||
_ "github.com/cloudflare/brotli-go/common"
|
||||
_ "github.com/cloudflare/brotli-go/dec"
|
||||
_ "github.com/cloudflare/brotli-go/enc"
|
||||
)
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Functions for clustering similar histograms together. */
|
||||
|
||||
#include "./enc/cluster.h"
|
||||
|
||||
#include <brotli/types.h>
|
||||
#include "./enc/bit_cost.h" /* BrotliPopulationCost */
|
||||
#include "./enc/fast_log.h"
|
||||
#include "./enc/histogram.h"
|
||||
#include "./enc/memory.h"
|
||||
#include "./enc/port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static BROTLI_INLINE BROTLI_BOOL HistogramPairIsLess(
|
||||
const HistogramPair* p1, const HistogramPair* p2) {
|
||||
if (p1->cost_diff != p2->cost_diff) {
|
||||
return TO_BROTLI_BOOL(p1->cost_diff > p2->cost_diff);
|
||||
}
|
||||
return TO_BROTLI_BOOL((p1->idx2 - p1->idx1) > (p2->idx2 - p2->idx1));
|
||||
}
|
||||
|
||||
/* Returns entropy reduction of the context map when we combine two clusters. */
|
||||
static BROTLI_INLINE double ClusterCostDiff(size_t size_a, size_t size_b) {
|
||||
size_t size_c = size_a + size_b;
|
||||
return (double)size_a * FastLog2(size_a) +
|
||||
(double)size_b * FastLog2(size_b) -
|
||||
(double)size_c * FastLog2(size_c);
|
||||
}
|
||||
|
||||
#define CODE(X) X
|
||||
|
||||
#define FN(X) X ## Literal
|
||||
#include "./enc/cluster_inc.h" /* NOLINT(build/include) */
|
||||
#undef FN
|
||||
|
||||
#define FN(X) X ## Command
|
||||
#include "./enc/cluster_inc.h" /* NOLINT(build/include) */
|
||||
#undef FN
|
||||
|
||||
#define FN(X) X ## Distance
|
||||
#include "./enc/cluster_inc.h" /* NOLINT(build/include) */
|
||||
#undef FN
|
||||
|
||||
#undef CODE
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
package common
|
||||
|
||||
import (
|
||||
"C"
|
||||
)
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
/* Copyright 2016 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_COMMON_CONSTANTS_H_
|
||||
#define BROTLI_COMMON_CONSTANTS_H_
|
||||
|
||||
/* Specification: 7.3. Encoding of the context map */
|
||||
#define BROTLI_CONTEXT_MAP_MAX_RLE 16
|
||||
|
||||
/* Specification: 2. Compressed representation overview */
|
||||
#define BROTLI_MAX_NUMBER_OF_BLOCK_TYPES 256
|
||||
|
||||
/* Specification: 3.3. Alphabet sizes: insert-and-copy length */
|
||||
#define BROTLI_NUM_LITERAL_SYMBOLS 256
|
||||
#define BROTLI_NUM_COMMAND_SYMBOLS 704
|
||||
#define BROTLI_NUM_BLOCK_LEN_SYMBOLS 26
|
||||
#define BROTLI_MAX_CONTEXT_MAP_SYMBOLS (BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + \
|
||||
BROTLI_CONTEXT_MAP_MAX_RLE)
|
||||
#define BROTLI_MAX_BLOCK_TYPE_SYMBOLS (BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + 2)
|
||||
|
||||
/* Specification: 3.5. Complex prefix codes */
|
||||
#define BROTLI_REPEAT_PREVIOUS_CODE_LENGTH 16
|
||||
#define BROTLI_REPEAT_ZERO_CODE_LENGTH 17
|
||||
#define BROTLI_CODE_LENGTH_CODES (BROTLI_REPEAT_ZERO_CODE_LENGTH + 1)
|
||||
/* "code length of 8 is repeated" */
|
||||
#define BROTLI_INITIAL_REPEATED_CODE_LENGTH 8
|
||||
|
||||
/* Specification: 4. Encoding of distances */
|
||||
#define BROTLI_NUM_DISTANCE_SHORT_CODES 16
|
||||
#define BROTLI_MAX_NPOSTFIX 3
|
||||
#define BROTLI_MAX_NDIRECT 120
|
||||
#define BROTLI_MAX_DISTANCE_BITS 24U
|
||||
/* BROTLI_NUM_DISTANCE_SYMBOLS == 520 */
|
||||
#define BROTLI_NUM_DISTANCE_SYMBOLS (BROTLI_NUM_DISTANCE_SHORT_CODES + \
|
||||
BROTLI_MAX_NDIRECT + \
|
||||
(BROTLI_MAX_DISTANCE_BITS << \
|
||||
(BROTLI_MAX_NPOSTFIX + 1)))
|
||||
|
||||
/* 7.1. Context modes and context ID lookup for literals */
|
||||
/* "context IDs for literals are in the range of 0..63" */
|
||||
#define BROTLI_LITERAL_CONTEXT_BITS 6
|
||||
|
||||
/* 7.2. Context ID for distances */
|
||||
#define BROTLI_DISTANCE_CONTEXT_BITS 2
|
||||
|
||||
/* 9.1. Format of the Stream Header */
|
||||
/* Number of slack bytes for window size. Don't confuse
|
||||
with BROTLI_NUM_DISTANCE_SHORT_CODES. */
|
||||
#define BROTLI_WINDOW_GAP 16
|
||||
#define BROTLI_MAX_BACKWARD_LIMIT(W) (((size_t)1 << (W)) - BROTLI_WINDOW_GAP)
|
||||
|
||||
#endif /* BROTLI_COMMON_CONSTANTS_H_ */
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Collection of static dictionary words. */
|
||||
|
||||
#ifndef BROTLI_COMMON_DICTIONARY_H_
|
||||
#define BROTLI_COMMON_DICTIONARY_H_
|
||||
|
||||
#include <brotli/port.h>
|
||||
#include <brotli/types.h>
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct BrotliDictionary {
|
||||
/**
|
||||
* Number of bits to encode index of dictionary word in a bucket.
|
||||
*
|
||||
* Specification: Appendix A. Static Dictionary Data
|
||||
*
|
||||
* Words in a dictionary are bucketed by length.
|
||||
* @c 0 means that there are no words of a given length.
|
||||
* Dictionary consists of words with length of [4..24] bytes.
|
||||
* Values at [0..3] and [25..31] indices should not be addressed.
|
||||
*/
|
||||
const uint8_t size_bits_by_length[32];
|
||||
|
||||
/* assert(offset[i + 1] == offset[i] + (bits[i] ? (i << bits[i]) : 0)) */
|
||||
const uint32_t offsets_by_length[32];
|
||||
|
||||
/* assert(data_size == offsets_by_length[31]) */
|
||||
const size_t data_size;
|
||||
|
||||
/* Data array is not bound, and should obey to size_bits_by_length values.
|
||||
Specified size matches default (RFC 7932) dictionary. Its size is
|
||||
defined by data_size */
|
||||
const uint8_t* data;
|
||||
} BrotliDictionary;
|
||||
|
||||
BROTLI_COMMON_API extern const BrotliDictionary* BrotliGetDictionary(void);
|
||||
|
||||
/**
|
||||
* Sets dictionary data.
|
||||
*
|
||||
* When dictionary data is already set / present, this method is no-op.
|
||||
*
|
||||
* Dictionary data MUST be provided before BrotliGetDictionary is invoked.
|
||||
* This method is used ONLY in multi-client environment (e.g. C + Java),
|
||||
* to reduce storage by sharing single dictionary between implementations.
|
||||
*/
|
||||
BROTLI_COMMON_API void BrotliSetDictionaryData(const uint8_t* data);
|
||||
|
||||
#define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4
|
||||
#define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_COMMON_DICTIONARY_H_ */
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
/* Copyright 2016 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Version definition. */
|
||||
|
||||
#ifndef BROTLI_COMMON_VERSION_H_
|
||||
#define BROTLI_COMMON_VERSION_H_
|
||||
|
||||
/* This macro should only be used when library is compiled together with client.
|
||||
If library is dynamically linked, use BrotliDecoderVersion and
|
||||
BrotliEncoderVersion methods. */
|
||||
|
||||
/* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */
|
||||
#define BROTLI_VERSION 0x1000000
|
||||
|
||||
#endif /* BROTLI_COMMON_VERSION_H_ */
|
||||
|
|
@ -1,791 +0,0 @@
|
|||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function for fast encoding of an input fragment, independently from the input
|
||||
history. This function uses one-pass processing: when we find a backward
|
||||
match, we immediately emit the corresponding command and literal codes to
|
||||
the bit stream.
|
||||
|
||||
Adapted from the CompressFragment() function in
|
||||
https://github.com/google/snappy/blob/master/snappy.cc */
|
||||
|
||||
#include "./enc/compress_fragment.h"
|
||||
|
||||
#include <string.h> /* memcmp, memcpy, memset */
|
||||
|
||||
#include "./common/constants.h"
|
||||
#include <brotli/types.h>
|
||||
#include "./enc/brotli_bit_stream.h"
|
||||
#include "./enc/entropy_encode.h"
|
||||
#include "./enc/fast_log.h"
|
||||
#include "./enc/find_match_length.h"
|
||||
#include "./enc/memory.h"
|
||||
#include "./enc/port.h"
|
||||
#include "./enc/write_bits.h"
|
||||
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define MAX_DISTANCE (long)BROTLI_MAX_BACKWARD_LIMIT(18)
|
||||
|
||||
/* kHashMul32 multiplier has these properties:
|
||||
* The multiplier must be odd. Otherwise we may lose the highest bit.
|
||||
* No long streaks of ones or zeros.
|
||||
* There is no effort to ensure that it is a prime, the oddity is enough
|
||||
for this use.
|
||||
* The number has been tuned heuristically against compression benchmarks. */
|
||||
static const uint32_t kHashMul32 = 0x1e35a7bd;
|
||||
|
||||
static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
|
||||
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 24) * kHashMul32;
|
||||
return (uint32_t)(h >> shift);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t HashBytesAtOffset(
|
||||
uint64_t v, int offset, size_t shift) {
|
||||
assert(offset >= 0);
|
||||
assert(offset <= 3);
|
||||
{
|
||||
const uint64_t h = ((v >> (8 * offset)) << 24) * kHashMul32;
|
||||
return (uint32_t)(h >> shift);
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE BROTLI_BOOL IsMatch(const uint8_t* p1, const uint8_t* p2) {
|
||||
return TO_BROTLI_BOOL(
|
||||
BROTLI_UNALIGNED_LOAD32(p1) == BROTLI_UNALIGNED_LOAD32(p2) &&
|
||||
p1[4] == p2[4]);
|
||||
}
|
||||
|
||||
/* Builds a literal prefix code into "depths" and "bits" based on the statistics
|
||||
of the "input" string and stores it into the bit stream.
|
||||
Note that the prefix code here is built from the pre-LZ77 input, therefore
|
||||
we can only approximate the statistics of the actual literal stream.
|
||||
Moreover, for long inputs we build a histogram from a sample of the input
|
||||
and thus have to assign a non-zero depth for each literal.
|
||||
Returns estimated compression ratio millibytes/char for encoding given input
|
||||
with generated code. */
|
||||
static size_t BuildAndStoreLiteralPrefixCode(MemoryManager* m,
|
||||
const uint8_t* input,
|
||||
const size_t input_size,
|
||||
uint8_t depths[256],
|
||||
uint16_t bits[256],
|
||||
size_t* storage_ix,
|
||||
uint8_t* storage) {
|
||||
uint32_t histogram[256] = { 0 };
|
||||
size_t histogram_total;
|
||||
size_t i;
|
||||
if (input_size < (1 << 15)) {
|
||||
for (i = 0; i < input_size; ++i) {
|
||||
++histogram[input[i]];
|
||||
}
|
||||
histogram_total = input_size;
|
||||
for (i = 0; i < 256; ++i) {
|
||||
/* We weigh the first 11 samples with weight 3 to account for the
|
||||
balancing effect of the LZ77 phase on the histogram. */
|
||||
const uint32_t adjust = 2 * BROTLI_MIN(uint32_t, histogram[i], 11u);
|
||||
histogram[i] += adjust;
|
||||
histogram_total += adjust;
|
||||
}
|
||||
} else {
|
||||
static const size_t kSampleRate = 29;
|
||||
for (i = 0; i < input_size; i += kSampleRate) {
|
||||
++histogram[input[i]];
|
||||
}
|
||||
histogram_total = (input_size + kSampleRate - 1) / kSampleRate;
|
||||
for (i = 0; i < 256; ++i) {
|
||||
/* We add 1 to each population count to avoid 0 bit depths (since this is
|
||||
only a sample and we don't know if the symbol appears or not), and we
|
||||
weigh the first 11 samples with weight 3 to account for the balancing
|
||||
effect of the LZ77 phase on the histogram (more frequent symbols are
|
||||
more likely to be in backward references instead as literals). */
|
||||
const uint32_t adjust = 1 + 2 * BROTLI_MIN(uint32_t, histogram[i], 11u);
|
||||
histogram[i] += adjust;
|
||||
histogram_total += adjust;
|
||||
}
|
||||
}
|
||||
BrotliBuildAndStoreHuffmanTreeFast(m, histogram, histogram_total,
|
||||
/* max_bits = */ 8,
|
||||
depths, bits, storage_ix, storage);
|
||||
if (BROTLI_IS_OOM(m)) return 0;
|
||||
{
|
||||
size_t literal_ratio = 0;
|
||||
for (i = 0; i < 256; ++i) {
|
||||
if (histogram[i]) literal_ratio += histogram[i] * depths[i];
|
||||
}
|
||||
/* Estimated encoding ratio, millibytes per symbol. */
|
||||
return (literal_ratio * 125) / histogram_total;
|
||||
}
|
||||
}
|
||||
|
||||
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
|
||||
"bits" based on "histogram" and stores it into the bit stream. */
|
||||
static void BuildAndStoreCommandPrefixCode(const uint32_t histogram[128],
|
||||
uint8_t depth[128], uint16_t bits[128], size_t* storage_ix,
|
||||
uint8_t* storage) {
|
||||
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
|
||||
HuffmanTree tree[129];
|
||||
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS] = { 0 };
|
||||
uint16_t cmd_bits[64];
|
||||
|
||||
BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth);
|
||||
BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]);
|
||||
/* We have to jump through a few hoops here in order to compute
|
||||
the command bits because the symbols are in a different order than in
|
||||
the full alphabet. This looks complicated, but having the symbols
|
||||
in this order in the command bits saves a few branches in the Emit*
|
||||
functions. */
|
||||
memcpy(cmd_depth, depth, 24);
|
||||
memcpy(cmd_depth + 24, depth + 40, 8);
|
||||
memcpy(cmd_depth + 32, depth + 24, 8);
|
||||
memcpy(cmd_depth + 40, depth + 48, 8);
|
||||
memcpy(cmd_depth + 48, depth + 32, 8);
|
||||
memcpy(cmd_depth + 56, depth + 56, 8);
|
||||
BrotliConvertBitDepthsToSymbols(cmd_depth, 64, cmd_bits);
|
||||
memcpy(bits, cmd_bits, 48);
|
||||
memcpy(bits + 24, cmd_bits + 32, 16);
|
||||
memcpy(bits + 32, cmd_bits + 48, 16);
|
||||
memcpy(bits + 40, cmd_bits + 24, 16);
|
||||
memcpy(bits + 48, cmd_bits + 40, 16);
|
||||
memcpy(bits + 56, cmd_bits + 56, 16);
|
||||
BrotliConvertBitDepthsToSymbols(&depth[64], 64, &bits[64]);
|
||||
{
|
||||
/* Create the bit length array for the full command alphabet. */
|
||||
size_t i;
|
||||
memset(cmd_depth, 0, 64); /* only 64 first values were used */
|
||||
memcpy(cmd_depth, depth, 8);
|
||||
memcpy(cmd_depth + 64, depth + 8, 8);
|
||||
memcpy(cmd_depth + 128, depth + 16, 8);
|
||||
memcpy(cmd_depth + 192, depth + 24, 8);
|
||||
memcpy(cmd_depth + 384, depth + 32, 8);
|
||||
for (i = 0; i < 8; ++i) {
|
||||
cmd_depth[128 + 8 * i] = depth[40 + i];
|
||||
cmd_depth[256 + 8 * i] = depth[48 + i];
|
||||
cmd_depth[448 + 8 * i] = depth[56 + i];
|
||||
}
|
||||
BrotliStoreHuffmanTree(
|
||||
cmd_depth, BROTLI_NUM_COMMAND_SYMBOLS, tree, storage_ix, storage);
|
||||
}
|
||||
BrotliStoreHuffmanTree(&depth[64], 64, tree, storage_ix, storage);
|
||||
}
|
||||
|
||||
/* REQUIRES: insertlen < 6210 */
|
||||
static BROTLI_INLINE void EmitInsertLen(size_t insertlen,
|
||||
const uint8_t depth[128],
|
||||
const uint16_t bits[128],
|
||||
uint32_t histo[128],
|
||||
size_t* storage_ix,
|
||||
uint8_t* storage) {
|
||||
if (insertlen < 6) {
|
||||
const size_t code = insertlen + 40;
|
||||
BrotliWriteBits(depth[code], bits[code], storage_ix, storage);
|
||||
++histo[code];
|
||||
} else if (insertlen < 130) {
|
||||
const size_t tail = insertlen - 2;
|
||||
const uint32_t nbits = Log2FloorNonZero(tail) - 1u;
|
||||
const size_t prefix = tail >> nbits;
|
||||
const size_t inscode = (nbits << 1) + prefix + 42;
|
||||
BrotliWriteBits(depth[inscode], bits[inscode], storage_ix, storage);
|
||||
BrotliWriteBits(nbits, tail - (prefix << nbits), storage_ix, storage);
|
||||
++histo[inscode];
|
||||
} else if (insertlen < 2114) {
|
||||
const size_t tail = insertlen - 66;
|
||||
const uint32_t nbits = Log2FloorNonZero(tail);
|
||||
const size_t code = nbits + 50;
|
||||
BrotliWriteBits(depth[code], bits[code], storage_ix, storage);
|
||||
BrotliWriteBits(nbits, tail - ((size_t)1 << nbits), storage_ix, storage);
|
||||
++histo[code];
|
||||
} else {
|
||||
BrotliWriteBits(depth[61], bits[61], storage_ix, storage);
|
||||
BrotliWriteBits(12, insertlen - 2114, storage_ix, storage);
|
||||
++histo[21];
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void EmitLongInsertLen(size_t insertlen,
|
||||
const uint8_t depth[128],
|
||||
const uint16_t bits[128],
|
||||
uint32_t histo[128],
|
||||
size_t* storage_ix,
|
||||
uint8_t* storage) {
|
||||
if (insertlen < 22594) {
|
||||
BrotliWriteBits(depth[62], bits[62], storage_ix, storage);
|
||||
BrotliWriteBits(14, insertlen - 6210, storage_ix, storage);
|
||||
++histo[22];
|
||||
} else {
|
||||
BrotliWriteBits(depth[63], bits[63], storage_ix, storage);
|
||||
BrotliWriteBits(24, insertlen - 22594, storage_ix, storage);
|
||||
++histo[23];
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void EmitCopyLen(size_t copylen,
|
||||
const uint8_t depth[128],
|
||||
const uint16_t bits[128],
|
||||
uint32_t histo[128],
|
||||
size_t* storage_ix,
|
||||
uint8_t* storage) {
|
||||
if (copylen < 10) {
|
||||
BrotliWriteBits(
|
||||
depth[copylen + 14], bits[copylen + 14], storage_ix, storage);
|
||||
++histo[copylen + 14];
|
||||
} else if (copylen < 134) {
|
||||
const size_t tail = copylen - 6;
|
||||
const uint32_t nbits = Log2FloorNonZero(tail) - 1u;
|
||||
const size_t prefix = tail >> nbits;
|
||||
const size_t code = (nbits << 1) + prefix + 20;
|
||||
BrotliWriteBits(depth[code], bits[code], storage_ix, storage);
|
||||
BrotliWriteBits(nbits, tail - (prefix << nbits), storage_ix, storage);
|
||||
++histo[code];
|
||||
} else if (copylen < 2118) {
|
||||
const size_t tail = copylen - 70;
|
||||
const uint32_t nbits = Log2FloorNonZero(tail);
|
||||
const size_t code = nbits + 28;
|
||||
BrotliWriteBits(depth[code], bits[code], storage_ix, storage);
|
||||
BrotliWriteBits(nbits, tail - ((size_t)1 << nbits), storage_ix, storage);
|
||||
++histo[code];
|
||||
} else {
|
||||
BrotliWriteBits(depth[39], bits[39], storage_ix, storage);
|
||||
BrotliWriteBits(24, copylen - 2118, storage_ix, storage);
|
||||
++histo[47];
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void EmitCopyLenLastDistance(size_t copylen,
|
||||
const uint8_t depth[128],
|
||||
const uint16_t bits[128],
|
||||
uint32_t histo[128],
|
||||
size_t* storage_ix,
|
||||
uint8_t* storage) {
|
||||
if (copylen < 12) {
|
||||
BrotliWriteBits(depth[copylen - 4], bits[copylen - 4], storage_ix, storage);
|
||||
++histo[copylen - 4];
|
||||
} else if (copylen < 72) {
|
||||
const size_t tail = copylen - 8;
|
||||
const uint32_t nbits = Log2FloorNonZero(tail) - 1;
|
||||
const size_t prefix = tail >> nbits;
|
||||
const size_t code = (nbits << 1) + prefix + 4;
|
||||
BrotliWriteBits(depth[code], bits[code], storage_ix, storage);
|
||||
BrotliWriteBits(nbits, tail - (prefix << nbits), storage_ix, storage);
|
||||
++histo[code];
|
||||
} else if (copylen < 136) {
|
||||
const size_t tail = copylen - 8;
|
||||
const size_t code = (tail >> 5) + 30;
|
||||
BrotliWriteBits(depth[code], bits[code], storage_ix, storage);
|
||||
BrotliWriteBits(5, tail & 31, storage_ix, storage);
|
||||
BrotliWriteBits(depth[64], bits[64], storage_ix, storage);
|
||||
++histo[code];
|
||||
++histo[64];
|
||||
} else if (copylen < 2120) {
|
||||
const size_t tail = copylen - 72;
|
||||
const uint32_t nbits = Log2FloorNonZero(tail);
|
||||
const size_t code = nbits + 28;
|
||||
BrotliWriteBits(depth[code], bits[code], storage_ix, storage);
|
||||
BrotliWriteBits(nbits, tail - ((size_t)1 << nbits), storage_ix, storage);
|
||||
BrotliWriteBits(depth[64], bits[64], storage_ix, storage);
|
||||
++histo[code];
|
||||
++histo[64];
|
||||
} else {
|
||||
BrotliWriteBits(depth[39], bits[39], storage_ix, storage);
|
||||
BrotliWriteBits(24, copylen - 2120, storage_ix, storage);
|
||||
BrotliWriteBits(depth[64], bits[64], storage_ix, storage);
|
||||
++histo[47];
|
||||
++histo[64];
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void EmitDistance(size_t distance,
|
||||
const uint8_t depth[128],
|
||||
const uint16_t bits[128],
|
||||
uint32_t histo[128],
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
const size_t d = distance + 3;
|
||||
const uint32_t nbits = Log2FloorNonZero(d) - 1u;
|
||||
const size_t prefix = (d >> nbits) & 1;
|
||||
const size_t offset = (2 + prefix) << nbits;
|
||||
const size_t distcode = 2 * (nbits - 1) + prefix + 80;
|
||||
BrotliWriteBits(depth[distcode], bits[distcode], storage_ix, storage);
|
||||
BrotliWriteBits(nbits, d - offset, storage_ix, storage);
|
||||
++histo[distcode];
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void EmitLiterals(const uint8_t* input, const size_t len,
|
||||
const uint8_t depth[256],
|
||||
const uint16_t bits[256],
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
size_t j;
|
||||
for (j = 0; j < len; j++) {
|
||||
const uint8_t lit = input[j];
|
||||
BrotliWriteBits(depth[lit], bits[lit], storage_ix, storage);
|
||||
}
|
||||
}
|
||||
|
||||
/* REQUIRES: len <= 1 << 24. */
|
||||
static void BrotliStoreMetaBlockHeader(
|
||||
size_t len, BROTLI_BOOL is_uncompressed, size_t* storage_ix,
|
||||
uint8_t* storage) {
|
||||
size_t nibbles = 6;
|
||||
/* ISLAST */
|
||||
BrotliWriteBits(1, 0, storage_ix, storage);
|
||||
if (len <= (1U << 16)) {
|
||||
nibbles = 4;
|
||||
} else if (len <= (1U << 20)) {
|
||||
nibbles = 5;
|
||||
}
|
||||
BrotliWriteBits(2, nibbles - 4, storage_ix, storage);
|
||||
BrotliWriteBits(nibbles * 4, len - 1, storage_ix, storage);
|
||||
/* ISUNCOMPRESSED */
|
||||
BrotliWriteBits(1, (uint64_t)is_uncompressed, storage_ix, storage);
|
||||
}
|
||||
|
||||
static void UpdateBits(size_t n_bits, uint32_t bits, size_t pos,
|
||||
uint8_t *array) {
|
||||
while (n_bits > 0) {
|
||||
size_t byte_pos = pos >> 3;
|
||||
size_t n_unchanged_bits = pos & 7;
|
||||
size_t n_changed_bits = BROTLI_MIN(size_t, n_bits, 8 - n_unchanged_bits);
|
||||
size_t total_bits = n_unchanged_bits + n_changed_bits;
|
||||
uint32_t mask =
|
||||
(~((1u << total_bits) - 1u)) | ((1u << n_unchanged_bits) - 1u);
|
||||
uint32_t unchanged_bits = array[byte_pos] & mask;
|
||||
uint32_t changed_bits = bits & ((1u << n_changed_bits) - 1u);
|
||||
array[byte_pos] =
|
||||
(uint8_t)((changed_bits << n_unchanged_bits) | unchanged_bits);
|
||||
n_bits -= n_changed_bits;
|
||||
bits >>= n_changed_bits;
|
||||
pos += n_changed_bits;
|
||||
}
|
||||
}
|
||||
|
||||
static void RewindBitPosition(const size_t new_storage_ix,
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
const size_t bitpos = new_storage_ix & 7;
|
||||
const size_t mask = (1u << bitpos) - 1;
|
||||
storage[new_storage_ix >> 3] &= (uint8_t)mask;
|
||||
*storage_ix = new_storage_ix;
|
||||
}
|
||||
|
||||
static BROTLI_BOOL ShouldMergeBlock(
|
||||
const uint8_t* data, size_t len, const uint8_t* depths) {
|
||||
size_t histo[256] = { 0 };
|
||||
static const size_t kSampleRate = 43;
|
||||
size_t i;
|
||||
for (i = 0; i < len; i += kSampleRate) {
|
||||
++histo[data[i]];
|
||||
}
|
||||
{
|
||||
const size_t total = (len + kSampleRate - 1) / kSampleRate;
|
||||
double r = (FastLog2(total) + 0.5) * (double)total + 200;
|
||||
for (i = 0; i < 256; ++i) {
|
||||
r -= (double)histo[i] * (depths[i] + FastLog2(histo[i]));
|
||||
}
|
||||
return TO_BROTLI_BOOL(r >= 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
/* Acceptable loss for uncompressible speedup is 2% */
|
||||
#define MIN_RATIO 980
|
||||
|
||||
static BROTLI_INLINE BROTLI_BOOL ShouldUseUncompressedMode(
|
||||
const uint8_t* metablock_start, const uint8_t* next_emit,
|
||||
const size_t insertlen, const size_t literal_ratio) {
|
||||
const size_t compressed = (size_t)(next_emit - metablock_start);
|
||||
if (compressed * 50 > insertlen) {
|
||||
return BROTLI_FALSE;
|
||||
} else {
|
||||
return TO_BROTLI_BOOL(literal_ratio > MIN_RATIO);
|
||||
}
|
||||
}
|
||||
|
||||
static void EmitUncompressedMetaBlock(const uint8_t* begin, const uint8_t* end,
|
||||
const size_t storage_ix_start,
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
const size_t len = (size_t)(end - begin);
|
||||
RewindBitPosition(storage_ix_start, storage_ix, storage);
|
||||
BrotliStoreMetaBlockHeader(len, 1, storage_ix, storage);
|
||||
*storage_ix = (*storage_ix + 7u) & ~7u;
|
||||
memcpy(&storage[*storage_ix >> 3], begin, len);
|
||||
*storage_ix += len << 3;
|
||||
storage[*storage_ix >> 3] = 0;
|
||||
}
|
||||
|
||||
static uint32_t kCmdHistoSeed[128] = {
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 0, 0, 0, 0,
|
||||
};
|
||||
|
||||
static BROTLI_INLINE void BrotliCompressFragmentFastImpl(
|
||||
MemoryManager* m, const uint8_t* input, size_t input_size,
|
||||
BROTLI_BOOL is_last, int* table, size_t table_bits, uint8_t cmd_depth[128],
|
||||
uint16_t cmd_bits[128], size_t* cmd_code_numbits, uint8_t* cmd_code,
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
uint32_t cmd_histo[128];
|
||||
const uint8_t* ip_end;
|
||||
|
||||
/* "next_emit" is a pointer to the first byte that is not covered by a
|
||||
previous copy. Bytes between "next_emit" and the start of the next copy or
|
||||
the end of the input will be emitted as literal bytes. */
|
||||
const uint8_t* next_emit = input;
|
||||
/* Save the start of the first block for position and distance computations.
|
||||
*/
|
||||
const uint8_t* base_ip = input;
|
||||
|
||||
static const size_t kFirstBlockSize = 3 << 15;
|
||||
static const size_t kMergeBlockSize = 1 << 16;
|
||||
|
||||
const size_t kInputMarginBytes = BROTLI_WINDOW_GAP;
|
||||
const size_t kMinMatchLen = 5;
|
||||
|
||||
const uint8_t* metablock_start = input;
|
||||
size_t block_size = BROTLI_MIN(size_t, input_size, kFirstBlockSize);
|
||||
size_t total_block_size = block_size;
|
||||
/* Save the bit position of the MLEN field of the meta-block header, so that
|
||||
we can update it later if we decide to extend this meta-block. */
|
||||
size_t mlen_storage_ix = *storage_ix + 3;
|
||||
|
||||
uint8_t lit_depth[256];
|
||||
uint16_t lit_bits[256];
|
||||
|
||||
size_t literal_ratio;
|
||||
|
||||
const uint8_t* ip;
|
||||
int last_distance;
|
||||
|
||||
const size_t shift = 64u - table_bits;
|
||||
|
||||
BrotliStoreMetaBlockHeader(block_size, 0, storage_ix, storage);
|
||||
/* No block splits, no contexts. */
|
||||
BrotliWriteBits(13, 0, storage_ix, storage);
|
||||
|
||||
literal_ratio = BuildAndStoreLiteralPrefixCode(
|
||||
m, input, block_size, lit_depth, lit_bits, storage_ix, storage);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
|
||||
{
|
||||
/* Store the pre-compressed command and distance prefix codes. */
|
||||
size_t i;
|
||||
for (i = 0; i + 7 < *cmd_code_numbits; i += 8) {
|
||||
BrotliWriteBits(8, cmd_code[i >> 3], storage_ix, storage);
|
||||
}
|
||||
}
|
||||
BrotliWriteBits(*cmd_code_numbits & 7, cmd_code[*cmd_code_numbits >> 3],
|
||||
storage_ix, storage);
|
||||
|
||||
emit_commands:
|
||||
/* Initialize the command and distance histograms. We will gather
|
||||
statistics of command and distance codes during the processing
|
||||
of this block and use it to update the command and distance
|
||||
prefix codes for the next block. */
|
||||
memcpy(cmd_histo, kCmdHistoSeed, sizeof(kCmdHistoSeed));
|
||||
|
||||
/* "ip" is the input pointer. */
|
||||
ip = input;
|
||||
last_distance = -1;
|
||||
ip_end = input + block_size;
|
||||
|
||||
if (BROTLI_PREDICT_TRUE(block_size >= kInputMarginBytes)) {
|
||||
/* For the last block, we need to keep a 16 bytes margin so that we can be
|
||||
sure that all distances are at most window size - 16.
|
||||
For all other blocks, we only need to keep a margin of 5 bytes so that
|
||||
we don't go over the block size with a copy. */
|
||||
const size_t len_limit = BROTLI_MIN(size_t, block_size - kMinMatchLen,
|
||||
input_size - kInputMarginBytes);
|
||||
const uint8_t* ip_limit = input + len_limit;
|
||||
|
||||
uint32_t next_hash;
|
||||
for (next_hash = Hash(++ip, shift); ; ) {
|
||||
/* Step 1: Scan forward in the input looking for a 5-byte-long match.
|
||||
If we get close to exhausting the input then goto emit_remainder.
|
||||
|
||||
Heuristic match skipping: If 32 bytes are scanned with no matches
|
||||
found, start looking only at every other byte. If 32 more bytes are
|
||||
scanned, look at every third byte, etc.. When a match is found,
|
||||
immediately go back to looking at every byte. This is a small loss
|
||||
(~5% performance, ~0.1% density) for compressible data due to more
|
||||
bookkeeping, but for non-compressible data (such as JPEG) it's a huge
|
||||
win since the compressor quickly "realizes" the data is incompressible
|
||||
and doesn't bother looking for matches everywhere.
|
||||
|
||||
The "skip" variable keeps track of how many bytes there are since the
|
||||
last match; dividing it by 32 (i.e. right-shifting by five) gives the
|
||||
number of bytes to move ahead for each iteration. */
|
||||
uint32_t skip = 32;
|
||||
|
||||
const uint8_t* next_ip = ip;
|
||||
const uint8_t* candidate;
|
||||
assert(next_emit < ip);
|
||||
trawl:
|
||||
do {
|
||||
uint32_t hash = next_hash;
|
||||
uint32_t bytes_between_hash_lookups = skip++ >> 5;
|
||||
assert(hash == Hash(next_ip, shift));
|
||||
ip = next_ip;
|
||||
next_ip = ip + bytes_between_hash_lookups;
|
||||
if (BROTLI_PREDICT_FALSE(next_ip > ip_limit)) {
|
||||
goto emit_remainder;
|
||||
}
|
||||
next_hash = Hash(next_ip, shift);
|
||||
candidate = ip - last_distance;
|
||||
if (IsMatch(ip, candidate)) {
|
||||
if (BROTLI_PREDICT_TRUE(candidate < ip)) {
|
||||
table[hash] = (int)(ip - base_ip);
|
||||
break;
|
||||
}
|
||||
}
|
||||
candidate = base_ip + table[hash];
|
||||
assert(candidate >= base_ip);
|
||||
assert(candidate < ip);
|
||||
|
||||
table[hash] = (int)(ip - base_ip);
|
||||
} while (BROTLI_PREDICT_TRUE(!IsMatch(ip, candidate)));
|
||||
|
||||
/* Check copy distance. If candidate is not feasible, continue search.
|
||||
Checking is done outside of hot loop to reduce overhead. */
|
||||
if (ip - candidate > MAX_DISTANCE) goto trawl;
|
||||
|
||||
/* Step 2: Emit the found match together with the literal bytes from
|
||||
"next_emit" to the bit stream, and then see if we can find a next match
|
||||
immediately afterwards. Repeat until we find no match for the input
|
||||
without emitting some literal bytes. */
|
||||
|
||||
{
|
||||
/* We have a 5-byte match at ip, and we need to emit bytes in
|
||||
[next_emit, ip). */
|
||||
const uint8_t* base = ip;
|
||||
size_t matched = 5 + FindMatchLengthWithLimit(
|
||||
candidate + 5, ip + 5, (size_t)(ip_end - ip) - 5);
|
||||
int distance = (int)(base - candidate); /* > 0 */
|
||||
size_t insert = (size_t)(base - next_emit);
|
||||
ip += matched;
|
||||
assert(0 == memcmp(base, candidate, matched));
|
||||
if (BROTLI_PREDICT_TRUE(insert < 6210)) {
|
||||
EmitInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
|
||||
storage_ix, storage);
|
||||
} else if (ShouldUseUncompressedMode(metablock_start, next_emit, insert,
|
||||
literal_ratio)) {
|
||||
EmitUncompressedMetaBlock(metablock_start, base, mlen_storage_ix - 3,
|
||||
storage_ix, storage);
|
||||
input_size -= (size_t)(base - input);
|
||||
input = base;
|
||||
next_emit = input;
|
||||
goto next_block;
|
||||
} else {
|
||||
EmitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
|
||||
storage_ix, storage);
|
||||
}
|
||||
EmitLiterals(next_emit, insert, lit_depth, lit_bits,
|
||||
storage_ix, storage);
|
||||
if (distance == last_distance) {
|
||||
BrotliWriteBits(cmd_depth[64], cmd_bits[64], storage_ix, storage);
|
||||
++cmd_histo[64];
|
||||
} else {
|
||||
EmitDistance((size_t)distance, cmd_depth, cmd_bits,
|
||||
cmd_histo, storage_ix, storage);
|
||||
last_distance = distance;
|
||||
}
|
||||
EmitCopyLenLastDistance(matched, cmd_depth, cmd_bits, cmd_histo,
|
||||
storage_ix, storage);
|
||||
|
||||
next_emit = ip;
|
||||
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
|
||||
goto emit_remainder;
|
||||
}
|
||||
/* We could immediately start working at ip now, but to improve
|
||||
compression we first update "table" with the hashes of some positions
|
||||
within the last copy. */
|
||||
{
|
||||
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3);
|
||||
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
|
||||
uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 3);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 1, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 2);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 1);
|
||||
|
||||
candidate = base_ip + table[cur_hash];
|
||||
table[cur_hash] = (int)(ip - base_ip);
|
||||
}
|
||||
}
|
||||
|
||||
while (IsMatch(ip, candidate)) {
|
||||
/* We have a 5-byte match at ip, and no need to emit any literal bytes
|
||||
prior to ip. */
|
||||
const uint8_t* base = ip;
|
||||
size_t matched = 5 + FindMatchLengthWithLimit(
|
||||
candidate + 5, ip + 5, (size_t)(ip_end - ip) - 5);
|
||||
if (ip - candidate > MAX_DISTANCE) break;
|
||||
ip += matched;
|
||||
last_distance = (int)(base - candidate); /* > 0 */
|
||||
assert(0 == memcmp(base, candidate, matched));
|
||||
EmitCopyLen(matched, cmd_depth, cmd_bits, cmd_histo,
|
||||
storage_ix, storage);
|
||||
EmitDistance((size_t)last_distance, cmd_depth, cmd_bits,
|
||||
cmd_histo, storage_ix, storage);
|
||||
|
||||
next_emit = ip;
|
||||
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
|
||||
goto emit_remainder;
|
||||
}
|
||||
/* We could immediately start working at ip now, but to improve
|
||||
compression we first update "table" with the hashes of some positions
|
||||
within the last copy. */
|
||||
{
|
||||
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3);
|
||||
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
|
||||
uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 3);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 1, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 2);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 1);
|
||||
|
||||
candidate = base_ip + table[cur_hash];
|
||||
table[cur_hash] = (int)(ip - base_ip);
|
||||
}
|
||||
}
|
||||
|
||||
next_hash = Hash(++ip, shift);
|
||||
}
|
||||
}
|
||||
|
||||
emit_remainder:
|
||||
assert(next_emit <= ip_end);
|
||||
input += block_size;
|
||||
input_size -= block_size;
|
||||
block_size = BROTLI_MIN(size_t, input_size, kMergeBlockSize);
|
||||
|
||||
/* Decide if we want to continue this meta-block instead of emitting the
|
||||
last insert-only command. */
|
||||
if (input_size > 0 &&
|
||||
total_block_size + block_size <= (1 << 20) &&
|
||||
ShouldMergeBlock(input, block_size, lit_depth)) {
|
||||
assert(total_block_size > (1 << 16));
|
||||
/* Update the size of the current meta-block and continue emitting commands.
|
||||
We can do this because the current size and the new size both have 5
|
||||
nibbles. */
|
||||
total_block_size += block_size;
|
||||
UpdateBits(20, (uint32_t)(total_block_size - 1), mlen_storage_ix, storage);
|
||||
goto emit_commands;
|
||||
}
|
||||
|
||||
/* Emit the remaining bytes as literals. */
|
||||
if (next_emit < ip_end) {
|
||||
const size_t insert = (size_t)(ip_end - next_emit);
|
||||
if (BROTLI_PREDICT_TRUE(insert < 6210)) {
|
||||
EmitInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
|
||||
storage_ix, storage);
|
||||
EmitLiterals(next_emit, insert, lit_depth, lit_bits, storage_ix, storage);
|
||||
} else if (ShouldUseUncompressedMode(metablock_start, next_emit, insert,
|
||||
literal_ratio)) {
|
||||
EmitUncompressedMetaBlock(metablock_start, ip_end, mlen_storage_ix - 3,
|
||||
storage_ix, storage);
|
||||
} else {
|
||||
EmitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
|
||||
storage_ix, storage);
|
||||
EmitLiterals(next_emit, insert, lit_depth, lit_bits,
|
||||
storage_ix, storage);
|
||||
}
|
||||
}
|
||||
next_emit = ip_end;
|
||||
|
||||
next_block:
|
||||
/* If we have more data, write a new meta-block header and prefix codes and
|
||||
then continue emitting commands. */
|
||||
if (input_size > 0) {
|
||||
metablock_start = input;
|
||||
block_size = BROTLI_MIN(size_t, input_size, kFirstBlockSize);
|
||||
total_block_size = block_size;
|
||||
/* Save the bit position of the MLEN field of the meta-block header, so that
|
||||
we can update it later if we decide to extend this meta-block. */
|
||||
mlen_storage_ix = *storage_ix + 3;
|
||||
BrotliStoreMetaBlockHeader(block_size, 0, storage_ix, storage);
|
||||
/* No block splits, no contexts. */
|
||||
BrotliWriteBits(13, 0, storage_ix, storage);
|
||||
literal_ratio = BuildAndStoreLiteralPrefixCode(
|
||||
m, input, block_size, lit_depth, lit_bits, storage_ix, storage);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
BuildAndStoreCommandPrefixCode(cmd_histo, cmd_depth, cmd_bits,
|
||||
storage_ix, storage);
|
||||
goto emit_commands;
|
||||
}
|
||||
|
||||
if (!is_last) {
|
||||
/* If this is not the last block, update the command and distance prefix
|
||||
codes for the next block and store the compressed forms. */
|
||||
cmd_code[0] = 0;
|
||||
*cmd_code_numbits = 0;
|
||||
BuildAndStoreCommandPrefixCode(cmd_histo, cmd_depth, cmd_bits,
|
||||
cmd_code_numbits, cmd_code);
|
||||
}
|
||||
}
|
||||
|
||||
#define FOR_TABLE_BITS_(X) X(9) X(11) X(13) X(15)
|
||||
|
||||
#define BAKE_METHOD_PARAM_(B) \
|
||||
static BROTLI_NOINLINE void BrotliCompressFragmentFastImpl ## B( \
|
||||
MemoryManager* m, const uint8_t* input, size_t input_size, \
|
||||
BROTLI_BOOL is_last, int* table, uint8_t cmd_depth[128], \
|
||||
uint16_t cmd_bits[128], size_t* cmd_code_numbits, uint8_t* cmd_code, \
|
||||
size_t* storage_ix, uint8_t* storage) { \
|
||||
BrotliCompressFragmentFastImpl(m, input, input_size, is_last, table, B, \
|
||||
cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage); \
|
||||
}
|
||||
FOR_TABLE_BITS_(BAKE_METHOD_PARAM_)
|
||||
#undef BAKE_METHOD_PARAM_
|
||||
|
||||
void BrotliCompressFragmentFast(
|
||||
MemoryManager* m, const uint8_t* input, size_t input_size,
|
||||
BROTLI_BOOL is_last, int* table, size_t table_size, uint8_t cmd_depth[128],
|
||||
uint16_t cmd_bits[128], size_t* cmd_code_numbits, uint8_t* cmd_code,
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
const size_t initial_storage_ix = *storage_ix;
|
||||
const size_t table_bits = Log2FloorNonZero(table_size);
|
||||
|
||||
if (input_size == 0) {
|
||||
assert(is_last);
|
||||
BrotliWriteBits(1, 1, storage_ix, storage); /* islast */
|
||||
BrotliWriteBits(1, 1, storage_ix, storage); /* isempty */
|
||||
*storage_ix = (*storage_ix + 7u) & ~7u;
|
||||
return;
|
||||
}
|
||||
|
||||
switch (table_bits) {
|
||||
#define CASE_(B) \
|
||||
case B: \
|
||||
BrotliCompressFragmentFastImpl ## B( \
|
||||
m, input, input_size, is_last, table, cmd_depth, cmd_bits, \
|
||||
cmd_code_numbits, cmd_code, storage_ix, storage); \
|
||||
break;
|
||||
FOR_TABLE_BITS_(CASE_)
|
||||
#undef CASE_
|
||||
default: assert(0); break;
|
||||
}
|
||||
|
||||
/* If output is larger than single uncompressed block, rewrite it. */
|
||||
if (*storage_ix - initial_storage_ix > 31 + (input_size << 3)) {
|
||||
EmitUncompressedMetaBlock(input, input + input_size, initial_storage_ix,
|
||||
storage_ix, storage);
|
||||
}
|
||||
|
||||
if (is_last) {
|
||||
BrotliWriteBits(1, 1, storage_ix, storage); /* islast */
|
||||
BrotliWriteBits(1, 1, storage_ix, storage); /* isempty */
|
||||
*storage_ix = (*storage_ix + 7u) & ~7u;
|
||||
}
|
||||
}
|
||||
|
||||
#undef FOR_TABLE_BITS_
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
|
@ -1,612 +0,0 @@
|
|||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function for fast encoding of an input fragment, independently from the input
|
||||
history. This function uses two-pass processing: in the first pass we save
|
||||
the found backward matches and literal bytes into a buffer, and in the
|
||||
second pass we emit them into the bit stream using prefix codes built based
|
||||
on the actual command and literal byte histograms. */
|
||||
|
||||
#include "./enc/compress_fragment_two_pass.h"
|
||||
|
||||
#include <string.h> /* memcmp, memcpy, memset */
|
||||
|
||||
#include "./common/constants.h"
|
||||
#include <brotli/types.h>
|
||||
#include "./enc/bit_cost.h"
|
||||
#include "./enc/brotli_bit_stream.h"
|
||||
#include "./enc/entropy_encode.h"
|
||||
#include "./enc/fast_log.h"
|
||||
#include "./enc/find_match_length.h"
|
||||
#include "./enc/memory.h"
|
||||
#include "./enc/port.h"
|
||||
#include "./enc/write_bits.h"
|
||||
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define MAX_DISTANCE (long)BROTLI_MAX_BACKWARD_LIMIT(18)
|
||||
|
||||
/* kHashMul32 multiplier has these properties:
|
||||
* The multiplier must be odd. Otherwise we may lose the highest bit.
|
||||
* No long streaks of ones or zeros.
|
||||
* There is no effort to ensure that it is a prime, the oddity is enough
|
||||
for this use.
|
||||
* The number has been tuned heuristically against compression benchmarks. */
|
||||
static const uint32_t kHashMul32 = 0x1e35a7bd;
|
||||
|
||||
static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
|
||||
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 16) * kHashMul32;
|
||||
return (uint32_t)(h >> shift);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t HashBytesAtOffset(
|
||||
uint64_t v, int offset, size_t shift) {
|
||||
assert(offset >= 0);
|
||||
assert(offset <= 2);
|
||||
{
|
||||
const uint64_t h = ((v >> (8 * offset)) << 16) * kHashMul32;
|
||||
return (uint32_t)(h >> shift);
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE BROTLI_BOOL IsMatch(const uint8_t* p1, const uint8_t* p2) {
|
||||
return TO_BROTLI_BOOL(
|
||||
BROTLI_UNALIGNED_LOAD32(p1) == BROTLI_UNALIGNED_LOAD32(p2) &&
|
||||
p1[4] == p2[4] &&
|
||||
p1[5] == p2[5]);
|
||||
}
|
||||
|
||||
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
|
||||
"bits" based on "histogram" and stores it into the bit stream. */
|
||||
static void BuildAndStoreCommandPrefixCode(
|
||||
const uint32_t histogram[128],
|
||||
uint8_t depth[128], uint16_t bits[128],
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
|
||||
HuffmanTree tree[129];
|
||||
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS] = { 0 };
|
||||
uint16_t cmd_bits[64];
|
||||
BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth);
|
||||
BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]);
|
||||
/* We have to jump through a few hoops here in order to compute
|
||||
the command bits because the symbols are in a different order than in
|
||||
the full alphabet. This looks complicated, but having the symbols
|
||||
in this order in the command bits saves a few branches in the Emit*
|
||||
functions. */
|
||||
memcpy(cmd_depth, depth + 24, 24);
|
||||
memcpy(cmd_depth + 24, depth, 8);
|
||||
memcpy(cmd_depth + 32, depth + 48, 8);
|
||||
memcpy(cmd_depth + 40, depth + 8, 8);
|
||||
memcpy(cmd_depth + 48, depth + 56, 8);
|
||||
memcpy(cmd_depth + 56, depth + 16, 8);
|
||||
BrotliConvertBitDepthsToSymbols(cmd_depth, 64, cmd_bits);
|
||||
memcpy(bits, cmd_bits + 24, 16);
|
||||
memcpy(bits + 8, cmd_bits + 40, 16);
|
||||
memcpy(bits + 16, cmd_bits + 56, 16);
|
||||
memcpy(bits + 24, cmd_bits, 48);
|
||||
memcpy(bits + 48, cmd_bits + 32, 16);
|
||||
memcpy(bits + 56, cmd_bits + 48, 16);
|
||||
BrotliConvertBitDepthsToSymbols(&depth[64], 64, &bits[64]);
|
||||
{
|
||||
/* Create the bit length array for the full command alphabet. */
|
||||
size_t i;
|
||||
memset(cmd_depth, 0, 64); /* only 64 first values were used */
|
||||
memcpy(cmd_depth, depth + 24, 8);
|
||||
memcpy(cmd_depth + 64, depth + 32, 8);
|
||||
memcpy(cmd_depth + 128, depth + 40, 8);
|
||||
memcpy(cmd_depth + 192, depth + 48, 8);
|
||||
memcpy(cmd_depth + 384, depth + 56, 8);
|
||||
for (i = 0; i < 8; ++i) {
|
||||
cmd_depth[128 + 8 * i] = depth[i];
|
||||
cmd_depth[256 + 8 * i] = depth[8 + i];
|
||||
cmd_depth[448 + 8 * i] = depth[16 + i];
|
||||
}
|
||||
BrotliStoreHuffmanTree(
|
||||
cmd_depth, BROTLI_NUM_COMMAND_SYMBOLS, tree, storage_ix, storage);
|
||||
}
|
||||
BrotliStoreHuffmanTree(&depth[64], 64, tree, storage_ix, storage);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void EmitInsertLen(
|
||||
uint32_t insertlen, uint32_t** commands) {
|
||||
if (insertlen < 6) {
|
||||
**commands = insertlen;
|
||||
} else if (insertlen < 130) {
|
||||
const uint32_t tail = insertlen - 2;
|
||||
const uint32_t nbits = Log2FloorNonZero(tail) - 1u;
|
||||
const uint32_t prefix = tail >> nbits;
|
||||
const uint32_t inscode = (nbits << 1) + prefix + 2;
|
||||
const uint32_t extra = tail - (prefix << nbits);
|
||||
**commands = inscode | (extra << 8);
|
||||
} else if (insertlen < 2114) {
|
||||
const uint32_t tail = insertlen - 66;
|
||||
const uint32_t nbits = Log2FloorNonZero(tail);
|
||||
const uint32_t code = nbits + 10;
|
||||
const uint32_t extra = tail - (1u << nbits);
|
||||
**commands = code | (extra << 8);
|
||||
} else if (insertlen < 6210) {
|
||||
const uint32_t extra = insertlen - 2114;
|
||||
**commands = 21 | (extra << 8);
|
||||
} else if (insertlen < 22594) {
|
||||
const uint32_t extra = insertlen - 6210;
|
||||
**commands = 22 | (extra << 8);
|
||||
} else {
|
||||
const uint32_t extra = insertlen - 22594;
|
||||
**commands = 23 | (extra << 8);
|
||||
}
|
||||
++(*commands);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void EmitCopyLen(size_t copylen, uint32_t** commands) {
|
||||
if (copylen < 10) {
|
||||
**commands = (uint32_t)(copylen + 38);
|
||||
} else if (copylen < 134) {
|
||||
const size_t tail = copylen - 6;
|
||||
const size_t nbits = Log2FloorNonZero(tail) - 1;
|
||||
const size_t prefix = tail >> nbits;
|
||||
const size_t code = (nbits << 1) + prefix + 44;
|
||||
const size_t extra = tail - (prefix << nbits);
|
||||
**commands = (uint32_t)(code | (extra << 8));
|
||||
} else if (copylen < 2118) {
|
||||
const size_t tail = copylen - 70;
|
||||
const size_t nbits = Log2FloorNonZero(tail);
|
||||
const size_t code = nbits + 52;
|
||||
const size_t extra = tail - ((size_t)1 << nbits);
|
||||
**commands = (uint32_t)(code | (extra << 8));
|
||||
} else {
|
||||
const size_t extra = copylen - 2118;
|
||||
**commands = (uint32_t)(63 | (extra << 8));
|
||||
}
|
||||
++(*commands);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void EmitCopyLenLastDistance(
|
||||
size_t copylen, uint32_t** commands) {
|
||||
if (copylen < 12) {
|
||||
**commands = (uint32_t)(copylen + 20);
|
||||
++(*commands);
|
||||
} else if (copylen < 72) {
|
||||
const size_t tail = copylen - 8;
|
||||
const size_t nbits = Log2FloorNonZero(tail) - 1;
|
||||
const size_t prefix = tail >> nbits;
|
||||
const size_t code = (nbits << 1) + prefix + 28;
|
||||
const size_t extra = tail - (prefix << nbits);
|
||||
**commands = (uint32_t)(code | (extra << 8));
|
||||
++(*commands);
|
||||
} else if (copylen < 136) {
|
||||
const size_t tail = copylen - 8;
|
||||
const size_t code = (tail >> 5) + 54;
|
||||
const size_t extra = tail & 31;
|
||||
**commands = (uint32_t)(code | (extra << 8));
|
||||
++(*commands);
|
||||
**commands = 64;
|
||||
++(*commands);
|
||||
} else if (copylen < 2120) {
|
||||
const size_t tail = copylen - 72;
|
||||
const size_t nbits = Log2FloorNonZero(tail);
|
||||
const size_t code = nbits + 52;
|
||||
const size_t extra = tail - ((size_t)1 << nbits);
|
||||
**commands = (uint32_t)(code | (extra << 8));
|
||||
++(*commands);
|
||||
**commands = 64;
|
||||
++(*commands);
|
||||
} else {
|
||||
const size_t extra = copylen - 2120;
|
||||
**commands = (uint32_t)(63 | (extra << 8));
|
||||
++(*commands);
|
||||
**commands = 64;
|
||||
++(*commands);
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void EmitDistance(uint32_t distance, uint32_t** commands) {
|
||||
uint32_t d = distance + 3;
|
||||
uint32_t nbits = Log2FloorNonZero(d) - 1;
|
||||
const uint32_t prefix = (d >> nbits) & 1;
|
||||
const uint32_t offset = (2 + prefix) << nbits;
|
||||
const uint32_t distcode = 2 * (nbits - 1) + prefix + 80;
|
||||
uint32_t extra = d - offset;
|
||||
**commands = distcode | (extra << 8);
|
||||
++(*commands);
|
||||
}
|
||||
|
||||
/* REQUIRES: len <= 1 << 24. */
|
||||
static void BrotliStoreMetaBlockHeader(
|
||||
size_t len, BROTLI_BOOL is_uncompressed, size_t* storage_ix,
|
||||
uint8_t* storage) {
|
||||
size_t nibbles = 6;
|
||||
/* ISLAST */
|
||||
BrotliWriteBits(1, 0, storage_ix, storage);
|
||||
if (len <= (1U << 16)) {
|
||||
nibbles = 4;
|
||||
} else if (len <= (1U << 20)) {
|
||||
nibbles = 5;
|
||||
}
|
||||
BrotliWriteBits(2, nibbles - 4, storage_ix, storage);
|
||||
BrotliWriteBits(nibbles * 4, len - 1, storage_ix, storage);
|
||||
/* ISUNCOMPRESSED */
|
||||
BrotliWriteBits(1, (uint64_t)is_uncompressed, storage_ix, storage);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void CreateCommands(const uint8_t* input,
|
||||
size_t block_size, size_t input_size, const uint8_t* base_ip, int* table,
|
||||
size_t table_bits, uint8_t** literals, uint32_t** commands) {
|
||||
/* "ip" is the input pointer. */
|
||||
const uint8_t* ip = input;
|
||||
const size_t shift = 64u - table_bits;
|
||||
const uint8_t* ip_end = input + block_size;
|
||||
/* "next_emit" is a pointer to the first byte that is not covered by a
|
||||
previous copy. Bytes between "next_emit" and the start of the next copy or
|
||||
the end of the input will be emitted as literal bytes. */
|
||||
const uint8_t* next_emit = input;
|
||||
|
||||
int last_distance = -1;
|
||||
const size_t kInputMarginBytes = BROTLI_WINDOW_GAP;
|
||||
const size_t kMinMatchLen = 6;
|
||||
|
||||
if (BROTLI_PREDICT_TRUE(block_size >= kInputMarginBytes)) {
|
||||
/* For the last block, we need to keep a 16 bytes margin so that we can be
|
||||
sure that all distances are at most window size - 16.
|
||||
For all other blocks, we only need to keep a margin of 5 bytes so that
|
||||
we don't go over the block size with a copy. */
|
||||
const size_t len_limit = BROTLI_MIN(size_t, block_size - kMinMatchLen,
|
||||
input_size - kInputMarginBytes);
|
||||
const uint8_t* ip_limit = input + len_limit;
|
||||
|
||||
uint32_t next_hash;
|
||||
for (next_hash = Hash(++ip, shift); ; ) {
|
||||
/* Step 1: Scan forward in the input looking for a 6-byte-long match.
|
||||
If we get close to exhausting the input then goto emit_remainder.
|
||||
|
||||
Heuristic match skipping: If 32 bytes are scanned with no matches
|
||||
found, start looking only at every other byte. If 32 more bytes are
|
||||
scanned, look at every third byte, etc.. When a match is found,
|
||||
immediately go back to looking at every byte. This is a small loss
|
||||
(~5% performance, ~0.1% density) for compressible data due to more
|
||||
bookkeeping, but for non-compressible data (such as JPEG) it's a huge
|
||||
win since the compressor quickly "realizes" the data is incompressible
|
||||
and doesn't bother looking for matches everywhere.
|
||||
|
||||
The "skip" variable keeps track of how many bytes there are since the
|
||||
last match; dividing it by 32 (ie. right-shifting by five) gives the
|
||||
number of bytes to move ahead for each iteration. */
|
||||
uint32_t skip = 32;
|
||||
|
||||
const uint8_t* next_ip = ip;
|
||||
const uint8_t* candidate;
|
||||
|
||||
assert(next_emit < ip);
|
||||
trawl:
|
||||
do {
|
||||
uint32_t hash = next_hash;
|
||||
uint32_t bytes_between_hash_lookups = skip++ >> 5;
|
||||
ip = next_ip;
|
||||
assert(hash == Hash(ip, shift));
|
||||
next_ip = ip + bytes_between_hash_lookups;
|
||||
if (BROTLI_PREDICT_FALSE(next_ip > ip_limit)) {
|
||||
goto emit_remainder;
|
||||
}
|
||||
next_hash = Hash(next_ip, shift);
|
||||
candidate = ip - last_distance;
|
||||
if (IsMatch(ip, candidate)) {
|
||||
if (BROTLI_PREDICT_TRUE(candidate < ip)) {
|
||||
table[hash] = (int)(ip - base_ip);
|
||||
break;
|
||||
}
|
||||
}
|
||||
candidate = base_ip + table[hash];
|
||||
assert(candidate >= base_ip);
|
||||
assert(candidate < ip);
|
||||
|
||||
table[hash] = (int)(ip - base_ip);
|
||||
} while (BROTLI_PREDICT_TRUE(!IsMatch(ip, candidate)));
|
||||
|
||||
/* Check copy distance. If candidate is not feasible, continue search.
|
||||
Checking is done outside of hot loop to reduce overhead. */
|
||||
if (ip - candidate > MAX_DISTANCE) goto trawl;
|
||||
|
||||
/* Step 2: Emit the found match together with the literal bytes from
|
||||
"next_emit", and then see if we can find a next match immediately
|
||||
afterwards. Repeat until we find no match for the input
|
||||
without emitting some literal bytes. */
|
||||
|
||||
{
|
||||
/* We have a 6-byte match at ip, and we need to emit bytes in
|
||||
[next_emit, ip). */
|
||||
const uint8_t* base = ip;
|
||||
size_t matched = 6 + FindMatchLengthWithLimit(
|
||||
candidate + 6, ip + 6, (size_t)(ip_end - ip) - 6);
|
||||
int distance = (int)(base - candidate); /* > 0 */
|
||||
int insert = (int)(base - next_emit);
|
||||
ip += matched;
|
||||
assert(0 == memcmp(base, candidate, matched));
|
||||
EmitInsertLen((uint32_t)insert, commands);
|
||||
memcpy(*literals, next_emit, (size_t)insert);
|
||||
*literals += insert;
|
||||
if (distance == last_distance) {
|
||||
**commands = 64;
|
||||
++(*commands);
|
||||
} else {
|
||||
EmitDistance((uint32_t)distance, commands);
|
||||
last_distance = distance;
|
||||
}
|
||||
EmitCopyLenLastDistance(matched, commands);
|
||||
|
||||
next_emit = ip;
|
||||
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
|
||||
goto emit_remainder;
|
||||
}
|
||||
{
|
||||
/* We could immediately start working at ip now, but to improve
|
||||
compression we first update "table" with the hashes of some
|
||||
positions within the last copy. */
|
||||
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5);
|
||||
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
|
||||
uint32_t cur_hash;
|
||||
table[prev_hash] = (int)(ip - base_ip - 5);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 1, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 4);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 3);
|
||||
input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2);
|
||||
cur_hash = HashBytesAtOffset(input_bytes, 2, shift);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 2);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 1, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 1);
|
||||
|
||||
candidate = base_ip + table[cur_hash];
|
||||
table[cur_hash] = (int)(ip - base_ip);
|
||||
}
|
||||
}
|
||||
|
||||
while (ip - candidate <= MAX_DISTANCE && IsMatch(ip, candidate)) {
|
||||
/* We have a 6-byte match at ip, and no need to emit any
|
||||
literal bytes prior to ip. */
|
||||
const uint8_t* base = ip;
|
||||
size_t matched = 6 + FindMatchLengthWithLimit(
|
||||
candidate + 6, ip + 6, (size_t)(ip_end - ip) - 6);
|
||||
ip += matched;
|
||||
last_distance = (int)(base - candidate); /* > 0 */
|
||||
assert(0 == memcmp(base, candidate, matched));
|
||||
EmitCopyLen(matched, commands);
|
||||
EmitDistance((uint32_t)last_distance, commands);
|
||||
|
||||
next_emit = ip;
|
||||
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
|
||||
goto emit_remainder;
|
||||
}
|
||||
{
|
||||
/* We could immediately start working at ip now, but to improve
|
||||
compression we first update "table" with the hashes of some
|
||||
positions within the last copy. */
|
||||
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5);
|
||||
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
|
||||
uint32_t cur_hash;
|
||||
table[prev_hash] = (int)(ip - base_ip - 5);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 1, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 4);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 3);
|
||||
input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2);
|
||||
cur_hash = HashBytesAtOffset(input_bytes, 2, shift);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 2);
|
||||
prev_hash = HashBytesAtOffset(input_bytes, 1, shift);
|
||||
table[prev_hash] = (int)(ip - base_ip - 1);
|
||||
|
||||
candidate = base_ip + table[cur_hash];
|
||||
table[cur_hash] = (int)(ip - base_ip);
|
||||
}
|
||||
}
|
||||
|
||||
next_hash = Hash(++ip, shift);
|
||||
}
|
||||
}
|
||||
|
||||
emit_remainder:
|
||||
assert(next_emit <= ip_end);
|
||||
/* Emit the remaining bytes as literals. */
|
||||
if (next_emit < ip_end) {
|
||||
const uint32_t insert = (uint32_t)(ip_end - next_emit);
|
||||
EmitInsertLen(insert, commands);
|
||||
memcpy(*literals, next_emit, insert);
|
||||
*literals += insert;
|
||||
}
|
||||
}
|
||||
|
||||
static void StoreCommands(MemoryManager* m,
|
||||
const uint8_t* literals, const size_t num_literals,
|
||||
const uint32_t* commands, const size_t num_commands,
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
static const uint32_t kNumExtraBits[128] = {
|
||||
0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
|
||||
9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16,
|
||||
17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 24,
|
||||
};
|
||||
static const uint32_t kInsertOffset[24] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 98, 130, 194, 322, 578,
|
||||
1090, 2114, 6210, 22594,
|
||||
};
|
||||
|
||||
uint8_t lit_depths[256];
|
||||
uint16_t lit_bits[256];
|
||||
uint32_t lit_histo[256] = { 0 };
|
||||
uint8_t cmd_depths[128] = { 0 };
|
||||
uint16_t cmd_bits[128] = { 0 };
|
||||
uint32_t cmd_histo[128] = { 0 };
|
||||
size_t i;
|
||||
for (i = 0; i < num_literals; ++i) {
|
||||
++lit_histo[literals[i]];
|
||||
}
|
||||
BrotliBuildAndStoreHuffmanTreeFast(m, lit_histo, num_literals,
|
||||
/* max_bits = */ 8,
|
||||
lit_depths, lit_bits,
|
||||
storage_ix, storage);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
|
||||
for (i = 0; i < num_commands; ++i) {
|
||||
const uint32_t code = commands[i] & 0xFF;
|
||||
assert(code < 128);
|
||||
++cmd_histo[code];
|
||||
}
|
||||
cmd_histo[1] += 1;
|
||||
cmd_histo[2] += 1;
|
||||
cmd_histo[64] += 1;
|
||||
cmd_histo[84] += 1;
|
||||
BuildAndStoreCommandPrefixCode(cmd_histo, cmd_depths, cmd_bits,
|
||||
storage_ix, storage);
|
||||
|
||||
for (i = 0; i < num_commands; ++i) {
|
||||
const uint32_t cmd = commands[i];
|
||||
const uint32_t code = cmd & 0xFF;
|
||||
const uint32_t extra = cmd >> 8;
|
||||
assert(code < 128);
|
||||
BrotliWriteBits(cmd_depths[code], cmd_bits[code], storage_ix, storage);
|
||||
BrotliWriteBits(kNumExtraBits[code], extra, storage_ix, storage);
|
||||
if (code < 24) {
|
||||
const uint32_t insert = kInsertOffset[code] + extra;
|
||||
uint32_t j;
|
||||
for (j = 0; j < insert; ++j) {
|
||||
const uint8_t lit = *literals;
|
||||
BrotliWriteBits(lit_depths[lit], lit_bits[lit], storage_ix, storage);
|
||||
++literals;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Acceptable loss for uncompressible speedup is 2% */
|
||||
#define MIN_RATIO 0.98
|
||||
#define SAMPLE_RATE 43
|
||||
|
||||
static BROTLI_BOOL ShouldCompress(
|
||||
const uint8_t* input, size_t input_size, size_t num_literals) {
|
||||
double corpus_size = (double)input_size;
|
||||
if (num_literals < MIN_RATIO * corpus_size) {
|
||||
return BROTLI_TRUE;
|
||||
} else {
|
||||
uint32_t literal_histo[256] = { 0 };
|
||||
const double max_total_bit_cost = corpus_size * 8 * MIN_RATIO / SAMPLE_RATE;
|
||||
size_t i;
|
||||
for (i = 0; i < input_size; i += SAMPLE_RATE) {
|
||||
++literal_histo[input[i]];
|
||||
}
|
||||
return TO_BROTLI_BOOL(BitsEntropy(literal_histo, 256) < max_total_bit_cost);
|
||||
}
|
||||
}
|
||||
|
||||
static void RewindBitPosition(const size_t new_storage_ix,
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
const size_t bitpos = new_storage_ix & 7;
|
||||
const size_t mask = (1u << bitpos) - 1;
|
||||
storage[new_storage_ix >> 3] &= (uint8_t)mask;
|
||||
*storage_ix = new_storage_ix;
|
||||
}
|
||||
|
||||
static void EmitUncompressedMetaBlock(const uint8_t* input, size_t input_size,
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
BrotliStoreMetaBlockHeader(input_size, 1, storage_ix, storage);
|
||||
*storage_ix = (*storage_ix + 7u) & ~7u;
|
||||
memcpy(&storage[*storage_ix >> 3], input, input_size);
|
||||
*storage_ix += input_size << 3;
|
||||
storage[*storage_ix >> 3] = 0;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void BrotliCompressFragmentTwoPassImpl(
|
||||
MemoryManager* m, const uint8_t* input, size_t input_size,
|
||||
BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf,
|
||||
int* table, size_t table_bits, size_t* storage_ix, uint8_t* storage) {
|
||||
/* Save the start of the first block for position and distance computations.
|
||||
*/
|
||||
const uint8_t* base_ip = input;
|
||||
BROTLI_UNUSED(is_last);
|
||||
|
||||
while (input_size > 0) {
|
||||
size_t block_size =
|
||||
BROTLI_MIN(size_t, input_size, kCompressFragmentTwoPassBlockSize);
|
||||
uint32_t* commands = command_buf;
|
||||
uint8_t* literals = literal_buf;
|
||||
size_t num_literals;
|
||||
CreateCommands(input, block_size, input_size, base_ip, table, table_bits,
|
||||
&literals, &commands);
|
||||
num_literals = (size_t)(literals - literal_buf);
|
||||
if (ShouldCompress(input, block_size, num_literals)) {
|
||||
const size_t num_commands = (size_t)(commands - command_buf);
|
||||
BrotliStoreMetaBlockHeader(block_size, 0, storage_ix, storage);
|
||||
/* No block splits, no contexts. */
|
||||
BrotliWriteBits(13, 0, storage_ix, storage);
|
||||
StoreCommands(m, literal_buf, num_literals, command_buf, num_commands,
|
||||
storage_ix, storage);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
} else {
|
||||
/* Since we did not find many backward references and the entropy of
|
||||
the data is close to 8 bits, we can simply emit an uncompressed block.
|
||||
This makes compression speed of uncompressible data about 3x faster. */
|
||||
EmitUncompressedMetaBlock(input, block_size, storage_ix, storage);
|
||||
}
|
||||
input += block_size;
|
||||
input_size -= block_size;
|
||||
}
|
||||
}
|
||||
|
||||
#define FOR_TABLE_BITS_(X) \
|
||||
X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17)
|
||||
|
||||
#define BAKE_METHOD_PARAM_(B) \
|
||||
static BROTLI_NOINLINE void BrotliCompressFragmentTwoPassImpl ## B( \
|
||||
MemoryManager* m, const uint8_t* input, size_t input_size, \
|
||||
BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf, \
|
||||
int* table, size_t* storage_ix, uint8_t* storage) { \
|
||||
BrotliCompressFragmentTwoPassImpl(m, input, input_size, is_last, command_buf,\
|
||||
literal_buf, table, B, storage_ix, storage); \
|
||||
}
|
||||
FOR_TABLE_BITS_(BAKE_METHOD_PARAM_)
|
||||
#undef BAKE_METHOD_PARAM_
|
||||
|
||||
void BrotliCompressFragmentTwoPass(
|
||||
MemoryManager* m, const uint8_t* input, size_t input_size,
|
||||
BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf,
|
||||
int* table, size_t table_size, size_t* storage_ix, uint8_t* storage) {
|
||||
const size_t initial_storage_ix = *storage_ix;
|
||||
const size_t table_bits = Log2FloorNonZero(table_size);
|
||||
switch (table_bits) {
|
||||
#define CASE_(B) \
|
||||
case B: \
|
||||
BrotliCompressFragmentTwoPassImpl ## B( \
|
||||
m, input, input_size, is_last, command_buf, \
|
||||
literal_buf, table, storage_ix, storage); \
|
||||
break;
|
||||
FOR_TABLE_BITS_(CASE_)
|
||||
#undef CASE_
|
||||
default: assert(0); break;
|
||||
}
|
||||
|
||||
/* If output is larger than single uncompressed block, rewrite it. */
|
||||
if (*storage_ix - initial_storage_ix > 31 + (input_size << 3)) {
|
||||
RewindBitPosition(initial_storage_ix, storage_ix, storage);
|
||||
EmitUncompressedMetaBlock(input, input_size, storage_ix, storage);
|
||||
}
|
||||
|
||||
if (is_last) {
|
||||
BrotliWriteBits(1, 1, storage_ix, storage); /* islast */
|
||||
BrotliWriteBits(1, 1, storage_ix, storage); /* isempty */
|
||||
*storage_ix = (*storage_ix + 7u) & ~7u;
|
||||
}
|
||||
}
|
||||
|
||||
#undef FOR_TABLE_BITS_
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
|
@ -1,360 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Bit reading helpers */
|
||||
|
||||
#ifndef BROTLI_DEC_BIT_READER_H_
|
||||
#define BROTLI_DEC_BIT_READER_H_
|
||||
|
||||
#include <string.h> /* memcpy */
|
||||
|
||||
#include <brotli/types.h>
|
||||
#include "./port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define BROTLI_SHORT_FILL_BIT_WINDOW_READ (sizeof(reg_t) >> 1)
|
||||
|
||||
static const uint32_t kBitMask[33] = { 0x0000,
|
||||
0x00000001, 0x00000003, 0x00000007, 0x0000000F,
|
||||
0x0000001F, 0x0000003F, 0x0000007F, 0x000000FF,
|
||||
0x000001FF, 0x000003FF, 0x000007FF, 0x00000FFF,
|
||||
0x00001FFF, 0x00003FFF, 0x00007FFF, 0x0000FFFF,
|
||||
0x0001FFFF, 0x0003FFFF, 0x0007FFFF, 0x000FFFFF,
|
||||
0x001FFFFF, 0x003FFFFF, 0x007FFFFF, 0x00FFFFFF,
|
||||
0x01FFFFFF, 0x03FFFFFF, 0x07FFFFFF, 0x0FFFFFFF,
|
||||
0x1FFFFFFF, 0x3FFFFFFF, 0x7FFFFFFF, 0xFFFFFFFF
|
||||
};
|
||||
|
||||
static BROTLI_INLINE uint32_t BitMask(uint32_t n) {
|
||||
if (IS_CONSTANT(n) || BROTLI_HAS_UBFX) {
|
||||
/* Masking with this expression turns to a single
|
||||
"Unsigned Bit Field Extract" UBFX instruction on ARM. */
|
||||
return ~((0xffffffffU) << n);
|
||||
} else {
|
||||
return kBitMask[n];
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
reg_t val_; /* pre-fetched bits */
|
||||
uint32_t bit_pos_; /* current bit-reading position in val_ */
|
||||
const uint8_t* next_in; /* the byte we're reading from */
|
||||
size_t avail_in;
|
||||
} BrotliBitReader;
|
||||
|
||||
typedef struct {
|
||||
reg_t val_;
|
||||
uint32_t bit_pos_;
|
||||
const uint8_t* next_in;
|
||||
size_t avail_in;
|
||||
} BrotliBitReaderState;
|
||||
|
||||
/* Initializes the BrotliBitReader fields. */
|
||||
BROTLI_INTERNAL void BrotliInitBitReader(BrotliBitReader* const br);
|
||||
|
||||
/* Ensures that accumulator is not empty. May consume one byte of input.
|
||||
Returns 0 if data is required but there is no input available.
|
||||
For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned
|
||||
reading. */
|
||||
BROTLI_INTERNAL BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* const br);
|
||||
|
||||
static BROTLI_INLINE void BrotliBitReaderSaveState(
|
||||
BrotliBitReader* const from, BrotliBitReaderState* to) {
|
||||
to->val_ = from->val_;
|
||||
to->bit_pos_ = from->bit_pos_;
|
||||
to->next_in = from->next_in;
|
||||
to->avail_in = from->avail_in;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void BrotliBitReaderRestoreState(
|
||||
BrotliBitReader* const to, BrotliBitReaderState* from) {
|
||||
to->val_ = from->val_;
|
||||
to->bit_pos_ = from->bit_pos_;
|
||||
to->next_in = from->next_in;
|
||||
to->avail_in = from->avail_in;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t BrotliGetAvailableBits(
|
||||
const BrotliBitReader* br) {
|
||||
return (BROTLI_64_BITS ? 64 : 32) - br->bit_pos_;
|
||||
}
|
||||
|
||||
/* Returns amount of unread bytes the bit reader still has buffered from the
|
||||
BrotliInput, including whole bytes in br->val_. */
|
||||
static BROTLI_INLINE size_t BrotliGetRemainingBytes(BrotliBitReader* br) {
|
||||
return br->avail_in + (BrotliGetAvailableBits(br) >> 3);
|
||||
}
|
||||
|
||||
/* Checks if there is at least |num| bytes left in the input ring-buffer
|
||||
(excluding the bits remaining in br->val_). */
|
||||
static BROTLI_INLINE BROTLI_BOOL BrotliCheckInputAmount(
|
||||
BrotliBitReader* const br, size_t num) {
|
||||
return TO_BROTLI_BOOL(br->avail_in >= num);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint16_t BrotliLoad16LE(const uint8_t* in) {
|
||||
if (BROTLI_LITTLE_ENDIAN) {
|
||||
return *((const uint16_t*)in);
|
||||
} else if (BROTLI_BIG_ENDIAN) {
|
||||
uint16_t value = *((const uint16_t*)in);
|
||||
return (uint16_t)(((value & 0xFFU) << 8) | ((value & 0xFF00U) >> 8));
|
||||
} else {
|
||||
return (uint16_t)(in[0] | (in[1] << 8));
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t BrotliLoad32LE(const uint8_t* in) {
|
||||
if (BROTLI_LITTLE_ENDIAN) {
|
||||
return *((const uint32_t*)in);
|
||||
} else if (BROTLI_BIG_ENDIAN) {
|
||||
uint32_t value = *((const uint32_t*)in);
|
||||
return ((value & 0xFFU) << 24) | ((value & 0xFF00U) << 8) |
|
||||
((value & 0xFF0000U) >> 8) | ((value & 0xFF000000U) >> 24);
|
||||
} else {
|
||||
uint32_t value = (uint32_t)(*(in++));
|
||||
value |= (uint32_t)(*(in++)) << 8;
|
||||
value |= (uint32_t)(*(in++)) << 16;
|
||||
value |= (uint32_t)(*(in++)) << 24;
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
#if (BROTLI_64_BITS)
|
||||
static BROTLI_INLINE uint64_t BrotliLoad64LE(const uint8_t* in) {
|
||||
if (BROTLI_LITTLE_ENDIAN) {
|
||||
return *((const uint64_t*)in);
|
||||
} else if (BROTLI_BIG_ENDIAN) {
|
||||
uint64_t value = *((const uint64_t*)in);
|
||||
return
|
||||
((value & 0xFFU) << 56) |
|
||||
((value & 0xFF00U) << 40) |
|
||||
((value & 0xFF0000U) << 24) |
|
||||
((value & 0xFF000000U) << 8) |
|
||||
((value & 0xFF00000000U) >> 8) |
|
||||
((value & 0xFF0000000000U) >> 24) |
|
||||
((value & 0xFF000000000000U) >> 40) |
|
||||
((value & 0xFF00000000000000U) >> 56);
|
||||
} else {
|
||||
uint64_t value = (uint64_t)(*(in++));
|
||||
value |= (uint64_t)(*(in++)) << 8;
|
||||
value |= (uint64_t)(*(in++)) << 16;
|
||||
value |= (uint64_t)(*(in++)) << 24;
|
||||
value |= (uint64_t)(*(in++)) << 32;
|
||||
value |= (uint64_t)(*(in++)) << 40;
|
||||
value |= (uint64_t)(*(in++)) << 48;
|
||||
value |= (uint64_t)(*(in++)) << 56;
|
||||
return value;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Guarantees that there are at least n_bits + 1 bits in accumulator.
|
||||
Precondition: accumulator contains at least 1 bit.
|
||||
n_bits should be in the range [1..24] for regular build. For portable
|
||||
non-64-bit little-endian build only 16 bits are safe to request. */
|
||||
static BROTLI_INLINE void BrotliFillBitWindow(
|
||||
BrotliBitReader* const br, uint32_t n_bits) {
|
||||
#if (BROTLI_64_BITS)
|
||||
if (!BROTLI_ALIGNED_READ && IS_CONSTANT(n_bits) && (n_bits <= 8)) {
|
||||
if (br->bit_pos_ >= 56) {
|
||||
br->val_ >>= 56;
|
||||
br->bit_pos_ ^= 56; /* here same as -= 56 because of the if condition */
|
||||
br->val_ |= BrotliLoad64LE(br->next_in) << 8;
|
||||
br->avail_in -= 7;
|
||||
br->next_in += 7;
|
||||
}
|
||||
} else if (!BROTLI_ALIGNED_READ && IS_CONSTANT(n_bits) && (n_bits <= 16)) {
|
||||
if (br->bit_pos_ >= 48) {
|
||||
br->val_ >>= 48;
|
||||
br->bit_pos_ ^= 48; /* here same as -= 48 because of the if condition */
|
||||
br->val_ |= BrotliLoad64LE(br->next_in) << 16;
|
||||
br->avail_in -= 6;
|
||||
br->next_in += 6;
|
||||
}
|
||||
} else {
|
||||
if (br->bit_pos_ >= 32) {
|
||||
br->val_ >>= 32;
|
||||
br->bit_pos_ ^= 32; /* here same as -= 32 because of the if condition */
|
||||
br->val_ |= ((uint64_t)BrotliLoad32LE(br->next_in)) << 32;
|
||||
br->avail_in -= BROTLI_SHORT_FILL_BIT_WINDOW_READ;
|
||||
br->next_in += BROTLI_SHORT_FILL_BIT_WINDOW_READ;
|
||||
}
|
||||
}
|
||||
#else
|
||||
if (!BROTLI_ALIGNED_READ && IS_CONSTANT(n_bits) && (n_bits <= 8)) {
|
||||
if (br->bit_pos_ >= 24) {
|
||||
br->val_ >>= 24;
|
||||
br->bit_pos_ ^= 24; /* here same as -= 24 because of the if condition */
|
||||
br->val_ |= BrotliLoad32LE(br->next_in) << 8;
|
||||
br->avail_in -= 3;
|
||||
br->next_in += 3;
|
||||
}
|
||||
} else {
|
||||
if (br->bit_pos_ >= 16) {
|
||||
br->val_ >>= 16;
|
||||
br->bit_pos_ ^= 16; /* here same as -= 16 because of the if condition */
|
||||
br->val_ |= ((uint32_t)BrotliLoad16LE(br->next_in)) << 16;
|
||||
br->avail_in -= BROTLI_SHORT_FILL_BIT_WINDOW_READ;
|
||||
br->next_in += BROTLI_SHORT_FILL_BIT_WINDOW_READ;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
|
||||
more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
|
||||
static BROTLI_INLINE void BrotliFillBitWindow16(BrotliBitReader* const br) {
|
||||
BrotliFillBitWindow(br, 17);
|
||||
}
|
||||
|
||||
/* Pulls one byte of input to accumulator. */
|
||||
static BROTLI_INLINE BROTLI_BOOL BrotliPullByte(BrotliBitReader* const br) {
|
||||
if (br->avail_in == 0) {
|
||||
return BROTLI_FALSE;
|
||||
}
|
||||
br->val_ >>= 8;
|
||||
#if (BROTLI_64_BITS)
|
||||
br->val_ |= ((uint64_t)*br->next_in) << 56;
|
||||
#else
|
||||
br->val_ |= ((uint32_t)*br->next_in) << 24;
|
||||
#endif
|
||||
br->bit_pos_ -= 8;
|
||||
--br->avail_in;
|
||||
++br->next_in;
|
||||
return BROTLI_TRUE;
|
||||
}
|
||||
|
||||
/* Returns currently available bits.
|
||||
The number of valid bits could be calculated by BrotliGetAvailableBits. */
|
||||
static BROTLI_INLINE reg_t BrotliGetBitsUnmasked(BrotliBitReader* const br) {
|
||||
return br->val_ >> br->bit_pos_;
|
||||
}
|
||||
|
||||
/* Like BrotliGetBits, but does not mask the result.
|
||||
The result contains at least 16 valid bits. */
|
||||
static BROTLI_INLINE uint32_t BrotliGet16BitsUnmasked(
|
||||
BrotliBitReader* const br) {
|
||||
BrotliFillBitWindow(br, 16);
|
||||
return (uint32_t)BrotliGetBitsUnmasked(br);
|
||||
}
|
||||
|
||||
/* Returns the specified number of bits from |br| without advancing bit pos. */
|
||||
static BROTLI_INLINE uint32_t BrotliGetBits(
|
||||
BrotliBitReader* const br, uint32_t n_bits) {
|
||||
BrotliFillBitWindow(br, n_bits);
|
||||
return (uint32_t)BrotliGetBitsUnmasked(br) & BitMask(n_bits);
|
||||
}
|
||||
|
||||
/* Tries to peek the specified amount of bits. Returns 0, if there is not
|
||||
enough input. */
|
||||
static BROTLI_INLINE BROTLI_BOOL BrotliSafeGetBits(
|
||||
BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) {
|
||||
while (BrotliGetAvailableBits(br) < n_bits) {
|
||||
if (!BrotliPullByte(br)) {
|
||||
return BROTLI_FALSE;
|
||||
}
|
||||
}
|
||||
*val = (uint32_t)BrotliGetBitsUnmasked(br) & BitMask(n_bits);
|
||||
return BROTLI_TRUE;
|
||||
}
|
||||
|
||||
/* Advances the bit pos by n_bits. */
|
||||
static BROTLI_INLINE void BrotliDropBits(
|
||||
BrotliBitReader* const br, uint32_t n_bits) {
|
||||
br->bit_pos_ += n_bits;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void BrotliBitReaderUnload(BrotliBitReader* br) {
|
||||
uint32_t unused_bytes = BrotliGetAvailableBits(br) >> 3;
|
||||
uint32_t unused_bits = unused_bytes << 3;
|
||||
br->avail_in += unused_bytes;
|
||||
br->next_in -= unused_bytes;
|
||||
if (unused_bits == sizeof(br->val_) << 3) {
|
||||
br->val_ = 0;
|
||||
} else {
|
||||
br->val_ <<= unused_bits;
|
||||
}
|
||||
br->bit_pos_ += unused_bits;
|
||||
}
|
||||
|
||||
/* Reads the specified number of bits from |br| and advances the bit pos.
|
||||
Precondition: accumulator MUST contain at least n_bits. */
|
||||
static BROTLI_INLINE void BrotliTakeBits(
|
||||
BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) {
|
||||
*val = (uint32_t)BrotliGetBitsUnmasked(br) & BitMask(n_bits);
|
||||
BROTLI_LOG(("[BrotliReadBits] %d %d %d val: %6x\n",
|
||||
(int)br->avail_in, (int)br->bit_pos_, n_bits, (int)*val));
|
||||
BrotliDropBits(br, n_bits);
|
||||
}
|
||||
|
||||
/* Reads the specified number of bits from |br| and advances the bit pos.
|
||||
Assumes that there is enough input to perform BrotliFillBitWindow. */
|
||||
static BROTLI_INLINE uint32_t BrotliReadBits(
|
||||
BrotliBitReader* const br, uint32_t n_bits) {
|
||||
if (BROTLI_64_BITS || (n_bits <= 16)) {
|
||||
uint32_t val;
|
||||
BrotliFillBitWindow(br, n_bits);
|
||||
BrotliTakeBits(br, n_bits, &val);
|
||||
return val;
|
||||
} else {
|
||||
uint32_t low_val;
|
||||
uint32_t high_val;
|
||||
BrotliFillBitWindow(br, 16);
|
||||
BrotliTakeBits(br, 16, &low_val);
|
||||
BrotliFillBitWindow(br, 8);
|
||||
BrotliTakeBits(br, n_bits - 16, &high_val);
|
||||
return low_val | (high_val << 16);
|
||||
}
|
||||
}
|
||||
|
||||
/* Tries to read the specified amount of bits. Returns 0, if there is not
|
||||
enough input. n_bits MUST be positive. */
|
||||
static BROTLI_INLINE BROTLI_BOOL BrotliSafeReadBits(
|
||||
BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) {
|
||||
while (BrotliGetAvailableBits(br) < n_bits) {
|
||||
if (!BrotliPullByte(br)) {
|
||||
return BROTLI_FALSE;
|
||||
}
|
||||
}
|
||||
BrotliTakeBits(br, n_bits, val);
|
||||
return BROTLI_TRUE;
|
||||
}
|
||||
|
||||
/* Advances the bit reader position to the next byte boundary and verifies
|
||||
that any skipped bits are set to zero. */
|
||||
static BROTLI_INLINE BROTLI_BOOL BrotliJumpToByteBoundary(BrotliBitReader* br) {
|
||||
uint32_t pad_bits_count = BrotliGetAvailableBits(br) & 0x7;
|
||||
uint32_t pad_bits = 0;
|
||||
if (pad_bits_count != 0) {
|
||||
BrotliTakeBits(br, pad_bits_count, &pad_bits);
|
||||
}
|
||||
return TO_BROTLI_BOOL(pad_bits == 0);
|
||||
}
|
||||
|
||||
/* Copies remaining input bytes stored in the bit reader to the output. Value
|
||||
num may not be larger than BrotliGetRemainingBytes. The bit reader must be
|
||||
warmed up again after this. */
|
||||
static BROTLI_INLINE void BrotliCopyBytes(uint8_t* dest,
|
||||
BrotliBitReader* br, size_t num) {
|
||||
while (BrotliGetAvailableBits(br) >= 8 && num > 0) {
|
||||
*dest = (uint8_t)BrotliGetBitsUnmasked(br);
|
||||
BrotliDropBits(br, 8);
|
||||
++dest;
|
||||
--num;
|
||||
}
|
||||
memcpy(dest, br->next_in, num);
|
||||
br->avail_in -= num;
|
||||
br->next_in += num;
|
||||
}
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_DEC_BIT_READER_H_ */
|
||||
|
|
@ -1,251 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Lookup table to map the previous two bytes to a context id.
|
||||
|
||||
There are four different context modeling modes defined here:
|
||||
CONTEXT_LSB6: context id is the least significant 6 bits of the last byte,
|
||||
CONTEXT_MSB6: context id is the most significant 6 bits of the last byte,
|
||||
CONTEXT_UTF8: second-order context model tuned for UTF8-encoded text,
|
||||
CONTEXT_SIGNED: second-order context model tuned for signed integers.
|
||||
|
||||
The context id for the UTF8 context model is calculated as follows. If p1
|
||||
and p2 are the previous two bytes, we calculate the context as
|
||||
|
||||
context = kContextLookup[p1] | kContextLookup[p2 + 256].
|
||||
|
||||
If the previous two bytes are ASCII characters (i.e. < 128), this will be
|
||||
equivalent to
|
||||
|
||||
context = 4 * context1(p1) + context2(p2),
|
||||
|
||||
where context1 is based on the previous byte in the following way:
|
||||
|
||||
0 : non-ASCII control
|
||||
1 : \t, \n, \r
|
||||
2 : space
|
||||
3 : other punctuation
|
||||
4 : " '
|
||||
5 : %
|
||||
6 : ( < [ {
|
||||
7 : ) > ] }
|
||||
8 : , ; :
|
||||
9 : .
|
||||
10 : =
|
||||
11 : number
|
||||
12 : upper-case vowel
|
||||
13 : upper-case consonant
|
||||
14 : lower-case vowel
|
||||
15 : lower-case consonant
|
||||
|
||||
and context2 is based on the second last byte:
|
||||
|
||||
0 : control, space
|
||||
1 : punctuation
|
||||
2 : upper-case letter, number
|
||||
3 : lower-case letter
|
||||
|
||||
If the last byte is ASCII, and the second last byte is not (in a valid UTF8
|
||||
stream it will be a continuation byte, value between 128 and 191), the
|
||||
context is the same as if the second last byte was an ASCII control or space.
|
||||
|
||||
If the last byte is a UTF8 lead byte (value >= 192), then the next byte will
|
||||
be a continuation byte and the context id is 2 or 3 depending on the LSB of
|
||||
the last byte and to a lesser extent on the second last byte if it is ASCII.
|
||||
|
||||
If the last byte is a UTF8 continuation byte, the second last byte can be:
|
||||
- continuation byte: the next byte is probably ASCII or lead byte (assuming
|
||||
4-byte UTF8 characters are rare) and the context id is 0 or 1.
|
||||
- lead byte (192 - 207): next byte is ASCII or lead byte, context is 0 or 1
|
||||
- lead byte (208 - 255): next byte is continuation byte, context is 2 or 3
|
||||
|
||||
The possible value combinations of the previous two bytes, the range of
|
||||
context ids and the type of the next byte is summarized in the table below:
|
||||
|
||||
|--------\-----------------------------------------------------------------|
|
||||
| \ Last byte |
|
||||
| Second \---------------------------------------------------------------|
|
||||
| last byte \ ASCII | cont. byte | lead byte |
|
||||
| \ (0-127) | (128-191) | (192-) |
|
||||
|=============|===================|=====================|==================|
|
||||
| ASCII | next: ASCII/lead | not valid | next: cont. |
|
||||
| (0-127) | context: 4 - 63 | | context: 2 - 3 |
|
||||
|-------------|-------------------|---------------------|------------------|
|
||||
| cont. byte | next: ASCII/lead | next: ASCII/lead | next: cont. |
|
||||
| (128-191) | context: 4 - 63 | context: 0 - 1 | context: 2 - 3 |
|
||||
|-------------|-------------------|---------------------|------------------|
|
||||
| lead byte | not valid | next: ASCII/lead | not valid |
|
||||
| (192-207) | | context: 0 - 1 | |
|
||||
|-------------|-------------------|---------------------|------------------|
|
||||
| lead byte | not valid | next: cont. | not valid |
|
||||
| (208-) | | context: 2 - 3 | |
|
||||
|-------------|-------------------|---------------------|------------------|
|
||||
|
||||
The context id for the signed context mode is calculated as:
|
||||
|
||||
context = (kContextLookup[512 + p1] << 3) | kContextLookup[512 + p2].
|
||||
|
||||
For any context modeling modes, the context ids can be calculated by |-ing
|
||||
together two lookups from one table using context model dependent offsets:
|
||||
|
||||
context = kContextLookup[offset1 + p1] | kContextLookup[offset2 + p2].
|
||||
|
||||
where offset1 and offset2 are dependent on the context mode.
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_DEC_CONTEXT_H_
|
||||
#define BROTLI_DEC_CONTEXT_H_
|
||||
|
||||
#include <brotli/types.h>
|
||||
|
||||
enum ContextType {
|
||||
CONTEXT_LSB6 = 0,
|
||||
CONTEXT_MSB6 = 1,
|
||||
CONTEXT_UTF8 = 2,
|
||||
CONTEXT_SIGNED = 3
|
||||
};
|
||||
|
||||
/* Common context lookup table for all context modes. */
|
||||
static const uint8_t kContextLookup[1792] = {
|
||||
/* CONTEXT_UTF8, last byte. */
|
||||
/* ASCII range. */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12,
|
||||
44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12,
|
||||
12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48,
|
||||
52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12,
|
||||
12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56,
|
||||
60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0,
|
||||
/* UTF8 continuation byte range. */
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
/* UTF8 lead byte range. */
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
/* CONTEXT_UTF8 second last byte. */
|
||||
/* ASCII range. */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1,
|
||||
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
|
||||
1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0,
|
||||
/* UTF8 continuation byte range. */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
/* UTF8 lead byte range. */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
/* CONTEXT_SIGNED, second last byte. */
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7,
|
||||
/* CONTEXT_SIGNED, last byte, same as the above values shifted by 3 bits. */
|
||||
0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
|
||||
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
|
||||
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
||||
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
||||
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
||||
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 56,
|
||||
/* CONTEXT_LSB6, last byte. */
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
|
||||
/* CONTEXT_MSB6, last byte. */
|
||||
0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
|
||||
4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
|
||||
8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,
|
||||
12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15,
|
||||
16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27,
|
||||
28, 28, 28, 28, 29, 29, 29, 29, 30, 30, 30, 30, 31, 31, 31, 31,
|
||||
32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35,
|
||||
36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 39, 39, 39,
|
||||
40, 40, 40, 40, 41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43,
|
||||
44, 44, 44, 44, 45, 45, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47,
|
||||
48, 48, 48, 48, 49, 49, 49, 49, 50, 50, 50, 50, 51, 51, 51, 51,
|
||||
52, 52, 52, 52, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 55, 55,
|
||||
56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58, 59, 59, 59, 59,
|
||||
60, 60, 60, 60, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63,
|
||||
/* CONTEXT_{M,L}SB6, second last byte, */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
};
|
||||
|
||||
static const int kContextLookupOffsets[8] = {
|
||||
/* CONTEXT_LSB6 */
|
||||
1024, 1536,
|
||||
/* CONTEXT_MSB6 */
|
||||
1280, 1536,
|
||||
/* CONTEXT_UTF8 */
|
||||
0, 256,
|
||||
/* CONTEXT_SIGNED */
|
||||
768, 512,
|
||||
};
|
||||
|
||||
#endif /* BROTLI_DEC_CONTEXT_H_ */
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
package dec
|
||||
|
||||
import (
|
||||
"C"
|
||||
)
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Utilities for building Huffman decoding tables. */
|
||||
|
||||
#ifndef BROTLI_DEC_HUFFMAN_H_
|
||||
#define BROTLI_DEC_HUFFMAN_H_
|
||||
|
||||
#include <brotli/types.h>
|
||||
#include "./port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define BROTLI_HUFFMAN_MAX_CODE_LENGTH 15
|
||||
|
||||
/* Maximum possible Huffman table size for an alphabet size of (index * 32),
|
||||
* max code length 15 and root table bits 8. */
|
||||
static const uint16_t kMaxHuffmanTableSize[] = {
|
||||
256, 402, 436, 468, 500, 534, 566, 598, 630, 662, 694, 726, 758, 790, 822,
|
||||
854, 886, 920, 952, 984, 1016, 1048, 1080};
|
||||
/* BROTLI_NUM_BLOCK_LEN_SYMBOLS == 26 */
|
||||
#define BROTLI_HUFFMAN_MAX_SIZE_26 396
|
||||
/* BROTLI_MAX_BLOCK_TYPE_SYMBOLS == 258 */
|
||||
#define BROTLI_HUFFMAN_MAX_SIZE_258 632
|
||||
/* BROTLI_MAX_CONTEXT_MAP_SYMBOLS == 272 */
|
||||
#define BROTLI_HUFFMAN_MAX_SIZE_272 646
|
||||
|
||||
#define BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH 5
|
||||
|
||||
typedef struct {
|
||||
uint8_t bits; /* number of bits used for this symbol */
|
||||
uint16_t value; /* symbol value or table offset */
|
||||
} HuffmanCode;
|
||||
|
||||
/* Builds Huffman lookup table assuming code lengths are in symbol order. */
|
||||
BROTLI_INTERNAL void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* root_table,
|
||||
const uint8_t* const code_lengths, uint16_t* count);
|
||||
|
||||
/* Builds Huffman lookup table assuming code lengths are in symbol order. */
|
||||
/* Returns size of resulting table. */
|
||||
BROTLI_INTERNAL uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table,
|
||||
int root_bits, const uint16_t* const symbol_lists, uint16_t* count_arg);
|
||||
|
||||
/* Builds a simple Huffman table. The num_symbols parameter is to be */
|
||||
/* interpreted as follows: 0 means 1 symbol, 1 means 2 symbols, 2 means 3 */
|
||||
/* symbols, 3 means 4 symbols with lengths 2,2,2,2, 4 means 4 symbols with */
|
||||
/* lengths 1,2,3,3. */
|
||||
BROTLI_INTERNAL uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table,
|
||||
int root_bits, uint16_t* symbols, uint32_t num_symbols);
|
||||
|
||||
/* Contains a collection of Huffman trees with the same alphabet size. */
|
||||
typedef struct {
|
||||
HuffmanCode** htrees;
|
||||
HuffmanCode* codes;
|
||||
uint16_t alphabet_size;
|
||||
uint16_t num_htrees;
|
||||
} HuffmanTreeGroup;
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_DEC_HUFFMAN_H_ */
|
||||
|
|
@ -1,168 +0,0 @@
|
|||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Macros for compiler / platform specific features and build options.
|
||||
|
||||
Build options are:
|
||||
* BROTLI_BUILD_32_BIT disables 64-bit optimizations
|
||||
* BROTLI_BUILD_64_BIT forces to use 64-bit optimizations
|
||||
* BROTLI_BUILD_BIG_ENDIAN forces to use big-endian optimizations
|
||||
* BROTLI_BUILD_ENDIAN_NEUTRAL disables endian-aware optimizations
|
||||
* BROTLI_BUILD_LITTLE_ENDIAN forces to use little-endian optimizations
|
||||
* BROTLI_BUILD_MODERN_COMPILER forces to use modern compilers built-ins,
|
||||
features and attributes
|
||||
* BROTLI_BUILD_PORTABLE disables dangerous optimizations, like unaligned
|
||||
read and overlapping memcpy; this reduces decompression speed by 5%
|
||||
* BROTLI_BUILD_NO_RBIT disables "rbit" optimization for ARM CPUs
|
||||
* BROTLI_DEBUG dumps file name and line number when decoder detects stream
|
||||
or memory error
|
||||
* BROTLI_ENABLE_LOG enables asserts and dumps various state information
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_DEC_PORT_H_
|
||||
#define BROTLI_DEC_PORT_H_
|
||||
|
||||
#if defined(BROTLI_ENABLE_LOG) || defined(BROTLI_DEBUG)
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#endif
|
||||
|
||||
#include <brotli/port.h>
|
||||
|
||||
#if defined(__arm__) || defined(__thumb__) || \
|
||||
defined(_M_ARM) || defined(_M_ARMT) || defined(__ARM64_ARCH_8__)
|
||||
#define BROTLI_TARGET_ARM
|
||||
#if (defined(__ARM_ARCH) && (__ARM_ARCH == 7)) || \
|
||||
(defined(M_ARM) && (M_ARM == 7))
|
||||
#define BROTLI_TARGET_ARMV7
|
||||
#endif /* ARMv7 */
|
||||
#if defined(__aarch64__) || defined(__ARM64_ARCH_8__)
|
||||
#define BROTLI_TARGET_ARMV8
|
||||
#endif /* ARMv8 */
|
||||
#endif /* ARM */
|
||||
|
||||
#if defined(__i386) || defined(_M_IX86)
|
||||
#define BROTLI_TARGET_X86
|
||||
#endif
|
||||
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define BROTLI_TARGET_X64
|
||||
#endif
|
||||
|
||||
#if defined(__PPC64__)
|
||||
#define BROTLI_TARGET_POWERPC64
|
||||
#endif
|
||||
|
||||
#ifdef BROTLI_BUILD_PORTABLE
|
||||
#define BROTLI_ALIGNED_READ (!!1)
|
||||
#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \
|
||||
defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8)
|
||||
/* Allow unaligned read only for white-listed CPUs. */
|
||||
#define BROTLI_ALIGNED_READ (!!0)
|
||||
#else
|
||||
#define BROTLI_ALIGNED_READ (!!1)
|
||||
#endif
|
||||
|
||||
/* IS_CONSTANT macros returns true for compile-time constant expressions. */
|
||||
#if BROTLI_MODERN_COMPILER || __has_builtin(__builtin_constant_p)
|
||||
#define IS_CONSTANT(x) (!!__builtin_constant_p(x))
|
||||
#else
|
||||
#define IS_CONSTANT(x) (!!0)
|
||||
#endif
|
||||
|
||||
#ifdef BROTLI_ENABLE_LOG
|
||||
#define BROTLI_DCHECK(x) assert(x)
|
||||
#define BROTLI_LOG(x) printf x
|
||||
#else
|
||||
#define BROTLI_DCHECK(x)
|
||||
#define BROTLI_LOG(x)
|
||||
#endif
|
||||
|
||||
#if defined(BROTLI_DEBUG) || defined(BROTLI_ENABLE_LOG)
|
||||
static BROTLI_INLINE void BrotliDump(const char* f, int l, const char* fn) {
|
||||
fprintf(stderr, "%s:%d (%s)\n", f, l, fn);
|
||||
fflush(stderr);
|
||||
}
|
||||
#define BROTLI_DUMP() BrotliDump(__FILE__, __LINE__, __FUNCTION__)
|
||||
#else
|
||||
#define BROTLI_DUMP() (void)(0)
|
||||
#endif
|
||||
|
||||
#if defined(BROTLI_BUILD_64_BIT)
|
||||
#define BROTLI_64_BITS 1
|
||||
#elif defined(BROTLI_BUILD_32_BIT)
|
||||
#define BROTLI_64_BITS 0
|
||||
#elif defined(BROTLI_TARGET_X64) || defined(BROTLI_TARGET_ARMV8) || \
|
||||
defined(BROTLI_TARGET_POWERPC64)
|
||||
#define BROTLI_64_BITS 1
|
||||
#else
|
||||
#define BROTLI_64_BITS 0
|
||||
#endif
|
||||
|
||||
#if (BROTLI_64_BITS)
|
||||
#define reg_t uint64_t
|
||||
#else
|
||||
#define reg_t uint32_t
|
||||
#endif
|
||||
|
||||
#if defined(BROTLI_BUILD_BIG_ENDIAN)
|
||||
#define BROTLI_LITTLE_ENDIAN 0
|
||||
#define BROTLI_BIG_ENDIAN 1
|
||||
#elif defined(BROTLI_BUILD_LITTLE_ENDIAN)
|
||||
#define BROTLI_LITTLE_ENDIAN 1
|
||||
#define BROTLI_BIG_ENDIAN 0
|
||||
#elif defined(BROTLI_BUILD_ENDIAN_NEUTRAL)
|
||||
#define BROTLI_LITTLE_ENDIAN 0
|
||||
#define BROTLI_BIG_ENDIAN 0
|
||||
#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
#define BROTLI_LITTLE_ENDIAN 1
|
||||
#define BROTLI_BIG_ENDIAN 0
|
||||
#elif defined(_WIN32)
|
||||
/* Win32 can currently always be assumed to be little endian */
|
||||
#define BROTLI_LITTLE_ENDIAN 1
|
||||
#define BROTLI_BIG_ENDIAN 0
|
||||
#else
|
||||
#if (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
|
||||
#define BROTLI_BIG_ENDIAN 1
|
||||
#else
|
||||
#define BROTLI_BIG_ENDIAN 0
|
||||
#endif
|
||||
#define BROTLI_LITTLE_ENDIAN 0
|
||||
#endif
|
||||
|
||||
#define BROTLI_REPEAT(N, X) { \
|
||||
if ((N & 1) != 0) {X;} \
|
||||
if ((N & 2) != 0) {X; X;} \
|
||||
if ((N & 4) != 0) {X; X; X; X;} \
|
||||
}
|
||||
|
||||
#if (BROTLI_MODERN_COMPILER || defined(__llvm__)) && \
|
||||
!defined(BROTLI_BUILD_NO_RBIT)
|
||||
#if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8)
|
||||
/* TODO: detect ARMv6T2 and enable this code for it. */
|
||||
static BROTLI_INLINE reg_t BrotliRBit(reg_t input) {
|
||||
reg_t output;
|
||||
__asm__("rbit %0, %1\n" : "=r"(output) : "r"(input));
|
||||
return output;
|
||||
}
|
||||
#define BROTLI_RBIT(x) BrotliRBit(x)
|
||||
#endif /* armv7 */
|
||||
#endif /* gcc || clang */
|
||||
|
||||
#if defined(BROTLI_TARGET_ARM)
|
||||
#define BROTLI_HAS_UBFX (!!1)
|
||||
#else
|
||||
#define BROTLI_HAS_UBFX (!!0)
|
||||
#endif
|
||||
|
||||
#define BROTLI_ALLOC(S, L) S->alloc_func(S->memory_manager_opaque, L)
|
||||
|
||||
#define BROTLI_FREE(S, X) { \
|
||||
S->free_func(S->memory_manager_opaque, X); \
|
||||
X = NULL; \
|
||||
}
|
||||
|
||||
#endif /* BROTLI_DEC_PORT_H_ */
|
||||
|
|
@ -1,751 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Lookup tables to map prefix codes to value ranges. This is used during
|
||||
decoding of the block lengths, literal insertion lengths and copy lengths.
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_DEC_PREFIX_H_
|
||||
#define BROTLI_DEC_PREFIX_H_
|
||||
|
||||
#include "../common/constants.h"
|
||||
#include <brotli/types.h>
|
||||
|
||||
/* Represents the range of values belonging to a prefix code: */
|
||||
/* [offset, offset + 2^nbits) */
|
||||
struct PrefixCodeRange {
|
||||
uint16_t offset;
|
||||
uint8_t nbits;
|
||||
};
|
||||
|
||||
static const struct PrefixCodeRange
|
||||
kBlockLengthPrefixCode[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
|
||||
{ 1, 2}, { 5, 2}, { 9, 2}, { 13, 2},
|
||||
{ 17, 3}, { 25, 3}, { 33, 3}, { 41, 3},
|
||||
{ 49, 4}, { 65, 4}, { 81, 4}, { 97, 4},
|
||||
{ 113, 5}, { 145, 5}, { 177, 5}, { 209, 5},
|
||||
{ 241, 6}, { 305, 6}, { 369, 7}, { 497, 8},
|
||||
{ 753, 9}, { 1265, 10}, {2289, 11}, {4337, 12},
|
||||
{8433, 13}, {16625, 24}
|
||||
};
|
||||
|
||||
typedef struct CmdLutElement {
|
||||
uint8_t insert_len_extra_bits;
|
||||
uint8_t copy_len_extra_bits;
|
||||
int8_t distance_code;
|
||||
uint8_t context;
|
||||
uint16_t insert_len_offset;
|
||||
uint16_t copy_len_offset;
|
||||
} CmdLutElement;
|
||||
|
||||
static const CmdLutElement kCmdLut[BROTLI_NUM_COMMAND_SYMBOLS] = {
|
||||
{ 0x00, 0x00, 0, 0x00, 0x0000, 0x0002 },
|
||||
{ 0x00, 0x00, 0, 0x01, 0x0000, 0x0003 },
|
||||
{ 0x00, 0x00, 0, 0x02, 0x0000, 0x0004 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0000, 0x0005 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0000, 0x0006 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0000, 0x0007 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0000, 0x0008 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0000, 0x0009 },
|
||||
{ 0x00, 0x00, 0, 0x00, 0x0001, 0x0002 },
|
||||
{ 0x00, 0x00, 0, 0x01, 0x0001, 0x0003 },
|
||||
{ 0x00, 0x00, 0, 0x02, 0x0001, 0x0004 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0001, 0x0005 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0001, 0x0006 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0001, 0x0007 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0001, 0x0008 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0001, 0x0009 },
|
||||
{ 0x00, 0x00, 0, 0x00, 0x0002, 0x0002 },
|
||||
{ 0x00, 0x00, 0, 0x01, 0x0002, 0x0003 },
|
||||
{ 0x00, 0x00, 0, 0x02, 0x0002, 0x0004 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0002, 0x0005 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0002, 0x0006 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0002, 0x0007 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0002, 0x0008 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0002, 0x0009 },
|
||||
{ 0x00, 0x00, 0, 0x00, 0x0003, 0x0002 },
|
||||
{ 0x00, 0x00, 0, 0x01, 0x0003, 0x0003 },
|
||||
{ 0x00, 0x00, 0, 0x02, 0x0003, 0x0004 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0003, 0x0005 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0003, 0x0006 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0003, 0x0007 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0003, 0x0008 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0003, 0x0009 },
|
||||
{ 0x00, 0x00, 0, 0x00, 0x0004, 0x0002 },
|
||||
{ 0x00, 0x00, 0, 0x01, 0x0004, 0x0003 },
|
||||
{ 0x00, 0x00, 0, 0x02, 0x0004, 0x0004 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0004, 0x0005 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0004, 0x0006 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0004, 0x0007 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0004, 0x0008 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0004, 0x0009 },
|
||||
{ 0x00, 0x00, 0, 0x00, 0x0005, 0x0002 },
|
||||
{ 0x00, 0x00, 0, 0x01, 0x0005, 0x0003 },
|
||||
{ 0x00, 0x00, 0, 0x02, 0x0005, 0x0004 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0005, 0x0005 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0005, 0x0006 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0005, 0x0007 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0005, 0x0008 },
|
||||
{ 0x00, 0x00, 0, 0x03, 0x0005, 0x0009 },
|
||||
{ 0x01, 0x00, 0, 0x00, 0x0006, 0x0002 },
|
||||
{ 0x01, 0x00, 0, 0x01, 0x0006, 0x0003 },
|
||||
{ 0x01, 0x00, 0, 0x02, 0x0006, 0x0004 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0006, 0x0005 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0006, 0x0006 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0006, 0x0007 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0006, 0x0008 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0006, 0x0009 },
|
||||
{ 0x01, 0x00, 0, 0x00, 0x0008, 0x0002 },
|
||||
{ 0x01, 0x00, 0, 0x01, 0x0008, 0x0003 },
|
||||
{ 0x01, 0x00, 0, 0x02, 0x0008, 0x0004 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0008, 0x0005 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0008, 0x0006 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0008, 0x0007 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0008, 0x0008 },
|
||||
{ 0x01, 0x00, 0, 0x03, 0x0008, 0x0009 },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0000, 0x000a },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0000, 0x000c },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0000, 0x000e },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0000, 0x0012 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0000, 0x0016 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0000, 0x001e },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0000, 0x0026 },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0000, 0x0036 },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0001, 0x000a },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0001, 0x000c },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0001, 0x000e },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0001, 0x0012 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0001, 0x0016 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0001, 0x001e },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0001, 0x0026 },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0001, 0x0036 },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0002, 0x000a },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0002, 0x000c },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0002, 0x000e },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0002, 0x0012 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0002, 0x0016 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0002, 0x001e },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0002, 0x0026 },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0002, 0x0036 },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0003, 0x000a },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0003, 0x000c },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0003, 0x000e },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0003, 0x0012 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0003, 0x0016 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0003, 0x001e },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0003, 0x0026 },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0003, 0x0036 },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0004, 0x000a },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0004, 0x000c },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0004, 0x000e },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0004, 0x0012 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0004, 0x0016 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0004, 0x001e },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0004, 0x0026 },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0004, 0x0036 },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0005, 0x000a },
|
||||
{ 0x00, 0x01, 0, 0x03, 0x0005, 0x000c },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0005, 0x000e },
|
||||
{ 0x00, 0x02, 0, 0x03, 0x0005, 0x0012 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0005, 0x0016 },
|
||||
{ 0x00, 0x03, 0, 0x03, 0x0005, 0x001e },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0005, 0x0026 },
|
||||
{ 0x00, 0x04, 0, 0x03, 0x0005, 0x0036 },
|
||||
{ 0x01, 0x01, 0, 0x03, 0x0006, 0x000a },
|
||||
{ 0x01, 0x01, 0, 0x03, 0x0006, 0x000c },
|
||||
{ 0x01, 0x02, 0, 0x03, 0x0006, 0x000e },
|
||||
{ 0x01, 0x02, 0, 0x03, 0x0006, 0x0012 },
|
||||
{ 0x01, 0x03, 0, 0x03, 0x0006, 0x0016 },
|
||||
{ 0x01, 0x03, 0, 0x03, 0x0006, 0x001e },
|
||||
{ 0x01, 0x04, 0, 0x03, 0x0006, 0x0026 },
|
||||
{ 0x01, 0x04, 0, 0x03, 0x0006, 0x0036 },
|
||||
{ 0x01, 0x01, 0, 0x03, 0x0008, 0x000a },
|
||||
{ 0x01, 0x01, 0, 0x03, 0x0008, 0x000c },
|
||||
{ 0x01, 0x02, 0, 0x03, 0x0008, 0x000e },
|
||||
{ 0x01, 0x02, 0, 0x03, 0x0008, 0x0012 },
|
||||
{ 0x01, 0x03, 0, 0x03, 0x0008, 0x0016 },
|
||||
{ 0x01, 0x03, 0, 0x03, 0x0008, 0x001e },
|
||||
{ 0x01, 0x04, 0, 0x03, 0x0008, 0x0026 },
|
||||
{ 0x01, 0x04, 0, 0x03, 0x0008, 0x0036 },
|
||||
{ 0x00, 0x00, -1, 0x00, 0x0000, 0x0002 },
|
||||
{ 0x00, 0x00, -1, 0x01, 0x0000, 0x0003 },
|
||||
{ 0x00, 0x00, -1, 0x02, 0x0000, 0x0004 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0000, 0x0005 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0000, 0x0006 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0000, 0x0007 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0000, 0x0008 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0000, 0x0009 },
|
||||
{ 0x00, 0x00, -1, 0x00, 0x0001, 0x0002 },
|
||||
{ 0x00, 0x00, -1, 0x01, 0x0001, 0x0003 },
|
||||
{ 0x00, 0x00, -1, 0x02, 0x0001, 0x0004 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0001, 0x0005 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0001, 0x0006 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0001, 0x0007 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0001, 0x0008 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0001, 0x0009 },
|
||||
{ 0x00, 0x00, -1, 0x00, 0x0002, 0x0002 },
|
||||
{ 0x00, 0x00, -1, 0x01, 0x0002, 0x0003 },
|
||||
{ 0x00, 0x00, -1, 0x02, 0x0002, 0x0004 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0002, 0x0005 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0002, 0x0006 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0002, 0x0007 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0002, 0x0008 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0002, 0x0009 },
|
||||
{ 0x00, 0x00, -1, 0x00, 0x0003, 0x0002 },
|
||||
{ 0x00, 0x00, -1, 0x01, 0x0003, 0x0003 },
|
||||
{ 0x00, 0x00, -1, 0x02, 0x0003, 0x0004 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0003, 0x0005 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0003, 0x0006 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0003, 0x0007 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0003, 0x0008 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0003, 0x0009 },
|
||||
{ 0x00, 0x00, -1, 0x00, 0x0004, 0x0002 },
|
||||
{ 0x00, 0x00, -1, 0x01, 0x0004, 0x0003 },
|
||||
{ 0x00, 0x00, -1, 0x02, 0x0004, 0x0004 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0004, 0x0005 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0004, 0x0006 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0004, 0x0007 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0004, 0x0008 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0004, 0x0009 },
|
||||
{ 0x00, 0x00, -1, 0x00, 0x0005, 0x0002 },
|
||||
{ 0x00, 0x00, -1, 0x01, 0x0005, 0x0003 },
|
||||
{ 0x00, 0x00, -1, 0x02, 0x0005, 0x0004 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0005, 0x0005 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0005, 0x0006 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0005, 0x0007 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0005, 0x0008 },
|
||||
{ 0x00, 0x00, -1, 0x03, 0x0005, 0x0009 },
|
||||
{ 0x01, 0x00, -1, 0x00, 0x0006, 0x0002 },
|
||||
{ 0x01, 0x00, -1, 0x01, 0x0006, 0x0003 },
|
||||
{ 0x01, 0x00, -1, 0x02, 0x0006, 0x0004 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0006, 0x0005 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0006, 0x0006 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0006, 0x0007 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0006, 0x0008 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0006, 0x0009 },
|
||||
{ 0x01, 0x00, -1, 0x00, 0x0008, 0x0002 },
|
||||
{ 0x01, 0x00, -1, 0x01, 0x0008, 0x0003 },
|
||||
{ 0x01, 0x00, -1, 0x02, 0x0008, 0x0004 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0008, 0x0005 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0008, 0x0006 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0008, 0x0007 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0008, 0x0008 },
|
||||
{ 0x01, 0x00, -1, 0x03, 0x0008, 0x0009 },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0000, 0x000a },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0000, 0x000c },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0000, 0x000e },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0000, 0x0012 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0000, 0x0016 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0000, 0x001e },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0000, 0x0026 },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0000, 0x0036 },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0001, 0x000a },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0001, 0x000c },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0001, 0x000e },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0001, 0x0012 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0001, 0x0016 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0001, 0x001e },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0001, 0x0026 },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0001, 0x0036 },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0002, 0x000a },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0002, 0x000c },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0002, 0x000e },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0002, 0x0012 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0002, 0x0016 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0002, 0x001e },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0002, 0x0026 },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0002, 0x0036 },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0003, 0x000a },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0003, 0x000c },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0003, 0x000e },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0003, 0x0012 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0003, 0x0016 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0003, 0x001e },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0003, 0x0026 },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0003, 0x0036 },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0004, 0x000a },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0004, 0x000c },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0004, 0x000e },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0004, 0x0012 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0004, 0x0016 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0004, 0x001e },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0004, 0x0026 },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0004, 0x0036 },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0005, 0x000a },
|
||||
{ 0x00, 0x01, -1, 0x03, 0x0005, 0x000c },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0005, 0x000e },
|
||||
{ 0x00, 0x02, -1, 0x03, 0x0005, 0x0012 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0005, 0x0016 },
|
||||
{ 0x00, 0x03, -1, 0x03, 0x0005, 0x001e },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0005, 0x0026 },
|
||||
{ 0x00, 0x04, -1, 0x03, 0x0005, 0x0036 },
|
||||
{ 0x01, 0x01, -1, 0x03, 0x0006, 0x000a },
|
||||
{ 0x01, 0x01, -1, 0x03, 0x0006, 0x000c },
|
||||
{ 0x01, 0x02, -1, 0x03, 0x0006, 0x000e },
|
||||
{ 0x01, 0x02, -1, 0x03, 0x0006, 0x0012 },
|
||||
{ 0x01, 0x03, -1, 0x03, 0x0006, 0x0016 },
|
||||
{ 0x01, 0x03, -1, 0x03, 0x0006, 0x001e },
|
||||
{ 0x01, 0x04, -1, 0x03, 0x0006, 0x0026 },
|
||||
{ 0x01, 0x04, -1, 0x03, 0x0006, 0x0036 },
|
||||
{ 0x01, 0x01, -1, 0x03, 0x0008, 0x000a },
|
||||
{ 0x01, 0x01, -1, 0x03, 0x0008, 0x000c },
|
||||
{ 0x01, 0x02, -1, 0x03, 0x0008, 0x000e },
|
||||
{ 0x01, 0x02, -1, 0x03, 0x0008, 0x0012 },
|
||||
{ 0x01, 0x03, -1, 0x03, 0x0008, 0x0016 },
|
||||
{ 0x01, 0x03, -1, 0x03, 0x0008, 0x001e },
|
||||
{ 0x01, 0x04, -1, 0x03, 0x0008, 0x0026 },
|
||||
{ 0x01, 0x04, -1, 0x03, 0x0008, 0x0036 },
|
||||
{ 0x02, 0x00, -1, 0x00, 0x000a, 0x0002 },
|
||||
{ 0x02, 0x00, -1, 0x01, 0x000a, 0x0003 },
|
||||
{ 0x02, 0x00, -1, 0x02, 0x000a, 0x0004 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000a, 0x0005 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000a, 0x0006 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000a, 0x0007 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000a, 0x0008 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000a, 0x0009 },
|
||||
{ 0x02, 0x00, -1, 0x00, 0x000e, 0x0002 },
|
||||
{ 0x02, 0x00, -1, 0x01, 0x000e, 0x0003 },
|
||||
{ 0x02, 0x00, -1, 0x02, 0x000e, 0x0004 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000e, 0x0005 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000e, 0x0006 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000e, 0x0007 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000e, 0x0008 },
|
||||
{ 0x02, 0x00, -1, 0x03, 0x000e, 0x0009 },
|
||||
{ 0x03, 0x00, -1, 0x00, 0x0012, 0x0002 },
|
||||
{ 0x03, 0x00, -1, 0x01, 0x0012, 0x0003 },
|
||||
{ 0x03, 0x00, -1, 0x02, 0x0012, 0x0004 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x0012, 0x0005 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x0012, 0x0006 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x0012, 0x0007 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x0012, 0x0008 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x0012, 0x0009 },
|
||||
{ 0x03, 0x00, -1, 0x00, 0x001a, 0x0002 },
|
||||
{ 0x03, 0x00, -1, 0x01, 0x001a, 0x0003 },
|
||||
{ 0x03, 0x00, -1, 0x02, 0x001a, 0x0004 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x001a, 0x0005 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x001a, 0x0006 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x001a, 0x0007 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x001a, 0x0008 },
|
||||
{ 0x03, 0x00, -1, 0x03, 0x001a, 0x0009 },
|
||||
{ 0x04, 0x00, -1, 0x00, 0x0022, 0x0002 },
|
||||
{ 0x04, 0x00, -1, 0x01, 0x0022, 0x0003 },
|
||||
{ 0x04, 0x00, -1, 0x02, 0x0022, 0x0004 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0022, 0x0005 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0022, 0x0006 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0022, 0x0007 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0022, 0x0008 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0022, 0x0009 },
|
||||
{ 0x04, 0x00, -1, 0x00, 0x0032, 0x0002 },
|
||||
{ 0x04, 0x00, -1, 0x01, 0x0032, 0x0003 },
|
||||
{ 0x04, 0x00, -1, 0x02, 0x0032, 0x0004 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0032, 0x0005 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0032, 0x0006 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0032, 0x0007 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0032, 0x0008 },
|
||||
{ 0x04, 0x00, -1, 0x03, 0x0032, 0x0009 },
|
||||
{ 0x05, 0x00, -1, 0x00, 0x0042, 0x0002 },
|
||||
{ 0x05, 0x00, -1, 0x01, 0x0042, 0x0003 },
|
||||
{ 0x05, 0x00, -1, 0x02, 0x0042, 0x0004 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0042, 0x0005 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0042, 0x0006 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0042, 0x0007 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0042, 0x0008 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0042, 0x0009 },
|
||||
{ 0x05, 0x00, -1, 0x00, 0x0062, 0x0002 },
|
||||
{ 0x05, 0x00, -1, 0x01, 0x0062, 0x0003 },
|
||||
{ 0x05, 0x00, -1, 0x02, 0x0062, 0x0004 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0062, 0x0005 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0062, 0x0006 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0062, 0x0007 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0062, 0x0008 },
|
||||
{ 0x05, 0x00, -1, 0x03, 0x0062, 0x0009 },
|
||||
{ 0x02, 0x01, -1, 0x03, 0x000a, 0x000a },
|
||||
{ 0x02, 0x01, -1, 0x03, 0x000a, 0x000c },
|
||||
{ 0x02, 0x02, -1, 0x03, 0x000a, 0x000e },
|
||||
{ 0x02, 0x02, -1, 0x03, 0x000a, 0x0012 },
|
||||
{ 0x02, 0x03, -1, 0x03, 0x000a, 0x0016 },
|
||||
{ 0x02, 0x03, -1, 0x03, 0x000a, 0x001e },
|
||||
{ 0x02, 0x04, -1, 0x03, 0x000a, 0x0026 },
|
||||
{ 0x02, 0x04, -1, 0x03, 0x000a, 0x0036 },
|
||||
{ 0x02, 0x01, -1, 0x03, 0x000e, 0x000a },
|
||||
{ 0x02, 0x01, -1, 0x03, 0x000e, 0x000c },
|
||||
{ 0x02, 0x02, -1, 0x03, 0x000e, 0x000e },
|
||||
{ 0x02, 0x02, -1, 0x03, 0x000e, 0x0012 },
|
||||
{ 0x02, 0x03, -1, 0x03, 0x000e, 0x0016 },
|
||||
{ 0x02, 0x03, -1, 0x03, 0x000e, 0x001e },
|
||||
{ 0x02, 0x04, -1, 0x03, 0x000e, 0x0026 },
|
||||
{ 0x02, 0x04, -1, 0x03, 0x000e, 0x0036 },
|
||||
{ 0x03, 0x01, -1, 0x03, 0x0012, 0x000a },
|
||||
{ 0x03, 0x01, -1, 0x03, 0x0012, 0x000c },
|
||||
{ 0x03, 0x02, -1, 0x03, 0x0012, 0x000e },
|
||||
{ 0x03, 0x02, -1, 0x03, 0x0012, 0x0012 },
|
||||
{ 0x03, 0x03, -1, 0x03, 0x0012, 0x0016 },
|
||||
{ 0x03, 0x03, -1, 0x03, 0x0012, 0x001e },
|
||||
{ 0x03, 0x04, -1, 0x03, 0x0012, 0x0026 },
|
||||
{ 0x03, 0x04, -1, 0x03, 0x0012, 0x0036 },
|
||||
{ 0x03, 0x01, -1, 0x03, 0x001a, 0x000a },
|
||||
{ 0x03, 0x01, -1, 0x03, 0x001a, 0x000c },
|
||||
{ 0x03, 0x02, -1, 0x03, 0x001a, 0x000e },
|
||||
{ 0x03, 0x02, -1, 0x03, 0x001a, 0x0012 },
|
||||
{ 0x03, 0x03, -1, 0x03, 0x001a, 0x0016 },
|
||||
{ 0x03, 0x03, -1, 0x03, 0x001a, 0x001e },
|
||||
{ 0x03, 0x04, -1, 0x03, 0x001a, 0x0026 },
|
||||
{ 0x03, 0x04, -1, 0x03, 0x001a, 0x0036 },
|
||||
{ 0x04, 0x01, -1, 0x03, 0x0022, 0x000a },
|
||||
{ 0x04, 0x01, -1, 0x03, 0x0022, 0x000c },
|
||||
{ 0x04, 0x02, -1, 0x03, 0x0022, 0x000e },
|
||||
{ 0x04, 0x02, -1, 0x03, 0x0022, 0x0012 },
|
||||
{ 0x04, 0x03, -1, 0x03, 0x0022, 0x0016 },
|
||||
{ 0x04, 0x03, -1, 0x03, 0x0022, 0x001e },
|
||||
{ 0x04, 0x04, -1, 0x03, 0x0022, 0x0026 },
|
||||
{ 0x04, 0x04, -1, 0x03, 0x0022, 0x0036 },
|
||||
{ 0x04, 0x01, -1, 0x03, 0x0032, 0x000a },
|
||||
{ 0x04, 0x01, -1, 0x03, 0x0032, 0x000c },
|
||||
{ 0x04, 0x02, -1, 0x03, 0x0032, 0x000e },
|
||||
{ 0x04, 0x02, -1, 0x03, 0x0032, 0x0012 },
|
||||
{ 0x04, 0x03, -1, 0x03, 0x0032, 0x0016 },
|
||||
{ 0x04, 0x03, -1, 0x03, 0x0032, 0x001e },
|
||||
{ 0x04, 0x04, -1, 0x03, 0x0032, 0x0026 },
|
||||
{ 0x04, 0x04, -1, 0x03, 0x0032, 0x0036 },
|
||||
{ 0x05, 0x01, -1, 0x03, 0x0042, 0x000a },
|
||||
{ 0x05, 0x01, -1, 0x03, 0x0042, 0x000c },
|
||||
{ 0x05, 0x02, -1, 0x03, 0x0042, 0x000e },
|
||||
{ 0x05, 0x02, -1, 0x03, 0x0042, 0x0012 },
|
||||
{ 0x05, 0x03, -1, 0x03, 0x0042, 0x0016 },
|
||||
{ 0x05, 0x03, -1, 0x03, 0x0042, 0x001e },
|
||||
{ 0x05, 0x04, -1, 0x03, 0x0042, 0x0026 },
|
||||
{ 0x05, 0x04, -1, 0x03, 0x0042, 0x0036 },
|
||||
{ 0x05, 0x01, -1, 0x03, 0x0062, 0x000a },
|
||||
{ 0x05, 0x01, -1, 0x03, 0x0062, 0x000c },
|
||||
{ 0x05, 0x02, -1, 0x03, 0x0062, 0x000e },
|
||||
{ 0x05, 0x02, -1, 0x03, 0x0062, 0x0012 },
|
||||
{ 0x05, 0x03, -1, 0x03, 0x0062, 0x0016 },
|
||||
{ 0x05, 0x03, -1, 0x03, 0x0062, 0x001e },
|
||||
{ 0x05, 0x04, -1, 0x03, 0x0062, 0x0026 },
|
||||
{ 0x05, 0x04, -1, 0x03, 0x0062, 0x0036 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0000, 0x0046 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0000, 0x0066 },
|
||||
{ 0x00, 0x06, -1, 0x03, 0x0000, 0x0086 },
|
||||
{ 0x00, 0x07, -1, 0x03, 0x0000, 0x00c6 },
|
||||
{ 0x00, 0x08, -1, 0x03, 0x0000, 0x0146 },
|
||||
{ 0x00, 0x09, -1, 0x03, 0x0000, 0x0246 },
|
||||
{ 0x00, 0x0a, -1, 0x03, 0x0000, 0x0446 },
|
||||
{ 0x00, 0x18, -1, 0x03, 0x0000, 0x0846 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0001, 0x0046 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0001, 0x0066 },
|
||||
{ 0x00, 0x06, -1, 0x03, 0x0001, 0x0086 },
|
||||
{ 0x00, 0x07, -1, 0x03, 0x0001, 0x00c6 },
|
||||
{ 0x00, 0x08, -1, 0x03, 0x0001, 0x0146 },
|
||||
{ 0x00, 0x09, -1, 0x03, 0x0001, 0x0246 },
|
||||
{ 0x00, 0x0a, -1, 0x03, 0x0001, 0x0446 },
|
||||
{ 0x00, 0x18, -1, 0x03, 0x0001, 0x0846 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0002, 0x0046 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0002, 0x0066 },
|
||||
{ 0x00, 0x06, -1, 0x03, 0x0002, 0x0086 },
|
||||
{ 0x00, 0x07, -1, 0x03, 0x0002, 0x00c6 },
|
||||
{ 0x00, 0x08, -1, 0x03, 0x0002, 0x0146 },
|
||||
{ 0x00, 0x09, -1, 0x03, 0x0002, 0x0246 },
|
||||
{ 0x00, 0x0a, -1, 0x03, 0x0002, 0x0446 },
|
||||
{ 0x00, 0x18, -1, 0x03, 0x0002, 0x0846 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0003, 0x0046 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0003, 0x0066 },
|
||||
{ 0x00, 0x06, -1, 0x03, 0x0003, 0x0086 },
|
||||
{ 0x00, 0x07, -1, 0x03, 0x0003, 0x00c6 },
|
||||
{ 0x00, 0x08, -1, 0x03, 0x0003, 0x0146 },
|
||||
{ 0x00, 0x09, -1, 0x03, 0x0003, 0x0246 },
|
||||
{ 0x00, 0x0a, -1, 0x03, 0x0003, 0x0446 },
|
||||
{ 0x00, 0x18, -1, 0x03, 0x0003, 0x0846 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0004, 0x0046 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0004, 0x0066 },
|
||||
{ 0x00, 0x06, -1, 0x03, 0x0004, 0x0086 },
|
||||
{ 0x00, 0x07, -1, 0x03, 0x0004, 0x00c6 },
|
||||
{ 0x00, 0x08, -1, 0x03, 0x0004, 0x0146 },
|
||||
{ 0x00, 0x09, -1, 0x03, 0x0004, 0x0246 },
|
||||
{ 0x00, 0x0a, -1, 0x03, 0x0004, 0x0446 },
|
||||
{ 0x00, 0x18, -1, 0x03, 0x0004, 0x0846 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0005, 0x0046 },
|
||||
{ 0x00, 0x05, -1, 0x03, 0x0005, 0x0066 },
|
||||
{ 0x00, 0x06, -1, 0x03, 0x0005, 0x0086 },
|
||||
{ 0x00, 0x07, -1, 0x03, 0x0005, 0x00c6 },
|
||||
{ 0x00, 0x08, -1, 0x03, 0x0005, 0x0146 },
|
||||
{ 0x00, 0x09, -1, 0x03, 0x0005, 0x0246 },
|
||||
{ 0x00, 0x0a, -1, 0x03, 0x0005, 0x0446 },
|
||||
{ 0x00, 0x18, -1, 0x03, 0x0005, 0x0846 },
|
||||
{ 0x01, 0x05, -1, 0x03, 0x0006, 0x0046 },
|
||||
{ 0x01, 0x05, -1, 0x03, 0x0006, 0x0066 },
|
||||
{ 0x01, 0x06, -1, 0x03, 0x0006, 0x0086 },
|
||||
{ 0x01, 0x07, -1, 0x03, 0x0006, 0x00c6 },
|
||||
{ 0x01, 0x08, -1, 0x03, 0x0006, 0x0146 },
|
||||
{ 0x01, 0x09, -1, 0x03, 0x0006, 0x0246 },
|
||||
{ 0x01, 0x0a, -1, 0x03, 0x0006, 0x0446 },
|
||||
{ 0x01, 0x18, -1, 0x03, 0x0006, 0x0846 },
|
||||
{ 0x01, 0x05, -1, 0x03, 0x0008, 0x0046 },
|
||||
{ 0x01, 0x05, -1, 0x03, 0x0008, 0x0066 },
|
||||
{ 0x01, 0x06, -1, 0x03, 0x0008, 0x0086 },
|
||||
{ 0x01, 0x07, -1, 0x03, 0x0008, 0x00c6 },
|
||||
{ 0x01, 0x08, -1, 0x03, 0x0008, 0x0146 },
|
||||
{ 0x01, 0x09, -1, 0x03, 0x0008, 0x0246 },
|
||||
{ 0x01, 0x0a, -1, 0x03, 0x0008, 0x0446 },
|
||||
{ 0x01, 0x18, -1, 0x03, 0x0008, 0x0846 },
|
||||
{ 0x06, 0x00, -1, 0x00, 0x0082, 0x0002 },
|
||||
{ 0x06, 0x00, -1, 0x01, 0x0082, 0x0003 },
|
||||
{ 0x06, 0x00, -1, 0x02, 0x0082, 0x0004 },
|
||||
{ 0x06, 0x00, -1, 0x03, 0x0082, 0x0005 },
|
||||
{ 0x06, 0x00, -1, 0x03, 0x0082, 0x0006 },
|
||||
{ 0x06, 0x00, -1, 0x03, 0x0082, 0x0007 },
|
||||
{ 0x06, 0x00, -1, 0x03, 0x0082, 0x0008 },
|
||||
{ 0x06, 0x00, -1, 0x03, 0x0082, 0x0009 },
|
||||
{ 0x07, 0x00, -1, 0x00, 0x00c2, 0x0002 },
|
||||
{ 0x07, 0x00, -1, 0x01, 0x00c2, 0x0003 },
|
||||
{ 0x07, 0x00, -1, 0x02, 0x00c2, 0x0004 },
|
||||
{ 0x07, 0x00, -1, 0x03, 0x00c2, 0x0005 },
|
||||
{ 0x07, 0x00, -1, 0x03, 0x00c2, 0x0006 },
|
||||
{ 0x07, 0x00, -1, 0x03, 0x00c2, 0x0007 },
|
||||
{ 0x07, 0x00, -1, 0x03, 0x00c2, 0x0008 },
|
||||
{ 0x07, 0x00, -1, 0x03, 0x00c2, 0x0009 },
|
||||
{ 0x08, 0x00, -1, 0x00, 0x0142, 0x0002 },
|
||||
{ 0x08, 0x00, -1, 0x01, 0x0142, 0x0003 },
|
||||
{ 0x08, 0x00, -1, 0x02, 0x0142, 0x0004 },
|
||||
{ 0x08, 0x00, -1, 0x03, 0x0142, 0x0005 },
|
||||
{ 0x08, 0x00, -1, 0x03, 0x0142, 0x0006 },
|
||||
{ 0x08, 0x00, -1, 0x03, 0x0142, 0x0007 },
|
||||
{ 0x08, 0x00, -1, 0x03, 0x0142, 0x0008 },
|
||||
{ 0x08, 0x00, -1, 0x03, 0x0142, 0x0009 },
|
||||
{ 0x09, 0x00, -1, 0x00, 0x0242, 0x0002 },
|
||||
{ 0x09, 0x00, -1, 0x01, 0x0242, 0x0003 },
|
||||
{ 0x09, 0x00, -1, 0x02, 0x0242, 0x0004 },
|
||||
{ 0x09, 0x00, -1, 0x03, 0x0242, 0x0005 },
|
||||
{ 0x09, 0x00, -1, 0x03, 0x0242, 0x0006 },
|
||||
{ 0x09, 0x00, -1, 0x03, 0x0242, 0x0007 },
|
||||
{ 0x09, 0x00, -1, 0x03, 0x0242, 0x0008 },
|
||||
{ 0x09, 0x00, -1, 0x03, 0x0242, 0x0009 },
|
||||
{ 0x0a, 0x00, -1, 0x00, 0x0442, 0x0002 },
|
||||
{ 0x0a, 0x00, -1, 0x01, 0x0442, 0x0003 },
|
||||
{ 0x0a, 0x00, -1, 0x02, 0x0442, 0x0004 },
|
||||
{ 0x0a, 0x00, -1, 0x03, 0x0442, 0x0005 },
|
||||
{ 0x0a, 0x00, -1, 0x03, 0x0442, 0x0006 },
|
||||
{ 0x0a, 0x00, -1, 0x03, 0x0442, 0x0007 },
|
||||
{ 0x0a, 0x00, -1, 0x03, 0x0442, 0x0008 },
|
||||
{ 0x0a, 0x00, -1, 0x03, 0x0442, 0x0009 },
|
||||
{ 0x0c, 0x00, -1, 0x00, 0x0842, 0x0002 },
|
||||
{ 0x0c, 0x00, -1, 0x01, 0x0842, 0x0003 },
|
||||
{ 0x0c, 0x00, -1, 0x02, 0x0842, 0x0004 },
|
||||
{ 0x0c, 0x00, -1, 0x03, 0x0842, 0x0005 },
|
||||
{ 0x0c, 0x00, -1, 0x03, 0x0842, 0x0006 },
|
||||
{ 0x0c, 0x00, -1, 0x03, 0x0842, 0x0007 },
|
||||
{ 0x0c, 0x00, -1, 0x03, 0x0842, 0x0008 },
|
||||
{ 0x0c, 0x00, -1, 0x03, 0x0842, 0x0009 },
|
||||
{ 0x0e, 0x00, -1, 0x00, 0x1842, 0x0002 },
|
||||
{ 0x0e, 0x00, -1, 0x01, 0x1842, 0x0003 },
|
||||
{ 0x0e, 0x00, -1, 0x02, 0x1842, 0x0004 },
|
||||
{ 0x0e, 0x00, -1, 0x03, 0x1842, 0x0005 },
|
||||
{ 0x0e, 0x00, -1, 0x03, 0x1842, 0x0006 },
|
||||
{ 0x0e, 0x00, -1, 0x03, 0x1842, 0x0007 },
|
||||
{ 0x0e, 0x00, -1, 0x03, 0x1842, 0x0008 },
|
||||
{ 0x0e, 0x00, -1, 0x03, 0x1842, 0x0009 },
|
||||
{ 0x18, 0x00, -1, 0x00, 0x5842, 0x0002 },
|
||||
{ 0x18, 0x00, -1, 0x01, 0x5842, 0x0003 },
|
||||
{ 0x18, 0x00, -1, 0x02, 0x5842, 0x0004 },
|
||||
{ 0x18, 0x00, -1, 0x03, 0x5842, 0x0005 },
|
||||
{ 0x18, 0x00, -1, 0x03, 0x5842, 0x0006 },
|
||||
{ 0x18, 0x00, -1, 0x03, 0x5842, 0x0007 },
|
||||
{ 0x18, 0x00, -1, 0x03, 0x5842, 0x0008 },
|
||||
{ 0x18, 0x00, -1, 0x03, 0x5842, 0x0009 },
|
||||
{ 0x02, 0x05, -1, 0x03, 0x000a, 0x0046 },
|
||||
{ 0x02, 0x05, -1, 0x03, 0x000a, 0x0066 },
|
||||
{ 0x02, 0x06, -1, 0x03, 0x000a, 0x0086 },
|
||||
{ 0x02, 0x07, -1, 0x03, 0x000a, 0x00c6 },
|
||||
{ 0x02, 0x08, -1, 0x03, 0x000a, 0x0146 },
|
||||
{ 0x02, 0x09, -1, 0x03, 0x000a, 0x0246 },
|
||||
{ 0x02, 0x0a, -1, 0x03, 0x000a, 0x0446 },
|
||||
{ 0x02, 0x18, -1, 0x03, 0x000a, 0x0846 },
|
||||
{ 0x02, 0x05, -1, 0x03, 0x000e, 0x0046 },
|
||||
{ 0x02, 0x05, -1, 0x03, 0x000e, 0x0066 },
|
||||
{ 0x02, 0x06, -1, 0x03, 0x000e, 0x0086 },
|
||||
{ 0x02, 0x07, -1, 0x03, 0x000e, 0x00c6 },
|
||||
{ 0x02, 0x08, -1, 0x03, 0x000e, 0x0146 },
|
||||
{ 0x02, 0x09, -1, 0x03, 0x000e, 0x0246 },
|
||||
{ 0x02, 0x0a, -1, 0x03, 0x000e, 0x0446 },
|
||||
{ 0x02, 0x18, -1, 0x03, 0x000e, 0x0846 },
|
||||
{ 0x03, 0x05, -1, 0x03, 0x0012, 0x0046 },
|
||||
{ 0x03, 0x05, -1, 0x03, 0x0012, 0x0066 },
|
||||
{ 0x03, 0x06, -1, 0x03, 0x0012, 0x0086 },
|
||||
{ 0x03, 0x07, -1, 0x03, 0x0012, 0x00c6 },
|
||||
{ 0x03, 0x08, -1, 0x03, 0x0012, 0x0146 },
|
||||
{ 0x03, 0x09, -1, 0x03, 0x0012, 0x0246 },
|
||||
{ 0x03, 0x0a, -1, 0x03, 0x0012, 0x0446 },
|
||||
{ 0x03, 0x18, -1, 0x03, 0x0012, 0x0846 },
|
||||
{ 0x03, 0x05, -1, 0x03, 0x001a, 0x0046 },
|
||||
{ 0x03, 0x05, -1, 0x03, 0x001a, 0x0066 },
|
||||
{ 0x03, 0x06, -1, 0x03, 0x001a, 0x0086 },
|
||||
{ 0x03, 0x07, -1, 0x03, 0x001a, 0x00c6 },
|
||||
{ 0x03, 0x08, -1, 0x03, 0x001a, 0x0146 },
|
||||
{ 0x03, 0x09, -1, 0x03, 0x001a, 0x0246 },
|
||||
{ 0x03, 0x0a, -1, 0x03, 0x001a, 0x0446 },
|
||||
{ 0x03, 0x18, -1, 0x03, 0x001a, 0x0846 },
|
||||
{ 0x04, 0x05, -1, 0x03, 0x0022, 0x0046 },
|
||||
{ 0x04, 0x05, -1, 0x03, 0x0022, 0x0066 },
|
||||
{ 0x04, 0x06, -1, 0x03, 0x0022, 0x0086 },
|
||||
{ 0x04, 0x07, -1, 0x03, 0x0022, 0x00c6 },
|
||||
{ 0x04, 0x08, -1, 0x03, 0x0022, 0x0146 },
|
||||
{ 0x04, 0x09, -1, 0x03, 0x0022, 0x0246 },
|
||||
{ 0x04, 0x0a, -1, 0x03, 0x0022, 0x0446 },
|
||||
{ 0x04, 0x18, -1, 0x03, 0x0022, 0x0846 },
|
||||
{ 0x04, 0x05, -1, 0x03, 0x0032, 0x0046 },
|
||||
{ 0x04, 0x05, -1, 0x03, 0x0032, 0x0066 },
|
||||
{ 0x04, 0x06, -1, 0x03, 0x0032, 0x0086 },
|
||||
{ 0x04, 0x07, -1, 0x03, 0x0032, 0x00c6 },
|
||||
{ 0x04, 0x08, -1, 0x03, 0x0032, 0x0146 },
|
||||
{ 0x04, 0x09, -1, 0x03, 0x0032, 0x0246 },
|
||||
{ 0x04, 0x0a, -1, 0x03, 0x0032, 0x0446 },
|
||||
{ 0x04, 0x18, -1, 0x03, 0x0032, 0x0846 },
|
||||
{ 0x05, 0x05, -1, 0x03, 0x0042, 0x0046 },
|
||||
{ 0x05, 0x05, -1, 0x03, 0x0042, 0x0066 },
|
||||
{ 0x05, 0x06, -1, 0x03, 0x0042, 0x0086 },
|
||||
{ 0x05, 0x07, -1, 0x03, 0x0042, 0x00c6 },
|
||||
{ 0x05, 0x08, -1, 0x03, 0x0042, 0x0146 },
|
||||
{ 0x05, 0x09, -1, 0x03, 0x0042, 0x0246 },
|
||||
{ 0x05, 0x0a, -1, 0x03, 0x0042, 0x0446 },
|
||||
{ 0x05, 0x18, -1, 0x03, 0x0042, 0x0846 },
|
||||
{ 0x05, 0x05, -1, 0x03, 0x0062, 0x0046 },
|
||||
{ 0x05, 0x05, -1, 0x03, 0x0062, 0x0066 },
|
||||
{ 0x05, 0x06, -1, 0x03, 0x0062, 0x0086 },
|
||||
{ 0x05, 0x07, -1, 0x03, 0x0062, 0x00c6 },
|
||||
{ 0x05, 0x08, -1, 0x03, 0x0062, 0x0146 },
|
||||
{ 0x05, 0x09, -1, 0x03, 0x0062, 0x0246 },
|
||||
{ 0x05, 0x0a, -1, 0x03, 0x0062, 0x0446 },
|
||||
{ 0x05, 0x18, -1, 0x03, 0x0062, 0x0846 },
|
||||
{ 0x06, 0x01, -1, 0x03, 0x0082, 0x000a },
|
||||
{ 0x06, 0x01, -1, 0x03, 0x0082, 0x000c },
|
||||
{ 0x06, 0x02, -1, 0x03, 0x0082, 0x000e },
|
||||
{ 0x06, 0x02, -1, 0x03, 0x0082, 0x0012 },
|
||||
{ 0x06, 0x03, -1, 0x03, 0x0082, 0x0016 },
|
||||
{ 0x06, 0x03, -1, 0x03, 0x0082, 0x001e },
|
||||
{ 0x06, 0x04, -1, 0x03, 0x0082, 0x0026 },
|
||||
{ 0x06, 0x04, -1, 0x03, 0x0082, 0x0036 },
|
||||
{ 0x07, 0x01, -1, 0x03, 0x00c2, 0x000a },
|
||||
{ 0x07, 0x01, -1, 0x03, 0x00c2, 0x000c },
|
||||
{ 0x07, 0x02, -1, 0x03, 0x00c2, 0x000e },
|
||||
{ 0x07, 0x02, -1, 0x03, 0x00c2, 0x0012 },
|
||||
{ 0x07, 0x03, -1, 0x03, 0x00c2, 0x0016 },
|
||||
{ 0x07, 0x03, -1, 0x03, 0x00c2, 0x001e },
|
||||
{ 0x07, 0x04, -1, 0x03, 0x00c2, 0x0026 },
|
||||
{ 0x07, 0x04, -1, 0x03, 0x00c2, 0x0036 },
|
||||
{ 0x08, 0x01, -1, 0x03, 0x0142, 0x000a },
|
||||
{ 0x08, 0x01, -1, 0x03, 0x0142, 0x000c },
|
||||
{ 0x08, 0x02, -1, 0x03, 0x0142, 0x000e },
|
||||
{ 0x08, 0x02, -1, 0x03, 0x0142, 0x0012 },
|
||||
{ 0x08, 0x03, -1, 0x03, 0x0142, 0x0016 },
|
||||
{ 0x08, 0x03, -1, 0x03, 0x0142, 0x001e },
|
||||
{ 0x08, 0x04, -1, 0x03, 0x0142, 0x0026 },
|
||||
{ 0x08, 0x04, -1, 0x03, 0x0142, 0x0036 },
|
||||
{ 0x09, 0x01, -1, 0x03, 0x0242, 0x000a },
|
||||
{ 0x09, 0x01, -1, 0x03, 0x0242, 0x000c },
|
||||
{ 0x09, 0x02, -1, 0x03, 0x0242, 0x000e },
|
||||
{ 0x09, 0x02, -1, 0x03, 0x0242, 0x0012 },
|
||||
{ 0x09, 0x03, -1, 0x03, 0x0242, 0x0016 },
|
||||
{ 0x09, 0x03, -1, 0x03, 0x0242, 0x001e },
|
||||
{ 0x09, 0x04, -1, 0x03, 0x0242, 0x0026 },
|
||||
{ 0x09, 0x04, -1, 0x03, 0x0242, 0x0036 },
|
||||
{ 0x0a, 0x01, -1, 0x03, 0x0442, 0x000a },
|
||||
{ 0x0a, 0x01, -1, 0x03, 0x0442, 0x000c },
|
||||
{ 0x0a, 0x02, -1, 0x03, 0x0442, 0x000e },
|
||||
{ 0x0a, 0x02, -1, 0x03, 0x0442, 0x0012 },
|
||||
{ 0x0a, 0x03, -1, 0x03, 0x0442, 0x0016 },
|
||||
{ 0x0a, 0x03, -1, 0x03, 0x0442, 0x001e },
|
||||
{ 0x0a, 0x04, -1, 0x03, 0x0442, 0x0026 },
|
||||
{ 0x0a, 0x04, -1, 0x03, 0x0442, 0x0036 },
|
||||
{ 0x0c, 0x01, -1, 0x03, 0x0842, 0x000a },
|
||||
{ 0x0c, 0x01, -1, 0x03, 0x0842, 0x000c },
|
||||
{ 0x0c, 0x02, -1, 0x03, 0x0842, 0x000e },
|
||||
{ 0x0c, 0x02, -1, 0x03, 0x0842, 0x0012 },
|
||||
{ 0x0c, 0x03, -1, 0x03, 0x0842, 0x0016 },
|
||||
{ 0x0c, 0x03, -1, 0x03, 0x0842, 0x001e },
|
||||
{ 0x0c, 0x04, -1, 0x03, 0x0842, 0x0026 },
|
||||
{ 0x0c, 0x04, -1, 0x03, 0x0842, 0x0036 },
|
||||
{ 0x0e, 0x01, -1, 0x03, 0x1842, 0x000a },
|
||||
{ 0x0e, 0x01, -1, 0x03, 0x1842, 0x000c },
|
||||
{ 0x0e, 0x02, -1, 0x03, 0x1842, 0x000e },
|
||||
{ 0x0e, 0x02, -1, 0x03, 0x1842, 0x0012 },
|
||||
{ 0x0e, 0x03, -1, 0x03, 0x1842, 0x0016 },
|
||||
{ 0x0e, 0x03, -1, 0x03, 0x1842, 0x001e },
|
||||
{ 0x0e, 0x04, -1, 0x03, 0x1842, 0x0026 },
|
||||
{ 0x0e, 0x04, -1, 0x03, 0x1842, 0x0036 },
|
||||
{ 0x18, 0x01, -1, 0x03, 0x5842, 0x000a },
|
||||
{ 0x18, 0x01, -1, 0x03, 0x5842, 0x000c },
|
||||
{ 0x18, 0x02, -1, 0x03, 0x5842, 0x000e },
|
||||
{ 0x18, 0x02, -1, 0x03, 0x5842, 0x0012 },
|
||||
{ 0x18, 0x03, -1, 0x03, 0x5842, 0x0016 },
|
||||
{ 0x18, 0x03, -1, 0x03, 0x5842, 0x001e },
|
||||
{ 0x18, 0x04, -1, 0x03, 0x5842, 0x0026 },
|
||||
{ 0x18, 0x04, -1, 0x03, 0x5842, 0x0036 },
|
||||
{ 0x06, 0x05, -1, 0x03, 0x0082, 0x0046 },
|
||||
{ 0x06, 0x05, -1, 0x03, 0x0082, 0x0066 },
|
||||
{ 0x06, 0x06, -1, 0x03, 0x0082, 0x0086 },
|
||||
{ 0x06, 0x07, -1, 0x03, 0x0082, 0x00c6 },
|
||||
{ 0x06, 0x08, -1, 0x03, 0x0082, 0x0146 },
|
||||
{ 0x06, 0x09, -1, 0x03, 0x0082, 0x0246 },
|
||||
{ 0x06, 0x0a, -1, 0x03, 0x0082, 0x0446 },
|
||||
{ 0x06, 0x18, -1, 0x03, 0x0082, 0x0846 },
|
||||
{ 0x07, 0x05, -1, 0x03, 0x00c2, 0x0046 },
|
||||
{ 0x07, 0x05, -1, 0x03, 0x00c2, 0x0066 },
|
||||
{ 0x07, 0x06, -1, 0x03, 0x00c2, 0x0086 },
|
||||
{ 0x07, 0x07, -1, 0x03, 0x00c2, 0x00c6 },
|
||||
{ 0x07, 0x08, -1, 0x03, 0x00c2, 0x0146 },
|
||||
{ 0x07, 0x09, -1, 0x03, 0x00c2, 0x0246 },
|
||||
{ 0x07, 0x0a, -1, 0x03, 0x00c2, 0x0446 },
|
||||
{ 0x07, 0x18, -1, 0x03, 0x00c2, 0x0846 },
|
||||
{ 0x08, 0x05, -1, 0x03, 0x0142, 0x0046 },
|
||||
{ 0x08, 0x05, -1, 0x03, 0x0142, 0x0066 },
|
||||
{ 0x08, 0x06, -1, 0x03, 0x0142, 0x0086 },
|
||||
{ 0x08, 0x07, -1, 0x03, 0x0142, 0x00c6 },
|
||||
{ 0x08, 0x08, -1, 0x03, 0x0142, 0x0146 },
|
||||
{ 0x08, 0x09, -1, 0x03, 0x0142, 0x0246 },
|
||||
{ 0x08, 0x0a, -1, 0x03, 0x0142, 0x0446 },
|
||||
{ 0x08, 0x18, -1, 0x03, 0x0142, 0x0846 },
|
||||
{ 0x09, 0x05, -1, 0x03, 0x0242, 0x0046 },
|
||||
{ 0x09, 0x05, -1, 0x03, 0x0242, 0x0066 },
|
||||
{ 0x09, 0x06, -1, 0x03, 0x0242, 0x0086 },
|
||||
{ 0x09, 0x07, -1, 0x03, 0x0242, 0x00c6 },
|
||||
{ 0x09, 0x08, -1, 0x03, 0x0242, 0x0146 },
|
||||
{ 0x09, 0x09, -1, 0x03, 0x0242, 0x0246 },
|
||||
{ 0x09, 0x0a, -1, 0x03, 0x0242, 0x0446 },
|
||||
{ 0x09, 0x18, -1, 0x03, 0x0242, 0x0846 },
|
||||
{ 0x0a, 0x05, -1, 0x03, 0x0442, 0x0046 },
|
||||
{ 0x0a, 0x05, -1, 0x03, 0x0442, 0x0066 },
|
||||
{ 0x0a, 0x06, -1, 0x03, 0x0442, 0x0086 },
|
||||
{ 0x0a, 0x07, -1, 0x03, 0x0442, 0x00c6 },
|
||||
{ 0x0a, 0x08, -1, 0x03, 0x0442, 0x0146 },
|
||||
{ 0x0a, 0x09, -1, 0x03, 0x0442, 0x0246 },
|
||||
{ 0x0a, 0x0a, -1, 0x03, 0x0442, 0x0446 },
|
||||
{ 0x0a, 0x18, -1, 0x03, 0x0442, 0x0846 },
|
||||
{ 0x0c, 0x05, -1, 0x03, 0x0842, 0x0046 },
|
||||
{ 0x0c, 0x05, -1, 0x03, 0x0842, 0x0066 },
|
||||
{ 0x0c, 0x06, -1, 0x03, 0x0842, 0x0086 },
|
||||
{ 0x0c, 0x07, -1, 0x03, 0x0842, 0x00c6 },
|
||||
{ 0x0c, 0x08, -1, 0x03, 0x0842, 0x0146 },
|
||||
{ 0x0c, 0x09, -1, 0x03, 0x0842, 0x0246 },
|
||||
{ 0x0c, 0x0a, -1, 0x03, 0x0842, 0x0446 },
|
||||
{ 0x0c, 0x18, -1, 0x03, 0x0842, 0x0846 },
|
||||
{ 0x0e, 0x05, -1, 0x03, 0x1842, 0x0046 },
|
||||
{ 0x0e, 0x05, -1, 0x03, 0x1842, 0x0066 },
|
||||
{ 0x0e, 0x06, -1, 0x03, 0x1842, 0x0086 },
|
||||
{ 0x0e, 0x07, -1, 0x03, 0x1842, 0x00c6 },
|
||||
{ 0x0e, 0x08, -1, 0x03, 0x1842, 0x0146 },
|
||||
{ 0x0e, 0x09, -1, 0x03, 0x1842, 0x0246 },
|
||||
{ 0x0e, 0x0a, -1, 0x03, 0x1842, 0x0446 },
|
||||
{ 0x0e, 0x18, -1, 0x03, 0x1842, 0x0846 },
|
||||
{ 0x18, 0x05, -1, 0x03, 0x5842, 0x0046 },
|
||||
{ 0x18, 0x05, -1, 0x03, 0x5842, 0x0066 },
|
||||
{ 0x18, 0x06, -1, 0x03, 0x5842, 0x0086 },
|
||||
{ 0x18, 0x07, -1, 0x03, 0x5842, 0x00c6 },
|
||||
{ 0x18, 0x08, -1, 0x03, 0x5842, 0x0146 },
|
||||
{ 0x18, 0x09, -1, 0x03, 0x5842, 0x0246 },
|
||||
{ 0x18, 0x0a, -1, 0x03, 0x5842, 0x0446 },
|
||||
{ 0x18, 0x18, -1, 0x03, 0x5842, 0x0846 },
|
||||
};
|
||||
|
||||
#endif /* BROTLI_DEC_PREFIX_H_ */
|
||||
|
|
@ -1,251 +0,0 @@
|
|||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Brotli state for partial streaming decoding. */
|
||||
|
||||
#ifndef BROTLI_DEC_STATE_H_
|
||||
#define BROTLI_DEC_STATE_H_
|
||||
|
||||
#include "../common/constants.h"
|
||||
#include "../common/dictionary.h"
|
||||
#include <brotli/types.h>
|
||||
#include "./bit_reader.h"
|
||||
#include "./huffman.h"
|
||||
#include "./port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef enum {
|
||||
BROTLI_STATE_UNINITED,
|
||||
BROTLI_STATE_METABLOCK_BEGIN,
|
||||
BROTLI_STATE_METABLOCK_HEADER,
|
||||
BROTLI_STATE_METABLOCK_HEADER_2,
|
||||
BROTLI_STATE_CONTEXT_MODES,
|
||||
BROTLI_STATE_COMMAND_BEGIN,
|
||||
BROTLI_STATE_COMMAND_INNER,
|
||||
BROTLI_STATE_COMMAND_POST_DECODE_LITERALS,
|
||||
BROTLI_STATE_COMMAND_POST_WRAP_COPY,
|
||||
BROTLI_STATE_UNCOMPRESSED,
|
||||
BROTLI_STATE_METADATA,
|
||||
BROTLI_STATE_COMMAND_INNER_WRITE,
|
||||
BROTLI_STATE_METABLOCK_DONE,
|
||||
BROTLI_STATE_COMMAND_POST_WRITE_1,
|
||||
BROTLI_STATE_COMMAND_POST_WRITE_2,
|
||||
BROTLI_STATE_HUFFMAN_CODE_0,
|
||||
BROTLI_STATE_HUFFMAN_CODE_1,
|
||||
BROTLI_STATE_HUFFMAN_CODE_2,
|
||||
BROTLI_STATE_HUFFMAN_CODE_3,
|
||||
BROTLI_STATE_CONTEXT_MAP_1,
|
||||
BROTLI_STATE_CONTEXT_MAP_2,
|
||||
BROTLI_STATE_TREE_GROUP,
|
||||
BROTLI_STATE_DONE
|
||||
} BrotliRunningState;
|
||||
|
||||
typedef enum {
|
||||
BROTLI_STATE_METABLOCK_HEADER_NONE,
|
||||
BROTLI_STATE_METABLOCK_HEADER_EMPTY,
|
||||
BROTLI_STATE_METABLOCK_HEADER_NIBBLES,
|
||||
BROTLI_STATE_METABLOCK_HEADER_SIZE,
|
||||
BROTLI_STATE_METABLOCK_HEADER_UNCOMPRESSED,
|
||||
BROTLI_STATE_METABLOCK_HEADER_RESERVED,
|
||||
BROTLI_STATE_METABLOCK_HEADER_BYTES,
|
||||
BROTLI_STATE_METABLOCK_HEADER_METADATA
|
||||
} BrotliRunningMetablockHeaderState;
|
||||
|
||||
typedef enum {
|
||||
BROTLI_STATE_UNCOMPRESSED_NONE,
|
||||
BROTLI_STATE_UNCOMPRESSED_WRITE
|
||||
} BrotliRunningUncompressedState;
|
||||
|
||||
typedef enum {
|
||||
BROTLI_STATE_TREE_GROUP_NONE,
|
||||
BROTLI_STATE_TREE_GROUP_LOOP
|
||||
} BrotliRunningTreeGroupState;
|
||||
|
||||
typedef enum {
|
||||
BROTLI_STATE_CONTEXT_MAP_NONE,
|
||||
BROTLI_STATE_CONTEXT_MAP_READ_PREFIX,
|
||||
BROTLI_STATE_CONTEXT_MAP_HUFFMAN,
|
||||
BROTLI_STATE_CONTEXT_MAP_DECODE,
|
||||
BROTLI_STATE_CONTEXT_MAP_TRANSFORM
|
||||
} BrotliRunningContextMapState;
|
||||
|
||||
typedef enum {
|
||||
BROTLI_STATE_HUFFMAN_NONE,
|
||||
BROTLI_STATE_HUFFMAN_SIMPLE_SIZE,
|
||||
BROTLI_STATE_HUFFMAN_SIMPLE_READ,
|
||||
BROTLI_STATE_HUFFMAN_SIMPLE_BUILD,
|
||||
BROTLI_STATE_HUFFMAN_COMPLEX,
|
||||
BROTLI_STATE_HUFFMAN_LENGTH_SYMBOLS
|
||||
} BrotliRunningHuffmanState;
|
||||
|
||||
typedef enum {
|
||||
BROTLI_STATE_DECODE_UINT8_NONE,
|
||||
BROTLI_STATE_DECODE_UINT8_SHORT,
|
||||
BROTLI_STATE_DECODE_UINT8_LONG
|
||||
} BrotliRunningDecodeUint8State;
|
||||
|
||||
typedef enum {
|
||||
BROTLI_STATE_READ_BLOCK_LENGTH_NONE,
|
||||
BROTLI_STATE_READ_BLOCK_LENGTH_SUFFIX
|
||||
} BrotliRunningReadBlockLengthState;
|
||||
|
||||
struct BrotliDecoderStateStruct {
|
||||
BrotliRunningState state;
|
||||
|
||||
/* This counter is reused for several disjoint loops. */
|
||||
int loop_counter;
|
||||
|
||||
BrotliBitReader br;
|
||||
|
||||
brotli_alloc_func alloc_func;
|
||||
brotli_free_func free_func;
|
||||
void* memory_manager_opaque;
|
||||
|
||||
/* Temporary storage for remaining input. */
|
||||
union {
|
||||
uint64_t u64;
|
||||
uint8_t u8[8];
|
||||
} buffer;
|
||||
uint32_t buffer_length;
|
||||
|
||||
int pos;
|
||||
int max_backward_distance;
|
||||
int max_distance;
|
||||
int ringbuffer_size;
|
||||
int ringbuffer_mask;
|
||||
int dist_rb_idx;
|
||||
int dist_rb[4];
|
||||
int error_code;
|
||||
uint32_t sub_loop_counter;
|
||||
uint8_t* ringbuffer;
|
||||
uint8_t* ringbuffer_end;
|
||||
HuffmanCode* htree_command;
|
||||
const uint8_t* context_lookup1;
|
||||
const uint8_t* context_lookup2;
|
||||
uint8_t* context_map_slice;
|
||||
uint8_t* dist_context_map_slice;
|
||||
|
||||
/* This ring buffer holds a few past copy distances that will be used by */
|
||||
/* some special distance codes. */
|
||||
HuffmanTreeGroup literal_hgroup;
|
||||
HuffmanTreeGroup insert_copy_hgroup;
|
||||
HuffmanTreeGroup distance_hgroup;
|
||||
HuffmanCode* block_type_trees;
|
||||
HuffmanCode* block_len_trees;
|
||||
/* This is true if the literal context map histogram type always matches the
|
||||
block type. It is then not needed to keep the context (faster decoding). */
|
||||
int trivial_literal_context;
|
||||
/* Distance context is actual after command is decoded and before distance
|
||||
is computed. After distance computation it is used as a temporary variable. */
|
||||
int distance_context;
|
||||
int meta_block_remaining_len;
|
||||
uint32_t block_length_index;
|
||||
uint32_t block_length[3];
|
||||
uint32_t num_block_types[3];
|
||||
uint32_t block_type_rb[6];
|
||||
uint32_t distance_postfix_bits;
|
||||
uint32_t num_direct_distance_codes;
|
||||
int distance_postfix_mask;
|
||||
uint32_t num_dist_htrees;
|
||||
uint8_t* dist_context_map;
|
||||
HuffmanCode* literal_htree;
|
||||
uint8_t dist_htree_index;
|
||||
uint32_t repeat_code_len;
|
||||
uint32_t prev_code_len;
|
||||
|
||||
int copy_length;
|
||||
int distance_code;
|
||||
|
||||
/* For partial write operations */
|
||||
size_t rb_roundtrips; /* How many times we went around the ring-buffer */
|
||||
size_t partial_pos_out; /* How much output to the user in total */
|
||||
|
||||
/* For ReadHuffmanCode */
|
||||
uint32_t symbol;
|
||||
uint32_t repeat;
|
||||
uint32_t space;
|
||||
|
||||
HuffmanCode table[32];
|
||||
/* List of heads of symbol chains. */
|
||||
uint16_t* symbol_lists;
|
||||
/* Storage from symbol_lists. */
|
||||
uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 +
|
||||
BROTLI_NUM_COMMAND_SYMBOLS];
|
||||
/* Tails of symbol chains. */
|
||||
int next_symbol[32];
|
||||
uint8_t code_length_code_lengths[BROTLI_CODE_LENGTH_CODES];
|
||||
/* Population counts for the code lengths */
|
||||
uint16_t code_length_histo[16];
|
||||
|
||||
/* For HuffmanTreeGroupDecode */
|
||||
int htree_index;
|
||||
HuffmanCode* next;
|
||||
|
||||
/* For DecodeContextMap */
|
||||
uint32_t context_index;
|
||||
uint32_t max_run_length_prefix;
|
||||
uint32_t code;
|
||||
HuffmanCode context_map_table[BROTLI_HUFFMAN_MAX_SIZE_272];
|
||||
|
||||
/* For InverseMoveToFrontTransform */
|
||||
uint32_t mtf_upper_bound;
|
||||
uint32_t mtf[64 + 1];
|
||||
|
||||
/* For custom dictionaries */
|
||||
const uint8_t* custom_dict;
|
||||
int custom_dict_size;
|
||||
|
||||
/* less used attributes are in the end of this struct */
|
||||
/* States inside function calls */
|
||||
BrotliRunningMetablockHeaderState substate_metablock_header;
|
||||
BrotliRunningTreeGroupState substate_tree_group;
|
||||
BrotliRunningContextMapState substate_context_map;
|
||||
BrotliRunningUncompressedState substate_uncompressed;
|
||||
BrotliRunningHuffmanState substate_huffman;
|
||||
BrotliRunningDecodeUint8State substate_decode_uint8;
|
||||
BrotliRunningReadBlockLengthState substate_read_block_length;
|
||||
|
||||
unsigned int is_last_metablock : 1;
|
||||
unsigned int is_uncompressed : 1;
|
||||
unsigned int is_metadata : 1;
|
||||
unsigned int should_wrap_ringbuffer : 1;
|
||||
unsigned int canny_ringbuffer_allocation : 1;
|
||||
unsigned int size_nibbles : 8;
|
||||
uint32_t window_bits;
|
||||
|
||||
int new_ringbuffer_size;
|
||||
|
||||
uint32_t num_literal_htrees;
|
||||
uint8_t* context_map;
|
||||
uint8_t* context_modes;
|
||||
const BrotliDictionary* dictionary;
|
||||
|
||||
uint32_t trivial_literal_contexts[8]; /* 256 bits */
|
||||
};
|
||||
|
||||
typedef struct BrotliDecoderStateStruct BrotliDecoderStateInternal;
|
||||
#define BrotliDecoderState BrotliDecoderStateInternal
|
||||
|
||||
BROTLI_INTERNAL void BrotliDecoderStateInit(BrotliDecoderState* s);
|
||||
BROTLI_INTERNAL void BrotliDecoderStateInitWithCustomAllocators(
|
||||
BrotliDecoderState* s, brotli_alloc_func alloc_func,
|
||||
brotli_free_func free_func, void* opaque);
|
||||
BROTLI_INTERNAL void BrotliDecoderStateCleanup(BrotliDecoderState* s);
|
||||
BROTLI_INTERNAL void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s);
|
||||
BROTLI_INTERNAL void BrotliDecoderStateCleanupAfterMetablock(
|
||||
BrotliDecoderState* s);
|
||||
BROTLI_INTERNAL BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(
|
||||
BrotliDecoderState* s, HuffmanTreeGroup* group, uint32_t alphabet_size,
|
||||
uint32_t ntrees);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_DEC_STATE_H_ */
|
||||
|
|
@ -1,300 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Transformations on dictionary words. */
|
||||
|
||||
#ifndef BROTLI_DEC_TRANSFORM_H_
|
||||
#define BROTLI_DEC_TRANSFORM_H_
|
||||
|
||||
#include <brotli/types.h>
|
||||
#include "./port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
enum WordTransformType {
|
||||
kIdentity = 0,
|
||||
kOmitLast1 = 1,
|
||||
kOmitLast2 = 2,
|
||||
kOmitLast3 = 3,
|
||||
kOmitLast4 = 4,
|
||||
kOmitLast5 = 5,
|
||||
kOmitLast6 = 6,
|
||||
kOmitLast7 = 7,
|
||||
kOmitLast8 = 8,
|
||||
kOmitLast9 = 9,
|
||||
kUppercaseFirst = 10,
|
||||
kUppercaseAll = 11,
|
||||
kOmitFirst1 = 12,
|
||||
kOmitFirst2 = 13,
|
||||
kOmitFirst3 = 14,
|
||||
kOmitFirst4 = 15,
|
||||
kOmitFirst5 = 16,
|
||||
kOmitFirst6 = 17,
|
||||
kOmitFirst7 = 18,
|
||||
kOmitFirst8 = 19,
|
||||
kOmitFirst9 = 20
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
const uint8_t prefix_id;
|
||||
const uint8_t transform;
|
||||
const uint8_t suffix_id;
|
||||
} Transform;
|
||||
|
||||
static const char kPrefixSuffix[208] =
|
||||
"\0 \0, \0 of the \0 of \0s \0.\0 and \0 in \0\"\0 to \0\">\0\n\0. \0]\0"
|
||||
" for \0 a \0 that \0\'\0 with \0 from \0 by \0(\0. The \0 on \0 as \0"
|
||||
" is \0ing \0\n\t\0:\0ed \0=\"\0 at \0ly \0,\0=\'\0.com/\0. This \0"
|
||||
" not \0er \0al \0ful \0ive \0less \0est \0ize \0\xc2\xa0\0ous ";
|
||||
|
||||
enum {
|
||||
/* EMPTY = ""
|
||||
SP = " "
|
||||
DQUOT = "\""
|
||||
SQUOT = "'"
|
||||
CLOSEBR = "]"
|
||||
OPEN = "("
|
||||
SLASH = "/"
|
||||
NBSP = non-breaking space "\0xc2\xa0"
|
||||
*/
|
||||
kPFix_EMPTY = 0,
|
||||
kPFix_SP = 1,
|
||||
kPFix_COMMASP = 3,
|
||||
kPFix_SPofSPtheSP = 6,
|
||||
kPFix_SPtheSP = 9,
|
||||
kPFix_eSP = 12,
|
||||
kPFix_SPofSP = 15,
|
||||
kPFix_sSP = 20,
|
||||
kPFix_DOT = 23,
|
||||
kPFix_SPandSP = 25,
|
||||
kPFix_SPinSP = 31,
|
||||
kPFix_DQUOT = 36,
|
||||
kPFix_SPtoSP = 38,
|
||||
kPFix_DQUOTGT = 43,
|
||||
kPFix_NEWLINE = 46,
|
||||
kPFix_DOTSP = 48,
|
||||
kPFix_CLOSEBR = 51,
|
||||
kPFix_SPforSP = 53,
|
||||
kPFix_SPaSP = 59,
|
||||
kPFix_SPthatSP = 63,
|
||||
kPFix_SQUOT = 70,
|
||||
kPFix_SPwithSP = 72,
|
||||
kPFix_SPfromSP = 79,
|
||||
kPFix_SPbySP = 86,
|
||||
kPFix_OPEN = 91,
|
||||
kPFix_DOTSPTheSP = 93,
|
||||
kPFix_SPonSP = 100,
|
||||
kPFix_SPasSP = 105,
|
||||
kPFix_SPisSP = 110,
|
||||
kPFix_ingSP = 115,
|
||||
kPFix_NEWLINETAB = 120,
|
||||
kPFix_COLON = 123,
|
||||
kPFix_edSP = 125,
|
||||
kPFix_EQDQUOT = 129,
|
||||
kPFix_SPatSP = 132,
|
||||
kPFix_lySP = 137,
|
||||
kPFix_COMMA = 141,
|
||||
kPFix_EQSQUOT = 143,
|
||||
kPFix_DOTcomSLASH = 146,
|
||||
kPFix_DOTSPThisSP = 152,
|
||||
kPFix_SPnotSP = 160,
|
||||
kPFix_erSP = 166,
|
||||
kPFix_alSP = 170,
|
||||
kPFix_fulSP = 174,
|
||||
kPFix_iveSP = 179,
|
||||
kPFix_lessSP = 184,
|
||||
kPFix_estSP = 190,
|
||||
kPFix_izeSP = 195,
|
||||
kPFix_NBSP = 200,
|
||||
kPFix_ousSP = 203
|
||||
};
|
||||
|
||||
static const Transform kTransforms[] = {
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SP },
|
||||
{ kPFix_SP, kIdentity, kPFix_SP },
|
||||
{ kPFix_EMPTY, kOmitFirst1, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_SP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPtheSP },
|
||||
{ kPFix_SP, kIdentity, kPFix_EMPTY },
|
||||
{ kPFix_sSP, kIdentity, kPFix_SP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPofSP },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPandSP },
|
||||
{ kPFix_EMPTY, kOmitFirst2, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kOmitLast1, kPFix_EMPTY },
|
||||
{ kPFix_COMMASP, kIdentity, kPFix_SP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_COMMASP },
|
||||
{ kPFix_SP, kUppercaseFirst, kPFix_SP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPinSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPtoSP },
|
||||
{ kPFix_eSP, kIdentity, kPFix_SP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_DQUOT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_DOT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_DQUOTGT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_NEWLINE },
|
||||
{ kPFix_EMPTY, kOmitLast3, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_CLOSEBR },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPforSP },
|
||||
{ kPFix_EMPTY, kOmitFirst3, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kOmitLast2, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPaSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPthatSP },
|
||||
{ kPFix_SP, kUppercaseFirst, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_DOTSP },
|
||||
{ kPFix_DOT, kIdentity, kPFix_EMPTY },
|
||||
{ kPFix_SP, kIdentity, kPFix_COMMASP },
|
||||
{ kPFix_EMPTY, kOmitFirst4, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPwithSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SQUOT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPfromSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPbySP },
|
||||
{ kPFix_EMPTY, kOmitFirst5, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kOmitFirst6, kPFix_EMPTY },
|
||||
{ kPFix_SPtheSP, kIdentity, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kOmitLast4, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_DOTSPTheSP },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPonSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPasSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPisSP },
|
||||
{ kPFix_EMPTY, kOmitLast7, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kOmitLast1, kPFix_ingSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_NEWLINETAB },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_COLON },
|
||||
{ kPFix_SP, kIdentity, kPFix_DOTSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_edSP },
|
||||
{ kPFix_EMPTY, kOmitFirst9, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kOmitFirst7, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kOmitLast6, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_OPEN },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_COMMASP },
|
||||
{ kPFix_EMPTY, kOmitLast8, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPatSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_lySP },
|
||||
{ kPFix_SPtheSP, kIdentity, kPFix_SPofSP },
|
||||
{ kPFix_EMPTY, kOmitLast5, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kOmitLast9, kPFix_EMPTY },
|
||||
{ kPFix_SP, kUppercaseFirst, kPFix_COMMASP },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_DQUOT },
|
||||
{ kPFix_DOT, kIdentity, kPFix_OPEN },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_SP },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_DQUOTGT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_EQDQUOT },
|
||||
{ kPFix_SP, kIdentity, kPFix_DOT },
|
||||
{ kPFix_DOTcomSLASH, kIdentity, kPFix_EMPTY },
|
||||
{ kPFix_SPtheSP, kIdentity, kPFix_SPofSPtheSP },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_SQUOT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_DOTSPThisSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_COMMA },
|
||||
{ kPFix_DOT, kIdentity, kPFix_SP },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_OPEN },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_DOT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_SPnotSP },
|
||||
{ kPFix_SP, kIdentity, kPFix_EQDQUOT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_erSP },
|
||||
{ kPFix_SP, kUppercaseAll, kPFix_SP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_alSP },
|
||||
{ kPFix_SP, kUppercaseAll, kPFix_EMPTY },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_EQSQUOT },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_DQUOT },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_DOTSP },
|
||||
{ kPFix_SP, kIdentity, kPFix_OPEN },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_fulSP },
|
||||
{ kPFix_SP, kUppercaseFirst, kPFix_DOTSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_iveSP },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_lessSP },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_SQUOT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_estSP },
|
||||
{ kPFix_SP, kUppercaseFirst, kPFix_DOT },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_DQUOTGT },
|
||||
{ kPFix_SP, kIdentity, kPFix_EQSQUOT },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_COMMA },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_izeSP },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_DOT },
|
||||
{ kPFix_NBSP, kIdentity, kPFix_EMPTY },
|
||||
{ kPFix_SP, kIdentity, kPFix_COMMA },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_EQDQUOT },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_EQDQUOT },
|
||||
{ kPFix_EMPTY, kIdentity, kPFix_ousSP },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_COMMASP },
|
||||
{ kPFix_EMPTY, kUppercaseFirst, kPFix_EQSQUOT },
|
||||
{ kPFix_SP, kUppercaseFirst, kPFix_COMMA },
|
||||
{ kPFix_SP, kUppercaseAll, kPFix_EQDQUOT },
|
||||
{ kPFix_SP, kUppercaseAll, kPFix_COMMASP },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_COMMA },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_OPEN },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_DOTSP },
|
||||
{ kPFix_SP, kUppercaseAll, kPFix_DOT },
|
||||
{ kPFix_EMPTY, kUppercaseAll, kPFix_EQSQUOT },
|
||||
{ kPFix_SP, kUppercaseAll, kPFix_DOTSP },
|
||||
{ kPFix_SP, kUppercaseFirst, kPFix_EQDQUOT },
|
||||
{ kPFix_SP, kUppercaseAll, kPFix_EQSQUOT },
|
||||
{ kPFix_SP, kUppercaseFirst, kPFix_EQSQUOT },
|
||||
};
|
||||
|
||||
static const int kNumTransforms = sizeof(kTransforms) / sizeof(kTransforms[0]);
|
||||
|
||||
static int ToUpperCase(uint8_t* p) {
|
||||
if (p[0] < 0xc0) {
|
||||
if (p[0] >= 'a' && p[0] <= 'z') {
|
||||
p[0] ^= 32;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
/* An overly simplified uppercasing model for UTF-8. */
|
||||
if (p[0] < 0xe0) {
|
||||
p[1] ^= 32;
|
||||
return 2;
|
||||
}
|
||||
/* An arbitrary transform for three byte characters. */
|
||||
p[2] ^= 5;
|
||||
return 3;
|
||||
}
|
||||
|
||||
static BROTLI_NOINLINE int TransformDictionaryWord(
|
||||
uint8_t* dst, const uint8_t* word, int len, int transform) {
|
||||
int idx = 0;
|
||||
{
|
||||
const char* prefix = &kPrefixSuffix[kTransforms[transform].prefix_id];
|
||||
while (*prefix) { dst[idx++] = (uint8_t)*prefix++; }
|
||||
}
|
||||
{
|
||||
const int t = kTransforms[transform].transform;
|
||||
int i = 0;
|
||||
int skip = t - (kOmitFirst1 - 1);
|
||||
if (skip > 0) {
|
||||
word += skip;
|
||||
len -= skip;
|
||||
} else if (t <= kOmitLast9) {
|
||||
len -= t;
|
||||
}
|
||||
while (i < len) { dst[idx++] = word[i++]; }
|
||||
if (t == kUppercaseFirst) {
|
||||
ToUpperCase(&dst[idx - len]);
|
||||
} else if (t == kUppercaseAll) {
|
||||
uint8_t* uppercase = &dst[idx - len];
|
||||
while (len > 0) {
|
||||
int step = ToUpperCase(uppercase);
|
||||
uppercase += step;
|
||||
len -= step;
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
const char* suffix = &kPrefixSuffix[kTransforms[transform].suffix_id];
|
||||
while (*suffix) { dst[idx++] = (uint8_t)*suffix++; }
|
||||
return idx;
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_DEC_TRANSFORM_H_ */
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,39 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function to find backward reference copies. */
|
||||
|
||||
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_
|
||||
#define BROTLI_ENC_BACKWARD_REFERENCES_H_
|
||||
|
||||
#include "../common/constants.h"
|
||||
#include "../common/dictionary.h"
|
||||
#include <brotli/types.h>
|
||||
#include "./command.h"
|
||||
#include "./hash.h"
|
||||
#include "./port.h"
|
||||
#include "./quality.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* "commands" points to the next output command to write to, "*num_commands" is
|
||||
initially the total amount of commands output by previous
|
||||
CreateBackwardReferences calls, and must be incremented by the amount written
|
||||
by this call. */
|
||||
BROTLI_INTERNAL void BrotliCreateBackwardReferences(
|
||||
const BrotliDictionary* dictionary, size_t num_bytes, size_t position,
|
||||
const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
|
||||
size_t* last_insert_len, Command* commands, size_t* num_commands,
|
||||
size_t* num_literals);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_ENC_BACKWARD_REFERENCES_H_ */
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function to find backward reference copies. */
|
||||
|
||||
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
|
||||
#define BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
|
||||
|
||||
#include "../common/constants.h"
|
||||
#include "../common/dictionary.h"
|
||||
#include <brotli/types.h>
|
||||
#include "./command.h"
|
||||
#include "./hash.h"
|
||||
#include "./memory.h"
|
||||
#include "./port.h"
|
||||
#include "./quality.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(
|
||||
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
|
||||
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
|
||||
size_t* last_insert_len, Command* commands, size_t* num_commands,
|
||||
size_t* num_literals);
|
||||
|
||||
BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(
|
||||
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
|
||||
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
|
||||
size_t* last_insert_len, Command* commands, size_t* num_commands,
|
||||
size_t* num_literals);
|
||||
|
||||
typedef struct ZopfliNode {
|
||||
/* best length to get up to this byte (not including this byte itself)
|
||||
highest 8 bit is used to reconstruct the length code */
|
||||
uint32_t length;
|
||||
/* distance associated with the length
|
||||
highest 7 bit contains distance short code + 1 (or zero if no short code)
|
||||
*/
|
||||
uint32_t distance;
|
||||
/* number of literal inserts before this copy */
|
||||
uint32_t insert_length;
|
||||
|
||||
/* This union holds information used by dynamic-programming. During forward
|
||||
pass |cost| it used to store the goal function. When node is processed its
|
||||
|cost| is invalidated in favor of |shortcut|. On path back-tracing pass
|
||||
|next| is assigned the offset to next node on the path. */
|
||||
union {
|
||||
/* Smallest cost to get to this byte from the beginning, as found so far. */
|
||||
float cost;
|
||||
/* Offset to the next node on the path. Equals to command_length() of the
|
||||
next node on the path. For last node equals to BROTLI_UINT32_MAX */
|
||||
uint32_t next;
|
||||
/* Node position that provides next distance for distance cache. */
|
||||
uint32_t shortcut;
|
||||
} u;
|
||||
} ZopfliNode;
|
||||
|
||||
BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
|
||||
|
||||
/* Computes the shortest path of commands from position to at most
|
||||
position + num_bytes.
|
||||
|
||||
On return, path->size() is the number of commands found and path[i] is the
|
||||
length of the i-th command (copy length plus insert length).
|
||||
Note that the sum of the lengths of all commands can be less than num_bytes.
|
||||
|
||||
On return, the nodes[0..num_bytes] array will have the following
|
||||
"ZopfliNode array invariant":
|
||||
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
|
||||
(1) nodes[i].copy_length() >= 2
|
||||
(2) nodes[i].command_length() <= i and
|
||||
(3) nodes[i - nodes[i].command_length()].cost < kInfinity */
|
||||
BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
|
||||
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
|
||||
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params, const size_t max_backward_limit,
|
||||
const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes);
|
||||
|
||||
BROTLI_INTERNAL void BrotliZopfliCreateCommands(const size_t num_bytes,
|
||||
const size_t block_start,
|
||||
const size_t max_backward_limit,
|
||||
const ZopfliNode* nodes,
|
||||
int* dist_cache,
|
||||
size_t* last_insert_len,
|
||||
Command* commands,
|
||||
size_t* num_literals);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_ */
|
||||
|
|
@ -1,143 +0,0 @@
|
|||
/* NOLINT(build/header_guard) */
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* template parameters: FN */
|
||||
|
||||
static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
|
||||
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
|
||||
size_t num_bytes, size_t position,
|
||||
const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
||||
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
|
||||
size_t* last_insert_len, Command* commands, size_t* num_commands,
|
||||
size_t* num_literals) {
|
||||
/* Set maximum distance, see section 9.1. of the spec. */
|
||||
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
|
||||
|
||||
const Command* const orig_commands = commands;
|
||||
size_t insert_length = *last_insert_len;
|
||||
const size_t pos_end = position + num_bytes;
|
||||
const size_t store_end = num_bytes >= FN(StoreLookahead)() ?
|
||||
position + num_bytes - FN(StoreLookahead)() + 1 : position;
|
||||
|
||||
/* For speed up heuristics for random data. */
|
||||
const size_t random_heuristics_window_size =
|
||||
LiteralSpreeLengthForSparseSearch(params);
|
||||
size_t apply_random_heuristics = position + random_heuristics_window_size;
|
||||
|
||||
/* Minimum score to accept a backward reference. */
|
||||
const score_t kMinScore = BROTLI_SCORE_BASE + 100;
|
||||
|
||||
FN(PrepareDistanceCache)(hasher, dist_cache);
|
||||
|
||||
while (position + FN(HashTypeLength)() < pos_end) {
|
||||
size_t max_length = pos_end - position;
|
||||
size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
|
||||
HasherSearchResult sr;
|
||||
sr.len = 0;
|
||||
sr.len_code_delta = 0;
|
||||
sr.distance = 0;
|
||||
sr.score = kMinScore;
|
||||
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
|
||||
ringbuffer_mask, dist_cache, position,
|
||||
max_length, max_distance, &sr);
|
||||
if (sr.score > kMinScore) {
|
||||
/* Found a match. Let's look for something even better ahead. */
|
||||
int delayed_backward_references_in_row = 0;
|
||||
--max_length;
|
||||
for (;; --max_length) {
|
||||
const score_t cost_diff_lazy = 175;
|
||||
HasherSearchResult sr2;
|
||||
sr2.len = params->quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH ?
|
||||
BROTLI_MIN(size_t, sr.len - 1, max_length) : 0;
|
||||
sr2.len_code_delta = 0;
|
||||
sr2.distance = 0;
|
||||
sr2.score = kMinScore;
|
||||
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
|
||||
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
|
||||
ringbuffer_mask, dist_cache, position + 1,
|
||||
max_length, max_distance, &sr2);
|
||||
if (sr2.score >= sr.score + cost_diff_lazy) {
|
||||
/* Ok, let's just write one byte for now and start a match from the
|
||||
next byte. */
|
||||
++position;
|
||||
++insert_length;
|
||||
sr = sr2;
|
||||
if (++delayed_backward_references_in_row < 4 &&
|
||||
position + FN(HashTypeLength)() < pos_end) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
apply_random_heuristics =
|
||||
position + 2 * sr.len + random_heuristics_window_size;
|
||||
max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
|
||||
{
|
||||
/* The first 16 codes are special short-codes,
|
||||
and the minimum offset is 1. */
|
||||
size_t distance_code =
|
||||
ComputeDistanceCode(sr.distance, max_distance, dist_cache);
|
||||
if (sr.distance <= max_distance && distance_code > 0) {
|
||||
dist_cache[3] = dist_cache[2];
|
||||
dist_cache[2] = dist_cache[1];
|
||||
dist_cache[1] = dist_cache[0];
|
||||
dist_cache[0] = (int)sr.distance;
|
||||
FN(PrepareDistanceCache)(hasher, dist_cache);
|
||||
}
|
||||
InitCommand(commands++, insert_length, sr.len, sr.len_code_delta,
|
||||
distance_code);
|
||||
}
|
||||
*num_literals += insert_length;
|
||||
insert_length = 0;
|
||||
/* Put the hash keys into the table, if there are enough bytes left.
|
||||
Depending on the hasher implementation, it can push all positions
|
||||
in the given range or only a subset of them. */
|
||||
FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, position + 2,
|
||||
BROTLI_MIN(size_t, position + sr.len, store_end));
|
||||
position += sr.len;
|
||||
} else {
|
||||
++insert_length;
|
||||
++position;
|
||||
/* If we have not seen matches for a long time, we can skip some
|
||||
match lookups. Unsuccessful match lookups are very very expensive
|
||||
and this kind of a heuristic speeds up compression quite
|
||||
a lot. */
|
||||
if (position > apply_random_heuristics) {
|
||||
/* Going through uncompressible data, jump. */
|
||||
if (position >
|
||||
apply_random_heuristics + 4 * random_heuristics_window_size) {
|
||||
/* It is quite a long time since we saw a copy, so we assume
|
||||
that this data is not compressible, and store hashes less
|
||||
often. Hashes of non compressible data are less likely to
|
||||
turn out to be useful in the future, too, so we store less of
|
||||
them to not to flood out the hash table of good compressible
|
||||
data. */
|
||||
const size_t kMargin =
|
||||
BROTLI_MAX(size_t, FN(StoreLookahead)() - 1, 4);
|
||||
size_t pos_jump =
|
||||
BROTLI_MIN(size_t, position + 16, pos_end - kMargin);
|
||||
for (; position < pos_jump; position += 4) {
|
||||
FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
|
||||
insert_length += 4;
|
||||
}
|
||||
} else {
|
||||
const size_t kMargin =
|
||||
BROTLI_MAX(size_t, FN(StoreLookahead)() - 1, 2);
|
||||
size_t pos_jump =
|
||||
BROTLI_MIN(size_t, position + 8, pos_end - kMargin);
|
||||
for (; position < pos_jump; position += 2) {
|
||||
FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
|
||||
insert_length += 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
insert_length += pos_end - position;
|
||||
*last_insert_len = insert_length;
|
||||
*num_commands += (size_t)(commands - orig_commands);
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Functions to estimate the bit cost of Huffman trees. */
|
||||
|
||||
#ifndef BROTLI_ENC_BIT_COST_H_
|
||||
#define BROTLI_ENC_BIT_COST_H_
|
||||
|
||||
#include <brotli/types.h>
|
||||
#include "./fast_log.h"
|
||||
#include "./histogram.h"
|
||||
#include "./port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static BROTLI_INLINE double ShannonEntropy(const uint32_t *population,
|
||||
size_t size, size_t *total) {
|
||||
size_t sum = 0;
|
||||
double retval = 0;
|
||||
const uint32_t *population_end = population + size;
|
||||
size_t p;
|
||||
if (size & 1) {
|
||||
goto odd_number_of_elements_left;
|
||||
}
|
||||
while (population < population_end) {
|
||||
p = *population++;
|
||||
sum += p;
|
||||
retval -= (double)p * FastLog2(p);
|
||||
odd_number_of_elements_left:
|
||||
p = *population++;
|
||||
sum += p;
|
||||
retval -= (double)p * FastLog2(p);
|
||||
}
|
||||
if (sum) retval += (double)sum * FastLog2(sum);
|
||||
*total = sum;
|
||||
return retval;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE double BitsEntropy(
|
||||
const uint32_t *population, size_t size) {
|
||||
size_t sum;
|
||||
double retval = ShannonEntropy(population, size, &sum);
|
||||
if (retval < sum) {
|
||||
/* At least one bit per literal is needed. */
|
||||
retval = (double)sum;
|
||||
}
|
||||
return retval;
|
||||
}
|
||||
|
||||
BROTLI_INTERNAL double BrotliPopulationCostLiteral(const HistogramLiteral*);
|
||||
BROTLI_INTERNAL double BrotliPopulationCostCommand(const HistogramCommand*);
|
||||
BROTLI_INTERNAL double BrotliPopulationCostDistance(const HistogramDistance*);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_ENC_BIT_COST_H_ */
|
||||
|
|
@ -1,127 +0,0 @@
|
|||
/* NOLINT(build/header_guard) */
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* template parameters: FN */
|
||||
|
||||
#define HistogramType FN(Histogram)
|
||||
|
||||
double FN(BrotliPopulationCost)(const HistogramType* histogram) {
|
||||
static const double kOneSymbolHistogramCost = 12;
|
||||
static const double kTwoSymbolHistogramCost = 20;
|
||||
static const double kThreeSymbolHistogramCost = 28;
|
||||
static const double kFourSymbolHistogramCost = 37;
|
||||
const size_t data_size = FN(HistogramDataSize)();
|
||||
int count = 0;
|
||||
size_t s[5];
|
||||
double bits = 0.0;
|
||||
size_t i;
|
||||
if (histogram->total_count_ == 0) {
|
||||
return kOneSymbolHistogramCost;
|
||||
}
|
||||
for (i = 0; i < data_size; ++i) {
|
||||
if (histogram->data_[i] > 0) {
|
||||
s[count] = i;
|
||||
++count;
|
||||
if (count > 4) break;
|
||||
}
|
||||
}
|
||||
if (count == 1) {
|
||||
return kOneSymbolHistogramCost;
|
||||
}
|
||||
if (count == 2) {
|
||||
return (kTwoSymbolHistogramCost + (double)histogram->total_count_);
|
||||
}
|
||||
if (count == 3) {
|
||||
const uint32_t histo0 = histogram->data_[s[0]];
|
||||
const uint32_t histo1 = histogram->data_[s[1]];
|
||||
const uint32_t histo2 = histogram->data_[s[2]];
|
||||
const uint32_t histomax =
|
||||
BROTLI_MAX(uint32_t, histo0, BROTLI_MAX(uint32_t, histo1, histo2));
|
||||
return (kThreeSymbolHistogramCost +
|
||||
2 * (histo0 + histo1 + histo2) - histomax);
|
||||
}
|
||||
if (count == 4) {
|
||||
uint32_t histo[4];
|
||||
uint32_t h23;
|
||||
uint32_t histomax;
|
||||
for (i = 0; i < 4; ++i) {
|
||||
histo[i] = histogram->data_[s[i]];
|
||||
}
|
||||
/* Sort */
|
||||
for (i = 0; i < 4; ++i) {
|
||||
size_t j;
|
||||
for (j = i + 1; j < 4; ++j) {
|
||||
if (histo[j] > histo[i]) {
|
||||
BROTLI_SWAP(uint32_t, histo, j, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
h23 = histo[2] + histo[3];
|
||||
histomax = BROTLI_MAX(uint32_t, h23, histo[0]);
|
||||
return (kFourSymbolHistogramCost +
|
||||
3 * h23 + 2 * (histo[0] + histo[1]) - histomax);
|
||||
}
|
||||
|
||||
{
|
||||
/* In this loop we compute the entropy of the histogram and simultaneously
|
||||
build a simplified histogram of the code length codes where we use the
|
||||
zero repeat code 17, but we don't use the non-zero repeat code 16. */
|
||||
size_t max_depth = 1;
|
||||
uint32_t depth_histo[BROTLI_CODE_LENGTH_CODES] = { 0 };
|
||||
const double log2total = FastLog2(histogram->total_count_);
|
||||
for (i = 0; i < data_size;) {
|
||||
if (histogram->data_[i] > 0) {
|
||||
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
|
||||
= log2(total_count) - log2(count(symbol)) */
|
||||
double log2p = log2total - FastLog2(histogram->data_[i]);
|
||||
/* Approximate the bit depth by round(-log2(P(symbol))) */
|
||||
size_t depth = (size_t)(log2p + 0.5);
|
||||
bits += histogram->data_[i] * log2p;
|
||||
if (depth > 15) {
|
||||
depth = 15;
|
||||
}
|
||||
if (depth > max_depth) {
|
||||
max_depth = depth;
|
||||
}
|
||||
++depth_histo[depth];
|
||||
++i;
|
||||
} else {
|
||||
/* Compute the run length of zeros and add the appropriate number of 0
|
||||
and 17 code length codes to the code length code histogram. */
|
||||
uint32_t reps = 1;
|
||||
size_t k;
|
||||
for (k = i + 1; k < data_size && histogram->data_[k] == 0; ++k) {
|
||||
++reps;
|
||||
}
|
||||
i += reps;
|
||||
if (i == data_size) {
|
||||
/* Don't add any cost for the last zero run, since these are encoded
|
||||
only implicitly. */
|
||||
break;
|
||||
}
|
||||
if (reps < 3) {
|
||||
depth_histo[0] += reps;
|
||||
} else {
|
||||
reps -= 2;
|
||||
while (reps > 0) {
|
||||
++depth_histo[BROTLI_REPEAT_ZERO_CODE_LENGTH];
|
||||
/* Add the 3 extra bits for the 17 code length code. */
|
||||
bits += 3;
|
||||
reps >>= 3;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/* Add the estimated encoding cost of the code length code histogram. */
|
||||
bits += (double)(18 + 2 * max_depth);
|
||||
/* Add the entropy of the code length code histogram. */
|
||||
bits += BitsEntropy(depth_histo, BROTLI_CODE_LENGTH_CODES);
|
||||
}
|
||||
return bits;
|
||||
}
|
||||
|
||||
#undef HistogramType
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
/* NOLINT(build/header_guard) */
|
||||
/* Copyright 2014 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* template parameters: FN */
|
||||
|
||||
#define HistogramType FN(Histogram)
|
||||
|
||||
/* Creates entropy codes for all block types and stores them to the bit
|
||||
stream. */
|
||||
static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self,
|
||||
const HistogramType* histograms, const size_t histograms_size,
|
||||
HuffmanTree* tree, size_t* storage_ix, uint8_t* storage) {
|
||||
const size_t alphabet_size = self->alphabet_size_;
|
||||
const size_t table_size = histograms_size * alphabet_size;
|
||||
self->depths_ = BROTLI_ALLOC(m, uint8_t, table_size);
|
||||
self->bits_ = BROTLI_ALLOC(m, uint16_t, table_size);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
|
||||
{
|
||||
size_t i;
|
||||
for (i = 0; i < histograms_size; ++i) {
|
||||
size_t ix = i * alphabet_size;
|
||||
BuildAndStoreHuffmanTree(&histograms[i].data_[0], alphabet_size, tree,
|
||||
&self->depths_[ix], &self->bits_[ix], storage_ix, storage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#undef HistogramType
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Block split point selection utilities. */
|
||||
|
||||
#ifndef BROTLI_ENC_BLOCK_SPLITTER_H_
|
||||
#define BROTLI_ENC_BLOCK_SPLITTER_H_
|
||||
|
||||
#include <brotli/types.h>
|
||||
#include "./command.h"
|
||||
#include "./memory.h"
|
||||
#include "./port.h"
|
||||
#include "./quality.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct BlockSplit {
|
||||
size_t num_types; /* Amount of distinct types */
|
||||
size_t num_blocks; /* Amount of values in types and length */
|
||||
uint8_t* types;
|
||||
uint32_t* lengths;
|
||||
|
||||
size_t types_alloc_size;
|
||||
size_t lengths_alloc_size;
|
||||
} BlockSplit;
|
||||
|
||||
BROTLI_INTERNAL void BrotliInitBlockSplit(BlockSplit* self);
|
||||
BROTLI_INTERNAL void BrotliDestroyBlockSplit(MemoryManager* m,
|
||||
BlockSplit* self);
|
||||
|
||||
BROTLI_INTERNAL void BrotliSplitBlock(MemoryManager* m,
|
||||
const Command* cmds,
|
||||
const size_t num_commands,
|
||||
const uint8_t* data,
|
||||
const size_t offset,
|
||||
const size_t mask,
|
||||
const BrotliEncoderParams* params,
|
||||
BlockSplit* literal_split,
|
||||
BlockSplit* insert_and_copy_split,
|
||||
BlockSplit* dist_split);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_ENC_BLOCK_SPLITTER_H_ */
|
||||
|
|
@ -1,432 +0,0 @@
|
|||
/* NOLINT(build/header_guard) */
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* template parameters: FN, DataType */
|
||||
|
||||
#define HistogramType FN(Histogram)
|
||||
|
||||
static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
|
||||
size_t stride,
|
||||
size_t num_histograms,
|
||||
HistogramType* histograms) {
|
||||
unsigned int seed = 7;
|
||||
size_t block_length = length / num_histograms;
|
||||
size_t i;
|
||||
FN(ClearHistograms)(histograms, num_histograms);
|
||||
for (i = 0; i < num_histograms; ++i) {
|
||||
size_t pos = length * i / num_histograms;
|
||||
if (i != 0) {
|
||||
pos += MyRand(&seed) % block_length;
|
||||
}
|
||||
if (pos + stride >= length) {
|
||||
pos = length - stride - 1;
|
||||
}
|
||||
FN(HistogramAddVector)(&histograms[i], data + pos, stride);
|
||||
}
|
||||
}
|
||||
|
||||
static void FN(RandomSample)(unsigned int* seed,
|
||||
const DataType* data,
|
||||
size_t length,
|
||||
size_t stride,
|
||||
HistogramType* sample) {
|
||||
size_t pos = 0;
|
||||
if (stride >= length) {
|
||||
pos = 0;
|
||||
stride = length;
|
||||
} else {
|
||||
pos = MyRand(seed) % (length - stride + 1);
|
||||
}
|
||||
FN(HistogramAddVector)(sample, data + pos, stride);
|
||||
}
|
||||
|
||||
static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
|
||||
size_t stride,
|
||||
size_t num_histograms,
|
||||
HistogramType* histograms) {
|
||||
size_t iters =
|
||||
kIterMulForRefining * length / stride + kMinItersForRefining;
|
||||
unsigned int seed = 7;
|
||||
size_t iter;
|
||||
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms;
|
||||
for (iter = 0; iter < iters; ++iter) {
|
||||
HistogramType sample;
|
||||
FN(HistogramClear)(&sample);
|
||||
FN(RandomSample)(&seed, data, length, stride, &sample);
|
||||
FN(HistogramAddHistogram)(&histograms[iter % num_histograms], &sample);
|
||||
}
|
||||
}
|
||||
|
||||
/* Assigns a block id from the range [0, num_histograms) to each data element
|
||||
in data[0..length) and fills in block_id[0..length) with the assigned values.
|
||||
Returns the number of blocks, i.e. one plus the number of block switches. */
|
||||
static size_t FN(FindBlocks)(const DataType* data, const size_t length,
|
||||
const double block_switch_bitcost,
|
||||
const size_t num_histograms,
|
||||
const HistogramType* histograms,
|
||||
double* insert_cost,
|
||||
double* cost,
|
||||
uint8_t* switch_signal,
|
||||
uint8_t *block_id) {
|
||||
const size_t data_size = FN(HistogramDataSize)();
|
||||
const size_t bitmaplen = (num_histograms + 7) >> 3;
|
||||
size_t num_blocks = 1;
|
||||
size_t i;
|
||||
size_t j;
|
||||
assert(num_histograms <= 256);
|
||||
if (num_histograms <= 1) {
|
||||
for (i = 0; i < length; ++i) {
|
||||
block_id[i] = 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
memset(insert_cost, 0, sizeof(insert_cost[0]) * data_size * num_histograms);
|
||||
for (i = 0; i < num_histograms; ++i) {
|
||||
insert_cost[i] = FastLog2((uint32_t)histograms[i].total_count_);
|
||||
}
|
||||
for (i = data_size; i != 0;) {
|
||||
--i;
|
||||
for (j = 0; j < num_histograms; ++j) {
|
||||
insert_cost[i * num_histograms + j] =
|
||||
insert_cost[j] - BitCost(histograms[j].data_[i]);
|
||||
}
|
||||
}
|
||||
memset(cost, 0, sizeof(cost[0]) * num_histograms);
|
||||
memset(switch_signal, 0, sizeof(switch_signal[0]) * length * bitmaplen);
|
||||
/* After each iteration of this loop, cost[k] will contain the difference
|
||||
between the minimum cost of arriving at the current byte position using
|
||||
entropy code k, and the minimum cost of arriving at the current byte
|
||||
position. This difference is capped at the block switch cost, and if it
|
||||
reaches block switch cost, it means that when we trace back from the last
|
||||
position, we need to switch here. */
|
||||
for (i = 0; i < length; ++i) {
|
||||
const size_t byte_ix = i;
|
||||
size_t ix = byte_ix * bitmaplen;
|
||||
size_t insert_cost_ix = data[byte_ix] * num_histograms;
|
||||
double min_cost = 1e99;
|
||||
double block_switch_cost = block_switch_bitcost;
|
||||
size_t k;
|
||||
for (k = 0; k < num_histograms; ++k) {
|
||||
/* We are coding the symbol in data[byte_ix] with entropy code k. */
|
||||
cost[k] += insert_cost[insert_cost_ix + k];
|
||||
if (cost[k] < min_cost) {
|
||||
min_cost = cost[k];
|
||||
block_id[byte_ix] = (uint8_t)k;
|
||||
}
|
||||
}
|
||||
/* More blocks for the beginning. */
|
||||
if (byte_ix < 2000) {
|
||||
block_switch_cost *= 0.77 + 0.07 * (double)byte_ix / 2000;
|
||||
}
|
||||
for (k = 0; k < num_histograms; ++k) {
|
||||
cost[k] -= min_cost;
|
||||
if (cost[k] >= block_switch_cost) {
|
||||
const uint8_t mask = (uint8_t)(1u << (k & 7));
|
||||
cost[k] = block_switch_cost;
|
||||
assert((k >> 3) < bitmaplen);
|
||||
switch_signal[ix + (k >> 3)] |= mask;
|
||||
}
|
||||
}
|
||||
}
|
||||
{ /* Trace back from the last position and switch at the marked places. */
|
||||
size_t byte_ix = length - 1;
|
||||
size_t ix = byte_ix * bitmaplen;
|
||||
uint8_t cur_id = block_id[byte_ix];
|
||||
while (byte_ix > 0) {
|
||||
const uint8_t mask = (uint8_t)(1u << (cur_id & 7));
|
||||
assert(((size_t)cur_id >> 3) < bitmaplen);
|
||||
--byte_ix;
|
||||
ix -= bitmaplen;
|
||||
if (switch_signal[ix + (cur_id >> 3)] & mask) {
|
||||
if (cur_id != block_id[byte_ix]) {
|
||||
cur_id = block_id[byte_ix];
|
||||
++num_blocks;
|
||||
}
|
||||
}
|
||||
block_id[byte_ix] = cur_id;
|
||||
}
|
||||
}
|
||||
return num_blocks;
|
||||
}
|
||||
|
||||
static size_t FN(RemapBlockIds)(uint8_t* block_ids, const size_t length,
|
||||
uint16_t* new_id, const size_t num_histograms) {
|
||||
static const uint16_t kInvalidId = 256;
|
||||
uint16_t next_id = 0;
|
||||
size_t i;
|
||||
for (i = 0; i < num_histograms; ++i) {
|
||||
new_id[i] = kInvalidId;
|
||||
}
|
||||
for (i = 0; i < length; ++i) {
|
||||
assert(block_ids[i] < num_histograms);
|
||||
if (new_id[block_ids[i]] == kInvalidId) {
|
||||
new_id[block_ids[i]] = next_id++;
|
||||
}
|
||||
}
|
||||
for (i = 0; i < length; ++i) {
|
||||
block_ids[i] = (uint8_t)new_id[block_ids[i]];
|
||||
assert(block_ids[i] < num_histograms);
|
||||
}
|
||||
assert(next_id <= num_histograms);
|
||||
return next_id;
|
||||
}
|
||||
|
||||
static void FN(BuildBlockHistograms)(const DataType* data, const size_t length,
|
||||
const uint8_t* block_ids,
|
||||
const size_t num_histograms,
|
||||
HistogramType* histograms) {
|
||||
size_t i;
|
||||
FN(ClearHistograms)(histograms, num_histograms);
|
||||
for (i = 0; i < length; ++i) {
|
||||
FN(HistogramAdd)(&histograms[block_ids[i]], data[i]);
|
||||
}
|
||||
}
|
||||
|
||||
static void FN(ClusterBlocks)(MemoryManager* m,
|
||||
const DataType* data, const size_t length,
|
||||
const size_t num_blocks,
|
||||
uint8_t* block_ids,
|
||||
BlockSplit* split) {
|
||||
uint32_t* histogram_symbols = BROTLI_ALLOC(m, uint32_t, num_blocks);
|
||||
uint32_t* block_lengths = BROTLI_ALLOC(m, uint32_t, num_blocks);
|
||||
const size_t expected_num_clusters = CLUSTERS_PER_BATCH *
|
||||
(num_blocks + HISTOGRAMS_PER_BATCH - 1) / HISTOGRAMS_PER_BATCH;
|
||||
size_t all_histograms_size = 0;
|
||||
size_t all_histograms_capacity = expected_num_clusters;
|
||||
HistogramType* all_histograms =
|
||||
BROTLI_ALLOC(m, HistogramType, all_histograms_capacity);
|
||||
size_t cluster_size_size = 0;
|
||||
size_t cluster_size_capacity = expected_num_clusters;
|
||||
uint32_t* cluster_size = BROTLI_ALLOC(m, uint32_t, cluster_size_capacity);
|
||||
size_t num_clusters = 0;
|
||||
HistogramType* histograms = BROTLI_ALLOC(m, HistogramType,
|
||||
BROTLI_MIN(size_t, num_blocks, HISTOGRAMS_PER_BATCH));
|
||||
size_t max_num_pairs =
|
||||
HISTOGRAMS_PER_BATCH * HISTOGRAMS_PER_BATCH / 2;
|
||||
size_t pairs_capacity = max_num_pairs + 1;
|
||||
HistogramPair* pairs = BROTLI_ALLOC(m, HistogramPair, pairs_capacity);
|
||||
size_t pos = 0;
|
||||
uint32_t* clusters;
|
||||
size_t num_final_clusters;
|
||||
static const uint32_t kInvalidIndex = BROTLI_UINT32_MAX;
|
||||
uint32_t* new_index;
|
||||
size_t i;
|
||||
uint32_t sizes[HISTOGRAMS_PER_BATCH] = { 0 };
|
||||
uint32_t new_clusters[HISTOGRAMS_PER_BATCH] = { 0 };
|
||||
uint32_t symbols[HISTOGRAMS_PER_BATCH] = { 0 };
|
||||
uint32_t remap[HISTOGRAMS_PER_BATCH] = { 0 };
|
||||
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
|
||||
memset(block_lengths, 0, num_blocks * sizeof(uint32_t));
|
||||
|
||||
{
|
||||
size_t block_idx = 0;
|
||||
for (i = 0; i < length; ++i) {
|
||||
assert(block_idx < num_blocks);
|
||||
++block_lengths[block_idx];
|
||||
if (i + 1 == length || block_ids[i] != block_ids[i + 1]) {
|
||||
++block_idx;
|
||||
}
|
||||
}
|
||||
assert(block_idx == num_blocks);
|
||||
}
|
||||
|
||||
for (i = 0; i < num_blocks; i += HISTOGRAMS_PER_BATCH) {
|
||||
const size_t num_to_combine =
|
||||
BROTLI_MIN(size_t, num_blocks - i, HISTOGRAMS_PER_BATCH);
|
||||
size_t num_new_clusters;
|
||||
size_t j;
|
||||
for (j = 0; j < num_to_combine; ++j) {
|
||||
size_t k;
|
||||
FN(HistogramClear)(&histograms[j]);
|
||||
for (k = 0; k < block_lengths[i + j]; ++k) {
|
||||
FN(HistogramAdd)(&histograms[j], data[pos++]);
|
||||
}
|
||||
histograms[j].bit_cost_ = FN(BrotliPopulationCost)(&histograms[j]);
|
||||
new_clusters[j] = (uint32_t)j;
|
||||
symbols[j] = (uint32_t)j;
|
||||
sizes[j] = 1;
|
||||
}
|
||||
num_new_clusters = FN(BrotliHistogramCombine)(
|
||||
histograms, sizes, symbols, new_clusters, pairs, num_to_combine,
|
||||
num_to_combine, HISTOGRAMS_PER_BATCH, max_num_pairs);
|
||||
BROTLI_ENSURE_CAPACITY(m, HistogramType, all_histograms,
|
||||
all_histograms_capacity, all_histograms_size + num_new_clusters);
|
||||
BROTLI_ENSURE_CAPACITY(m, uint32_t, cluster_size,
|
||||
cluster_size_capacity, cluster_size_size + num_new_clusters);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
for (j = 0; j < num_new_clusters; ++j) {
|
||||
all_histograms[all_histograms_size++] = histograms[new_clusters[j]];
|
||||
cluster_size[cluster_size_size++] = sizes[new_clusters[j]];
|
||||
remap[new_clusters[j]] = (uint32_t)j;
|
||||
}
|
||||
for (j = 0; j < num_to_combine; ++j) {
|
||||
histogram_symbols[i + j] = (uint32_t)num_clusters + remap[symbols[j]];
|
||||
}
|
||||
num_clusters += num_new_clusters;
|
||||
assert(num_clusters == cluster_size_size);
|
||||
assert(num_clusters == all_histograms_size);
|
||||
}
|
||||
BROTLI_FREE(m, histograms);
|
||||
|
||||
max_num_pairs =
|
||||
BROTLI_MIN(size_t, 64 * num_clusters, (num_clusters / 2) * num_clusters);
|
||||
if (pairs_capacity < max_num_pairs + 1) {
|
||||
BROTLI_FREE(m, pairs);
|
||||
pairs = BROTLI_ALLOC(m, HistogramPair, max_num_pairs + 1);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
}
|
||||
|
||||
clusters = BROTLI_ALLOC(m, uint32_t, num_clusters);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
for (i = 0; i < num_clusters; ++i) {
|
||||
clusters[i] = (uint32_t)i;
|
||||
}
|
||||
num_final_clusters = FN(BrotliHistogramCombine)(
|
||||
all_histograms, cluster_size, histogram_symbols, clusters, pairs,
|
||||
num_clusters, num_blocks, BROTLI_MAX_NUMBER_OF_BLOCK_TYPES,
|
||||
max_num_pairs);
|
||||
BROTLI_FREE(m, pairs);
|
||||
BROTLI_FREE(m, cluster_size);
|
||||
|
||||
new_index = BROTLI_ALLOC(m, uint32_t, num_clusters);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
for (i = 0; i < num_clusters; ++i) new_index[i] = kInvalidIndex;
|
||||
pos = 0;
|
||||
{
|
||||
uint32_t next_index = 0;
|
||||
for (i = 0; i < num_blocks; ++i) {
|
||||
HistogramType histo;
|
||||
size_t j;
|
||||
uint32_t best_out;
|
||||
double best_bits;
|
||||
FN(HistogramClear)(&histo);
|
||||
for (j = 0; j < block_lengths[i]; ++j) {
|
||||
FN(HistogramAdd)(&histo, data[pos++]);
|
||||
}
|
||||
best_out = (i == 0) ? histogram_symbols[0] : histogram_symbols[i - 1];
|
||||
best_bits =
|
||||
FN(BrotliHistogramBitCostDistance)(&histo, &all_histograms[best_out]);
|
||||
for (j = 0; j < num_final_clusters; ++j) {
|
||||
const double cur_bits = FN(BrotliHistogramBitCostDistance)(
|
||||
&histo, &all_histograms[clusters[j]]);
|
||||
if (cur_bits < best_bits) {
|
||||
best_bits = cur_bits;
|
||||
best_out = clusters[j];
|
||||
}
|
||||
}
|
||||
histogram_symbols[i] = best_out;
|
||||
if (new_index[best_out] == kInvalidIndex) {
|
||||
new_index[best_out] = next_index++;
|
||||
}
|
||||
}
|
||||
}
|
||||
BROTLI_FREE(m, clusters);
|
||||
BROTLI_FREE(m, all_histograms);
|
||||
BROTLI_ENSURE_CAPACITY(
|
||||
m, uint8_t, split->types, split->types_alloc_size, num_blocks);
|
||||
BROTLI_ENSURE_CAPACITY(
|
||||
m, uint32_t, split->lengths, split->lengths_alloc_size, num_blocks);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
{
|
||||
uint32_t cur_length = 0;
|
||||
size_t block_idx = 0;
|
||||
uint8_t max_type = 0;
|
||||
for (i = 0; i < num_blocks; ++i) {
|
||||
cur_length += block_lengths[i];
|
||||
if (i + 1 == num_blocks ||
|
||||
histogram_symbols[i] != histogram_symbols[i + 1]) {
|
||||
const uint8_t id = (uint8_t)new_index[histogram_symbols[i]];
|
||||
split->types[block_idx] = id;
|
||||
split->lengths[block_idx] = cur_length;
|
||||
max_type = BROTLI_MAX(uint8_t, max_type, id);
|
||||
cur_length = 0;
|
||||
++block_idx;
|
||||
}
|
||||
}
|
||||
split->num_blocks = block_idx;
|
||||
split->num_types = (size_t)max_type + 1;
|
||||
}
|
||||
BROTLI_FREE(m, new_index);
|
||||
BROTLI_FREE(m, block_lengths);
|
||||
BROTLI_FREE(m, histogram_symbols);
|
||||
}
|
||||
|
||||
static void FN(SplitByteVector)(MemoryManager* m,
|
||||
const DataType* data, const size_t length,
|
||||
const size_t literals_per_histogram,
|
||||
const size_t max_histograms,
|
||||
const size_t sampling_stride_length,
|
||||
const double block_switch_cost,
|
||||
const BrotliEncoderParams* params,
|
||||
BlockSplit* split) {
|
||||
const size_t data_size = FN(HistogramDataSize)();
|
||||
size_t num_histograms = length / literals_per_histogram + 1;
|
||||
HistogramType* histograms;
|
||||
if (num_histograms > max_histograms) {
|
||||
num_histograms = max_histograms;
|
||||
}
|
||||
if (length == 0) {
|
||||
split->num_types = 1;
|
||||
return;
|
||||
} else if (length < kMinLengthForBlockSplitting) {
|
||||
BROTLI_ENSURE_CAPACITY(m, uint8_t,
|
||||
split->types, split->types_alloc_size, split->num_blocks + 1);
|
||||
BROTLI_ENSURE_CAPACITY(m, uint32_t,
|
||||
split->lengths, split->lengths_alloc_size, split->num_blocks + 1);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
split->num_types = 1;
|
||||
split->types[split->num_blocks] = 0;
|
||||
split->lengths[split->num_blocks] = (uint32_t)length;
|
||||
split->num_blocks++;
|
||||
return;
|
||||
}
|
||||
histograms = BROTLI_ALLOC(m, HistogramType, num_histograms);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
/* Find good entropy codes. */
|
||||
FN(InitialEntropyCodes)(data, length,
|
||||
sampling_stride_length,
|
||||
num_histograms, histograms);
|
||||
FN(RefineEntropyCodes)(data, length,
|
||||
sampling_stride_length,
|
||||
num_histograms, histograms);
|
||||
{
|
||||
/* Find a good path through literals with the good entropy codes. */
|
||||
uint8_t* block_ids = BROTLI_ALLOC(m, uint8_t, length);
|
||||
size_t num_blocks = 0;
|
||||
const size_t bitmaplen = (num_histograms + 7) >> 3;
|
||||
double* insert_cost = BROTLI_ALLOC(m, double, data_size * num_histograms);
|
||||
double* cost = BROTLI_ALLOC(m, double, num_histograms);
|
||||
uint8_t* switch_signal = BROTLI_ALLOC(m, uint8_t, length * bitmaplen);
|
||||
uint16_t* new_id = BROTLI_ALLOC(m, uint16_t, num_histograms);
|
||||
const size_t iters = params->quality < HQ_ZOPFLIFICATION_QUALITY ? 3 : 10;
|
||||
size_t i;
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
for (i = 0; i < iters; ++i) {
|
||||
num_blocks = FN(FindBlocks)(data, length,
|
||||
block_switch_cost,
|
||||
num_histograms, histograms,
|
||||
insert_cost, cost, switch_signal,
|
||||
block_ids);
|
||||
num_histograms = FN(RemapBlockIds)(block_ids, length,
|
||||
new_id, num_histograms);
|
||||
FN(BuildBlockHistograms)(data, length, block_ids,
|
||||
num_histograms, histograms);
|
||||
}
|
||||
BROTLI_FREE(m, insert_cost);
|
||||
BROTLI_FREE(m, cost);
|
||||
BROTLI_FREE(m, switch_signal);
|
||||
BROTLI_FREE(m, new_id);
|
||||
BROTLI_FREE(m, histograms);
|
||||
FN(ClusterBlocks)(m, data, length, num_blocks, block_ids, split);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
BROTLI_FREE(m, block_ids);
|
||||
}
|
||||
}
|
||||
|
||||
#undef HistogramType
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
/* Copyright 2014 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Functions to convert brotli-related data structures into the
|
||||
brotli bit stream. The functions here operate under
|
||||
assumption that there is enough space in the storage, i.e., there are
|
||||
no out-of-range checks anywhere.
|
||||
|
||||
These functions do bit addressing into a byte array. The byte array
|
||||
is called "storage" and the index to the bit is called storage_ix
|
||||
in function arguments. */
|
||||
|
||||
#ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
||||
#define BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
||||
|
||||
#include <brotli/types.h>
|
||||
#include "./command.h"
|
||||
#include "./context.h"
|
||||
#include "./entropy_encode.h"
|
||||
#include "./memory.h"
|
||||
#include "./metablock.h"
|
||||
#include "./port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* All Store functions here will use a storage_ix, which is always the bit
|
||||
position for the current storage. */
|
||||
|
||||
BROTLI_INTERNAL void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
||||
HuffmanTree* tree, size_t *storage_ix, uint8_t *storage);
|
||||
|
||||
BROTLI_INTERNAL void BrotliBuildAndStoreHuffmanTreeFast(
|
||||
MemoryManager* m, const uint32_t* histogram, const size_t histogram_total,
|
||||
const size_t max_bits, uint8_t* depth, uint16_t* bits, size_t* storage_ix,
|
||||
uint8_t* storage);
|
||||
|
||||
/* REQUIRES: length > 0 */
|
||||
/* REQUIRES: length <= (1 << 24) */
|
||||
BROTLI_INTERNAL void BrotliStoreMetaBlock(MemoryManager* m,
|
||||
const uint8_t* input,
|
||||
size_t start_pos,
|
||||
size_t length,
|
||||
size_t mask,
|
||||
uint8_t prev_byte,
|
||||
uint8_t prev_byte2,
|
||||
BROTLI_BOOL is_final_block,
|
||||
uint32_t num_direct_distance_codes,
|
||||
uint32_t distance_postfix_bits,
|
||||
ContextType literal_context_mode,
|
||||
const Command* commands,
|
||||
size_t n_commands,
|
||||
const MetaBlockSplit* mb,
|
||||
size_t* storage_ix,
|
||||
uint8_t* storage);
|
||||
|
||||
/* Stores the meta-block without doing any block splitting, just collects
|
||||
one histogram per block category and uses that for entropy coding.
|
||||
REQUIRES: length > 0
|
||||
REQUIRES: length <= (1 << 24) */
|
||||
BROTLI_INTERNAL void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
||||
const uint8_t* input,
|
||||
size_t start_pos,
|
||||
size_t length,
|
||||
size_t mask,
|
||||
BROTLI_BOOL is_last,
|
||||
const Command *commands,
|
||||
size_t n_commands,
|
||||
size_t* storage_ix,
|
||||
uint8_t* storage);
|
||||
|
||||
/* Same as above, but uses static prefix codes for histograms with a only a few
|
||||
symbols, and uses static code length prefix codes for all other histograms.
|
||||
REQUIRES: length > 0
|
||||
REQUIRES: length <= (1 << 24) */
|
||||
BROTLI_INTERNAL void BrotliStoreMetaBlockFast(MemoryManager* m,
|
||||
const uint8_t* input,
|
||||
size_t start_pos,
|
||||
size_t length,
|
||||
size_t mask,
|
||||
BROTLI_BOOL is_last,
|
||||
const Command *commands,
|
||||
size_t n_commands,
|
||||
size_t* storage_ix,
|
||||
uint8_t* storage);
|
||||
|
||||
/* This is for storing uncompressed blocks (simple raw storage of
|
||||
bytes-as-bytes).
|
||||
REQUIRES: length > 0
|
||||
REQUIRES: length <= (1 << 24) */
|
||||
BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
|
||||
BROTLI_BOOL is_final_block, const uint8_t* input, size_t position,
|
||||
size_t mask, size_t len, size_t* storage_ix, uint8_t* storage);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_ENC_BROTLI_BIT_STREAM_H_ */
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Functions for clustering similar histograms together. */
|
||||
|
||||
#ifndef BROTLI_ENC_CLUSTER_H_
|
||||
#define BROTLI_ENC_CLUSTER_H_
|
||||
|
||||
#include <brotli/types.h>
|
||||
#include "./histogram.h"
|
||||
#include "./memory.h"
|
||||
#include "./port.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct HistogramPair {
|
||||
uint32_t idx1;
|
||||
uint32_t idx2;
|
||||
double cost_combo;
|
||||
double cost_diff;
|
||||
} HistogramPair;
|
||||
|
||||
#define CODE(X) /* Declaration */;
|
||||
|
||||
#define FN(X) X ## Literal
|
||||
#include "./cluster_inc.h" /* NOLINT(build/include) */
|
||||
#undef FN
|
||||
|
||||
#define FN(X) X ## Command
|
||||
#include "./cluster_inc.h" /* NOLINT(build/include) */
|
||||
#undef FN
|
||||
|
||||
#define FN(X) X ## Distance
|
||||
#include "./cluster_inc.h" /* NOLINT(build/include) */
|
||||
#undef FN
|
||||
|
||||
#undef CODE
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_ENC_CLUSTER_H_ */
|
||||
|
|
@ -1,317 +0,0 @@
|
|||
/* NOLINT(build/header_guard) */
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* template parameters: FN, CODE */
|
||||
|
||||
#define HistogramType FN(Histogram)
|
||||
|
||||
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
|
||||
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
|
||||
BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
|
||||
const HistogramType* out, const uint32_t* cluster_size, uint32_t idx1,
|
||||
uint32_t idx2, size_t max_num_pairs, HistogramPair* pairs,
|
||||
size_t* num_pairs) CODE({
|
||||
BROTLI_BOOL is_good_pair = BROTLI_FALSE;
|
||||
HistogramPair p;
|
||||
p.idx1 = p.idx2 = 0;
|
||||
p.cost_diff = p.cost_combo = 0;
|
||||
if (idx1 == idx2) {
|
||||
return;
|
||||
}
|
||||
if (idx2 < idx1) {
|
||||
uint32_t t = idx2;
|
||||
idx2 = idx1;
|
||||
idx1 = t;
|
||||
}
|
||||
p.idx1 = idx1;
|
||||
p.idx2 = idx2;
|
||||
p.cost_diff = 0.5 * ClusterCostDiff(cluster_size[idx1], cluster_size[idx2]);
|
||||
p.cost_diff -= out[idx1].bit_cost_;
|
||||
p.cost_diff -= out[idx2].bit_cost_;
|
||||
|
||||
if (out[idx1].total_count_ == 0) {
|
||||
p.cost_combo = out[idx2].bit_cost_;
|
||||
is_good_pair = BROTLI_TRUE;
|
||||
} else if (out[idx2].total_count_ == 0) {
|
||||
p.cost_combo = out[idx1].bit_cost_;
|
||||
is_good_pair = BROTLI_TRUE;
|
||||
} else {
|
||||
double threshold = *num_pairs == 0 ? 1e99 :
|
||||
BROTLI_MAX(double, 0.0, pairs[0].cost_diff);
|
||||
HistogramType combo = out[idx1];
|
||||
double cost_combo;
|
||||
FN(HistogramAddHistogram)(&combo, &out[idx2]);
|
||||
cost_combo = FN(BrotliPopulationCost)(&combo);
|
||||
if (cost_combo < threshold - p.cost_diff) {
|
||||
p.cost_combo = cost_combo;
|
||||
is_good_pair = BROTLI_TRUE;
|
||||
}
|
||||
}
|
||||
if (is_good_pair) {
|
||||
p.cost_diff += p.cost_combo;
|
||||
if (*num_pairs > 0 && HistogramPairIsLess(&pairs[0], &p)) {
|
||||
/* Replace the top of the queue if needed. */
|
||||
if (*num_pairs < max_num_pairs) {
|
||||
pairs[*num_pairs] = pairs[0];
|
||||
++(*num_pairs);
|
||||
}
|
||||
pairs[0] = p;
|
||||
} else if (*num_pairs < max_num_pairs) {
|
||||
pairs[*num_pairs] = p;
|
||||
++(*num_pairs);
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
BROTLI_INTERNAL size_t FN(BrotliHistogramCombine)(HistogramType* out,
|
||||
uint32_t* cluster_size,
|
||||
uint32_t* symbols,
|
||||
uint32_t* clusters,
|
||||
HistogramPair* pairs,
|
||||
size_t num_clusters,
|
||||
size_t symbols_size,
|
||||
size_t max_clusters,
|
||||
size_t max_num_pairs) CODE({
|
||||
double cost_diff_threshold = 0.0;
|
||||
size_t min_cluster_size = 1;
|
||||
size_t num_pairs = 0;
|
||||
|
||||
{
|
||||
/* We maintain a vector of histogram pairs, with the property that the pair
|
||||
with the maximum bit cost reduction is the first. */
|
||||
size_t idx1;
|
||||
for (idx1 = 0; idx1 < num_clusters; ++idx1) {
|
||||
size_t idx2;
|
||||
for (idx2 = idx1 + 1; idx2 < num_clusters; ++idx2) {
|
||||
FN(BrotliCompareAndPushToQueue)(out, cluster_size, clusters[idx1],
|
||||
clusters[idx2], max_num_pairs, &pairs[0], &num_pairs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (num_clusters > min_cluster_size) {
|
||||
uint32_t best_idx1;
|
||||
uint32_t best_idx2;
|
||||
size_t i;
|
||||
if (pairs[0].cost_diff >= cost_diff_threshold) {
|
||||
cost_diff_threshold = 1e99;
|
||||
min_cluster_size = max_clusters;
|
||||
continue;
|
||||
}
|
||||
/* Take the best pair from the top of heap. */
|
||||
best_idx1 = pairs[0].idx1;
|
||||
best_idx2 = pairs[0].idx2;
|
||||
FN(HistogramAddHistogram)(&out[best_idx1], &out[best_idx2]);
|
||||
out[best_idx1].bit_cost_ = pairs[0].cost_combo;
|
||||
cluster_size[best_idx1] += cluster_size[best_idx2];
|
||||
for (i = 0; i < symbols_size; ++i) {
|
||||
if (symbols[i] == best_idx2) {
|
||||
symbols[i] = best_idx1;
|
||||
}
|
||||
}
|
||||
for (i = 0; i < num_clusters; ++i) {
|
||||
if (clusters[i] == best_idx2) {
|
||||
memmove(&clusters[i], &clusters[i + 1],
|
||||
(num_clusters - i - 1) * sizeof(clusters[0]));
|
||||
break;
|
||||
}
|
||||
}
|
||||
--num_clusters;
|
||||
{
|
||||
/* Remove pairs intersecting the just combined best pair. */
|
||||
size_t copy_to_idx = 0;
|
||||
for (i = 0; i < num_pairs; ++i) {
|
||||
HistogramPair* p = &pairs[i];
|
||||
if (p->idx1 == best_idx1 || p->idx2 == best_idx1 ||
|
||||
p->idx1 == best_idx2 || p->idx2 == best_idx2) {
|
||||
/* Remove invalid pair from the queue. */
|
||||
continue;
|
||||
}
|
||||
if (HistogramPairIsLess(&pairs[0], p)) {
|
||||
/* Replace the top of the queue if needed. */
|
||||
HistogramPair front = pairs[0];
|
||||
pairs[0] = *p;
|
||||
pairs[copy_to_idx] = front;
|
||||
} else {
|
||||
pairs[copy_to_idx] = *p;
|
||||
}
|
||||
++copy_to_idx;
|
||||
}
|
||||
num_pairs = copy_to_idx;
|
||||
}
|
||||
|
||||
/* Push new pairs formed with the combined histogram to the heap. */
|
||||
for (i = 0; i < num_clusters; ++i) {
|
||||
FN(BrotliCompareAndPushToQueue)(out, cluster_size, best_idx1, clusters[i],
|
||||
max_num_pairs, &pairs[0], &num_pairs);
|
||||
}
|
||||
}
|
||||
return num_clusters;
|
||||
})
|
||||
|
||||
/* What is the bit cost of moving histogram from cur_symbol to candidate. */
|
||||
BROTLI_INTERNAL double FN(BrotliHistogramBitCostDistance)(
|
||||
const HistogramType* histogram, const HistogramType* candidate) CODE({
|
||||
if (histogram->total_count_ == 0) {
|
||||
return 0.0;
|
||||
} else {
|
||||
HistogramType tmp = *histogram;
|
||||
FN(HistogramAddHistogram)(&tmp, candidate);
|
||||
return FN(BrotliPopulationCost)(&tmp) - candidate->bit_cost_;
|
||||
}
|
||||
})
|
||||
|
||||
/* Find the best 'out' histogram for each of the 'in' histograms.
|
||||
When called, clusters[0..num_clusters) contains the unique values from
|
||||
symbols[0..in_size), but this property is not preserved in this function.
|
||||
Note: we assume that out[]->bit_cost_ is already up-to-date. */
|
||||
BROTLI_INTERNAL void FN(BrotliHistogramRemap)(const HistogramType* in,
|
||||
size_t in_size, const uint32_t* clusters, size_t num_clusters,
|
||||
HistogramType* out, uint32_t* symbols) CODE({
|
||||
size_t i;
|
||||
for (i = 0; i < in_size; ++i) {
|
||||
uint32_t best_out = i == 0 ? symbols[0] : symbols[i - 1];
|
||||
double best_bits =
|
||||
FN(BrotliHistogramBitCostDistance)(&in[i], &out[best_out]);
|
||||
size_t j;
|
||||
for (j = 0; j < num_clusters; ++j) {
|
||||
const double cur_bits =
|
||||
FN(BrotliHistogramBitCostDistance)(&in[i], &out[clusters[j]]);
|
||||
if (cur_bits < best_bits) {
|
||||
best_bits = cur_bits;
|
||||
best_out = clusters[j];
|
||||
}
|
||||
}
|
||||
symbols[i] = best_out;
|
||||
}
|
||||
|
||||
/* Recompute each out based on raw and symbols. */
|
||||
for (i = 0; i < num_clusters; ++i) {
|
||||
FN(HistogramClear)(&out[clusters[i]]);
|
||||
}
|
||||
for (i = 0; i < in_size; ++i) {
|
||||
FN(HistogramAddHistogram)(&out[symbols[i]], &in[i]);
|
||||
}
|
||||
})
|
||||
|
||||
/* Reorders elements of the out[0..length) array and changes values in
|
||||
symbols[0..length) array in the following way:
|
||||
* when called, symbols[] contains indexes into out[], and has N unique
|
||||
values (possibly N < length)
|
||||
* on return, symbols'[i] = f(symbols[i]) and
|
||||
out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
|
||||
where f is a bijection between the range of symbols[] and [0..N), and
|
||||
the first occurrences of values in symbols'[i] come in consecutive
|
||||
increasing order.
|
||||
Returns N, the number of unique values in symbols[]. */
|
||||
BROTLI_INTERNAL size_t FN(BrotliHistogramReindex)(MemoryManager* m,
|
||||
HistogramType* out, uint32_t* symbols, size_t length) CODE({
|
||||
static const uint32_t kInvalidIndex = BROTLI_UINT32_MAX;
|
||||
uint32_t* new_index = BROTLI_ALLOC(m, uint32_t, length);
|
||||
uint32_t next_index;
|
||||
HistogramType* tmp;
|
||||
size_t i;
|
||||
if (BROTLI_IS_OOM(m)) return 0;
|
||||
for (i = 0; i < length; ++i) {
|
||||
new_index[i] = kInvalidIndex;
|
||||
}
|
||||
next_index = 0;
|
||||
for (i = 0; i < length; ++i) {
|
||||
if (new_index[symbols[i]] == kInvalidIndex) {
|
||||
new_index[symbols[i]] = next_index;
|
||||
++next_index;
|
||||
}
|
||||
}
|
||||
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
|
||||
tmp and reduce the number of copying by the factor of 2. */
|
||||
tmp = BROTLI_ALLOC(m, HistogramType, next_index);
|
||||
if (BROTLI_IS_OOM(m)) return 0;
|
||||
next_index = 0;
|
||||
for (i = 0; i < length; ++i) {
|
||||
if (new_index[symbols[i]] == next_index) {
|
||||
tmp[next_index] = out[symbols[i]];
|
||||
++next_index;
|
||||
}
|
||||
symbols[i] = new_index[symbols[i]];
|
||||
}
|
||||
BROTLI_FREE(m, new_index);
|
||||
for (i = 0; i < next_index; ++i) {
|
||||
out[i] = tmp[i];
|
||||
}
|
||||
BROTLI_FREE(m, tmp);
|
||||
return next_index;
|
||||
})
|
||||
|
||||
BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
|
||||
MemoryManager* m, const HistogramType* in, const size_t in_size,
|
||||
size_t max_histograms, HistogramType* out, size_t* out_size,
|
||||
uint32_t* histogram_symbols) CODE({
|
||||
uint32_t* cluster_size = BROTLI_ALLOC(m, uint32_t, in_size);
|
||||
uint32_t* clusters = BROTLI_ALLOC(m, uint32_t, in_size);
|
||||
size_t num_clusters = 0;
|
||||
const size_t max_input_histograms = 64;
|
||||
size_t pairs_capacity = max_input_histograms * max_input_histograms / 2;
|
||||
/* For the first pass of clustering, we allow all pairs. */
|
||||
HistogramPair* pairs = BROTLI_ALLOC(m, HistogramPair, pairs_capacity + 1);
|
||||
size_t i;
|
||||
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
|
||||
for (i = 0; i < in_size; ++i) {
|
||||
cluster_size[i] = 1;
|
||||
}
|
||||
|
||||
for (i = 0; i < in_size; ++i) {
|
||||
out[i] = in[i];
|
||||
out[i].bit_cost_ = FN(BrotliPopulationCost)(&in[i]);
|
||||
histogram_symbols[i] = (uint32_t)i;
|
||||
}
|
||||
|
||||
for (i = 0; i < in_size; i += max_input_histograms) {
|
||||
size_t num_to_combine =
|
||||
BROTLI_MIN(size_t, in_size - i, max_input_histograms);
|
||||
size_t num_new_clusters;
|
||||
size_t j;
|
||||
for (j = 0; j < num_to_combine; ++j) {
|
||||
clusters[num_clusters + j] = (uint32_t)(i + j);
|
||||
}
|
||||
num_new_clusters =
|
||||
FN(BrotliHistogramCombine)(out, cluster_size,
|
||||
&histogram_symbols[i],
|
||||
&clusters[num_clusters], pairs,
|
||||
num_to_combine, num_to_combine,
|
||||
max_histograms, pairs_capacity);
|
||||
num_clusters += num_new_clusters;
|
||||
}
|
||||
|
||||
{
|
||||
/* For the second pass, we limit the total number of histogram pairs.
|
||||
After this limit is reached, we only keep searching for the best pair. */
|
||||
size_t max_num_pairs = BROTLI_MIN(size_t,
|
||||
64 * num_clusters, (num_clusters / 2) * num_clusters);
|
||||
BROTLI_ENSURE_CAPACITY(
|
||||
m, HistogramPair, pairs, pairs_capacity, max_num_pairs + 1);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
|
||||
/* Collapse similar histograms. */
|
||||
num_clusters = FN(BrotliHistogramCombine)(out, cluster_size,
|
||||
histogram_symbols, clusters,
|
||||
pairs, num_clusters, in_size,
|
||||
max_histograms, max_num_pairs);
|
||||
}
|
||||
BROTLI_FREE(m, pairs);
|
||||
BROTLI_FREE(m, cluster_size);
|
||||
/* Find the optimal map from original histograms to the final ones. */
|
||||
FN(BrotliHistogramRemap)(in, in_size, clusters, num_clusters,
|
||||
out, histogram_symbols);
|
||||
BROTLI_FREE(m, clusters);
|
||||
/* Convert the context map to a canonical form. */
|
||||
*out_size = FN(BrotliHistogramReindex)(m, out, histogram_symbols, in_size);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
})
|
||||
|
||||
#undef HistogramType
|
||||
|
|
@ -1,180 +0,0 @@
|
|||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* This class models a sequence of literals and a backward reference copy. */
|
||||
|
||||
#ifndef BROTLI_ENC_COMMAND_H_
|
||||
#define BROTLI_ENC_COMMAND_H_
|
||||
|
||||
#include "../common/constants.h"
|
||||
#include <brotli/port.h>
|
||||
#include <brotli/types.h>
|
||||
#include "./fast_log.h"
|
||||
#include "./prefix.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static uint32_t kInsBase[] = { 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50,
|
||||
66, 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594 };
|
||||
static uint32_t kInsExtra[] = { 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4,
|
||||
5, 5, 6, 7, 8, 9, 10, 12, 14, 24 };
|
||||
static uint32_t kCopyBase[] = { 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22, 30,
|
||||
38, 54, 70, 102, 134, 198, 326, 582, 1094, 2118 };
|
||||
static uint32_t kCopyExtra[] = { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
|
||||
4, 4, 5, 5, 6, 7, 8, 9, 10, 24 };
|
||||
|
||||
static BROTLI_INLINE uint16_t GetInsertLengthCode(size_t insertlen) {
|
||||
if (insertlen < 6) {
|
||||
return (uint16_t)insertlen;
|
||||
} else if (insertlen < 130) {
|
||||
uint32_t nbits = Log2FloorNonZero(insertlen - 2) - 1u;
|
||||
return (uint16_t)((nbits << 1) + ((insertlen - 2) >> nbits) + 2);
|
||||
} else if (insertlen < 2114) {
|
||||
return (uint16_t)(Log2FloorNonZero(insertlen - 66) + 10);
|
||||
} else if (insertlen < 6210) {
|
||||
return 21u;
|
||||
} else if (insertlen < 22594) {
|
||||
return 22u;
|
||||
} else {
|
||||
return 23u;
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint16_t GetCopyLengthCode(size_t copylen) {
|
||||
if (copylen < 10) {
|
||||
return (uint16_t)(copylen - 2);
|
||||
} else if (copylen < 134) {
|
||||
uint32_t nbits = Log2FloorNonZero(copylen - 6) - 1u;
|
||||
return (uint16_t)((nbits << 1) + ((copylen - 6) >> nbits) + 4);
|
||||
} else if (copylen < 2118) {
|
||||
return (uint16_t)(Log2FloorNonZero(copylen - 70) + 12);
|
||||
} else {
|
||||
return 23u;
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint16_t CombineLengthCodes(
|
||||
uint16_t inscode, uint16_t copycode, BROTLI_BOOL use_last_distance) {
|
||||
uint16_t bits64 =
|
||||
(uint16_t)((copycode & 0x7u) | ((inscode & 0x7u) << 3));
|
||||
if (use_last_distance && inscode < 8 && copycode < 16) {
|
||||
return (copycode < 8) ? bits64 : (bits64 | 64);
|
||||
} else {
|
||||
/* Specification: 5 Encoding of ... (last table) */
|
||||
/* offset = 2 * index, where index is in range [0..8] */
|
||||
int offset = 2 * ((copycode >> 3) + 3 * (inscode >> 3));
|
||||
/* All values in specification are K * 64,
|
||||
where K = [2, 3, 6, 4, 5, 8, 7, 9, 10],
|
||||
i + 1 = [1, 2, 3, 4, 5, 6, 7, 8, 9],
|
||||
K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1, 2] = D.
|
||||
All values in D require only 2 bits to encode.
|
||||
Magic constant is shifted 6 bits left, to avoid final multiplication. */
|
||||
offset = (offset << 5) + 0x40 + ((0x520D40 >> offset) & 0xC0);
|
||||
return (uint16_t)offset | bits64;
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void GetLengthCode(size_t insertlen, size_t copylen,
|
||||
BROTLI_BOOL use_last_distance,
|
||||
uint16_t* code) {
|
||||
uint16_t inscode = GetInsertLengthCode(insertlen);
|
||||
uint16_t copycode = GetCopyLengthCode(copylen);
|
||||
*code = CombineLengthCodes(inscode, copycode, use_last_distance);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t GetInsertBase(uint16_t inscode) {
|
||||
return kInsBase[inscode];
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t GetInsertExtra(uint16_t inscode) {
|
||||
return kInsExtra[inscode];
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t GetCopyBase(uint16_t copycode) {
|
||||
return kCopyBase[copycode];
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t GetCopyExtra(uint16_t copycode) {
|
||||
return kCopyExtra[copycode];
|
||||
}
|
||||
|
||||
typedef struct Command {
|
||||
uint32_t insert_len_;
|
||||
/* Stores copy_len in low 24 bits and copy_len XOR copy_code in high 8 bit. */
|
||||
uint32_t copy_len_;
|
||||
uint32_t dist_extra_;
|
||||
uint16_t cmd_prefix_;
|
||||
uint16_t dist_prefix_;
|
||||
} Command;
|
||||
|
||||
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
|
||||
static BROTLI_INLINE void InitCommand(Command* self, size_t insertlen,
|
||||
size_t copylen, int copylen_code_delta, size_t distance_code) {
|
||||
/* Don't rely on signed int representation, use honest casts. */
|
||||
uint32_t delta = (uint8_t)((int8_t)copylen_code_delta);
|
||||
self->insert_len_ = (uint32_t)insertlen;
|
||||
self->copy_len_ = (uint32_t)(copylen | (delta << 24));
|
||||
/* The distance prefix and extra bits are stored in this Command as if
|
||||
npostfix and ndirect were 0, they are only recomputed later after the
|
||||
clustering if needed. */
|
||||
PrefixEncodeCopyDistance(
|
||||
distance_code, 0, 0, &self->dist_prefix_, &self->dist_extra_);
|
||||
GetLengthCode(
|
||||
insertlen, (size_t)((int)copylen + copylen_code_delta),
|
||||
TO_BROTLI_BOOL(self->dist_prefix_ == 0), &self->cmd_prefix_);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) {
|
||||
self->insert_len_ = (uint32_t)insertlen;
|
||||
self->copy_len_ = 4 << 24;
|
||||
self->dist_extra_ = 0;
|
||||
self->dist_prefix_ = BROTLI_NUM_DISTANCE_SHORT_CODES;
|
||||
GetLengthCode(insertlen, 4, BROTLI_FALSE, &self->cmd_prefix_);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t CommandRestoreDistanceCode(const Command* self) {
|
||||
if (self->dist_prefix_ < BROTLI_NUM_DISTANCE_SHORT_CODES) {
|
||||
return self->dist_prefix_;
|
||||
} else {
|
||||
uint32_t nbits = self->dist_extra_ >> 24;
|
||||
uint32_t extra = self->dist_extra_ & 0xffffff;
|
||||
/* It is assumed that the distance was first encoded with NPOSTFIX = 0 and
|
||||
NDIRECT = 0, so the code itself is of this form:
|
||||
BROTLI_NUM_DISTANCE_SHORT_CODES + 2 * (nbits - 1) + prefix_bit
|
||||
Therefore, the following expression results in (2 + prefix_bit). */
|
||||
uint32_t prefix =
|
||||
self->dist_prefix_ + 4u - BROTLI_NUM_DISTANCE_SHORT_CODES - 2u * nbits;
|
||||
/* Subtract 4 for offset (Chapter 4.) and
|
||||
increase by BROTLI_NUM_DISTANCE_SHORT_CODES - 1 */
|
||||
return (prefix << nbits) + extra + BROTLI_NUM_DISTANCE_SHORT_CODES - 4u;
|
||||
}
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t CommandDistanceContext(const Command* self) {
|
||||
uint32_t r = self->cmd_prefix_ >> 6;
|
||||
uint32_t c = self->cmd_prefix_ & 7;
|
||||
if ((r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2)) {
|
||||
return c;
|
||||
}
|
||||
return 3;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t CommandCopyLen(const Command* self) {
|
||||
return self->copy_len_ & 0xFFFFFF;
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint32_t CommandCopyLenCode(const Command* self) {
|
||||
int32_t delta = (int8_t)((uint8_t)(self->copy_len_ >> 24));
|
||||
return (uint32_t)((int32_t)(self->copy_len_ & 0xFFFFFF) + delta);
|
||||
}
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_ENC_COMMAND_H_ */
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue