forked from toolshed/abra
chore: vendor
This commit is contained in:
.gitignoremodules.txt
vendor
coopcloud.tech
dario.cat
mergo
git.coopcloud.tech
coop-cloud
github.com
AlecAivazis
survey
v2
CONTRIBUTING.mdLICENSEREADME.mdconfirm.go
core
editor.gofilter.goinput.gomultiline.gomultiselect.gopassword.gorenderer.goselect.gosurvey.goterminal
LICENSE.txtREADME.mdbuffered_reader.gocursor.gocursor_windows.godisplay.godisplay_posix.godisplay_windows.goerror.gooutput.gooutput_windows.gorunereader.gorunereader_bsd.gorunereader_linux.gorunereader_posix.gorunereader_ppc64le.gorunereader_windows.gosequences.gostdio.gosyscall_windows.goterminal.go
transform.govalidate.goAzure
go-ansiterm
BurntSushi
toml
Microsoft
go-winio
ProtonMail
go-crypto
AUTHORSCONTRIBUTORSLICENSEPATENTSkey_generation.gokeys.gokeys_test_data.go
bitcurves
brainpool
eax
internal
byteutil
ocb
openpgp
aes
keywrap
armor
canonical_text.goecdh
ecdsa
eddsa
elgamal
errors
hash.gointernal
algorithm
ecc
encoding
packet
aead_config.goaead_crypter.goaead_encrypted.gocompressed.goconfig.goencrypted_key.goliteral.gonotation.goocfb.goone_pass_signature.goopaque.gopacket.goprivate_key.goprivate_key_test_data.gopublic_key.gopublic_key_test_data.goreader.gosignature.gosymmetric_key_encrypted.gosymmetrically_encrypted.gosymmetrically_encrypted_aead.gosymmetrically_encrypted_mdc.gouserattribute.gouserid.go
read.goread_write_test_data.gos2k
write.goaymanbagabas
beorn7
perks
cenkalti
backoff
cespare
xxhash
charmbracelet
lipgloss
.gitignore.golangci-soft.yml.golangci.ymlLICENSEREADME.mdalign.goansi_unix.goansi_windows.goborders.gocolor.goget.gojoin.goposition.gorenderer.gorunes.goset.gosize.gostyle.go
table
unset.gowhitespace.golog
.gitignore.golangci.yml.goreleaser.ymlLICENSEREADME.mdcontext.goformatter.gojson.golevel.golevel_121.golevel_no121.gologfmt.gologger.gologger_121.gologger_no121.gooptions.gopkg.gostdlog.gostyles.gotext.go
x
cloudflare
containerd
containerd
log
containers
image
LICENSE
docker
cache.godocker_client.godocker_image.godocker_image_dest.godocker_image_src.godocker_transport.golookaside.go
policyconfiguration
reference
wwwauthenticate.goimage
manifest
pkg
blobinfocache
none
docker
keyctl
strslice
sysregistriesv2
tlsclientconfig
transports
types
cpuguy83
go-md2man
v2
cyphar
davecgh
decentral1se
distribution
reference
docker
cli
AUTHORSLICENSENOTICE
cli-plugins
hooks
manager
cli
cobra.go
command
cli.gocli_options.gocontext.godefaultcontextstore.gotelemetry.gotelemetry_docker.gotelemetry_options.gotelemetry_utils.gotrust.goutils.go
formatter
buildcache.gocontainer.gocontext.gocustom.godisk_usage.godisplayutils.goformatter.goimage.goreflect.go
registry.gotabwriter
volume.goservice
progress
stack
formatter
compose
interpolation
loader
schema
data
config_schema_v3.0.jsonconfig_schema_v3.1.jsonconfig_schema_v3.10.jsonconfig_schema_v3.11.jsonconfig_schema_v3.12.jsonconfig_schema_v3.13.jsonconfig_schema_v3.2.jsonconfig_schema_v3.3.jsonconfig_schema_v3.4.jsonconfig_schema_v3.5.jsonconfig_schema_v3.6.jsonconfig_schema_v3.7.jsonconfig_schema_v3.8.jsonconfig_schema_v3.9.json
schema.gotemplate
types
config
config.go
configfile
credentials
credentials.godefault_store.godefault_store_darwin.godefault_store_linux.godefault_store_unsupported.godefault_store_windows.gofile_store.gonative_store.go
types
connhelper
context
debug
error.goflags
hints
manifest
registry
required.gostreams
trust
version
opts
capabilities.goconfig.goduration.goenv.goenvfile.gofile.gogpus.gohosts.gohosts_unix.gohosts_windows.gomount.gonetwork.goopts.goparse.goport.goquotedstring.gosecret.gothrottledevice.goulimit.goweightdevice.go
templates
distribution
.dockerignore.gitignore.golangci.yml.mailmapBUILDING.mdCONTRIBUTING.mdDockerfileLICENSEMAINTAINERSMakefileREADME.mdROADMAP.mdblobs.godoc.godocker-bake.hclerrors.go
manifest
manifests.gometrics
registry.goregistry
tags.gouuid
vendor.confdocker-credential-helpers
docker
AUTHORSLICENSENOTICE
api
README.mdcommon.goswagger-gen.yamlswagger.yaml
types
blkiodev
checkpoint
client.gocontainer
change_type.gochange_types.goconfig.gocontainer.gocontainer_top.gocontainer_update.gocreate_request.gocreate_response.goerrors.goexec.gofilesystem_change.gohostconfig.gohostconfig_unix.gohostconfig_windows.gooptions.gostats.gowait_exit_error.gowait_response.gowaitcondition.go
error_response.goerror_response_ext.goevents
filters
graph_driver_data.goid_response.goimage
mount
network
plugin.goplugin_device.goplugin_env.goplugin_interface_type.goplugin_mount.goplugin_responses.goport.goregistry
strslice
swarm
common.goconfig.gocontainer.gonetwork.gonode.goruntime.go
runtime
secret.goservice.goservice_create_response.goservice_update_response.goswarm.gotask.gosystem
time
types.gotypes_deprecated.goversions
volume
client
README.mdbuild_cancel.gobuild_prune.gocheckpoint_create.gocheckpoint_delete.gocheckpoint_list.goclient.goclient_deprecated.goclient_unix.goclient_windows.goconfig_create.goconfig_inspect.goconfig_list.goconfig_remove.goconfig_update.gocontainer_attach.gocontainer_commit.gocontainer_copy.gocontainer_create.gocontainer_diff.gocontainer_exec.gocontainer_export.gocontainer_inspect.gocontainer_kill.gocontainer_list.gocontainer_logs.gocontainer_pause.gocontainer_prune.gocontainer_remove.gocontainer_rename.gocontainer_resize.gocontainer_restart.gocontainer_start.gocontainer_stats.gocontainer_stop.gocontainer_top.gocontainer_unpause.gocontainer_update.gocontainer_wait.godisk_usage.godistribution_inspect.goenvvars.goerrors.goevents.gohijack.goimage_build.goimage_create.goimage_history.goimage_import.goimage_inspect.goimage_list.goimage_load.goimage_prune.goimage_pull.goimage_push.goimage_remove.goimage_save.goimage_search.goimage_tag.goinfo.gointerface.gointerface_experimental.gointerface_stable.gologin.gonetwork_connect.gonetwork_create.gonetwork_disconnect.gonetwork_inspect.gonetwork_list.gonetwork_prune.gonetwork_remove.gonode_inspect.gonode_list.gonode_remove.gonode_update.gooptions.goping.goplugin_create.goplugin_disable.goplugin_enable.goplugin_inspect.goplugin_install.goplugin_list.goplugin_push.goplugin_remove.goplugin_set.goplugin_upgrade.gorequest.gosecret_create.gosecret_inspect.gosecret_list.gosecret_remove.gosecret_update.goservice_create.goservice_inspect.goservice_list.goservice_logs.goservice_remove.goservice_update.goswarm_get_unlock_key.goswarm_init.goswarm_inspect.goswarm_join.goswarm_leave.goswarm_unlock.goswarm_update.gotask_inspect.gotask_list.gotask_logs.goutils.goversion.govolume_create.govolume_inspect.govolume_list.govolume_prune.govolume_remove.govolume_update.go
errdefs
internal
multierror
pkg
archive
archive.goarchive_linux.goarchive_other.goarchive_unix.goarchive_windows.gochanges.gochanges_linux.gochanges_other.gochanges_unix.gochanges_windows.gocopy.gocopy_unix.gocopy_windows.godiff.godiff_unix.godiff_windows.gopath.gopath_unix.gopath_windows.gotime_linux.gotime_unsupported.gowhiteouts.gowrap.go
homedir
idtools
idtools.goidtools_unix.goidtools_windows.gousergroupadd_linux.gousergroupadd_unsupported.goutils_unix.go
ioutils
jsonmessage
pools
progress
stdcopy
streamformatter
stringid
system
args_windows.gochtimes.gochtimes_nowindows.gochtimes_windows.goerrors.gofilesys.gofilesys_unix.gofilesys_windows.goinit_windows.golstat_unix.golstat_windows.gomknod.gomknod_freebsd.gomknod_unix.gostat_bsd.gostat_darwin.gostat_illumos.gostat_linux.gostat_openbsd.gostat_unix.gostat_windows.goutimes_unix.goutimes_unsupported.goxattrs.goxattrs_linux.goxattrs_unsupported.go
registry
go-connections
LICENSE
nat
sockets
README.mdinmem_socket.goproxy.gosockets.gosockets_unix.gosockets_windows.gotcp_socket.gounix_socket.go
tlsconfig
go-metrics
CONTRIBUTING.mdLICENSELICENSE.docsNOTICEREADME.mdcounter.godocs.gogauge.gohandler.gohelpers.gonamespace.goregister.gotimer.gounit.go
go-units
go
libtrust
emirpasic
gods
felixge
httpsnoop
fvbommel
ghodss
go-git
gcfg
go-billy
v5
go-git
v5
.gitignoreCODE_OF_CONDUCT.mdCOMPATIBILITY.mdCONTRIBUTING.mdEXTENDING.mdLICENSEMakefileREADME.mdSECURITY.mdblame.gocommon.gohash.goreference.gorevision.goprune.goremote.gorepository.gosigner.gostatus.gosubmodule.goworktree.goworktree_bsd.goworktree_commit.goworktree_js.goworktree_linux.goworktree_plan9.goworktree_status.goworktree_unix_other.goworktree_windows.go
config
doc.gointernal
object_walker.gooptions.gooss-fuzz.shplumbing
cache
color
error.gofilemode
format
config
diff
gitignore
idxfile
index
objfile
packfile
common.godelta_index.godelta_selector.godiff_delta.godoc.goencoder.goerror.gofsobject.goobject_pack.gopackfile.goparser.gopatch_delta.goscanner.go
pktline
hash
memory.goobject.goobject
blob.gochange.gochange_adaptor.gocommit.gocommit_walker.gocommit_walker_bfs.gocommit_walker_bfs_filtered.gocommit_walker_ctime.gocommit_walker_limit.gocommit_walker_path.godifftree.gofile.gomerge_base.goobject.gopatch.gorename.gosignature.gotag.gotree.gotreenoder.go
protocol
packp
revlist
storer
transport
storage
filesystem
memory
storer.goutils
binary
diff
ioutil
merkletrie
sync
trace
go-logfmt
go-logr
logr
.golangci.yamlCHANGELOG.mdCONTRIBUTING.mdLICENSEREADME.mdSECURITY.mdcontext.gocontext_noslog.gocontext_slog.godiscard.go
funcr
logr.gosloghandler.goslogr.goslogsink.gostdr
go-viper
mapstructure
gogo
protobuf
AUTHORSCONTRIBUTORSLICENSE
proto
Makefileclone.gocustom_gogo.godecode.godeprecated.godiscard.goduration.goduration_gogo.goencode.goencode_gogo.goequal.goextensions.goextensions_gogo.golib.golib_gogo.gomessage_set.gopointer_reflect.gopointer_reflect_gogo.gopointer_unsafe.gopointer_unsafe_gogo.goproperties.goproperties_gogo.goskip_gogo.gotable_marshal.gotable_marshal_gogo.gotable_merge.gotable_unmarshal.gotable_unmarshal_gogo.gotext.gotext_gogo.gotext_parser.gotimestamp.gotimestamp_gogo.gowrappers.gowrappers_gogo.go
golang
google
gorilla
mux
grpc-ecosystem
grpc-gateway
hashicorp
go-cleanhttp
go-retryablehttp
inconshreveable
jbenet
kballard
go-shellquote
kevinburke
ssh_config
klauspost
compress
.gitattributes.gitignore.goreleaser.ymlLICENSEREADME.mdSECURITY.mdcompressible.gos2sx.mods2sx.sum
fse
gen.shhuff0
.gitignoreREADME.mdbitreader.gobitwriter.gocompress.godecompress.godecompress_amd64.godecompress_amd64.sdecompress_generic.gohuff0.go
internal
cpuinfo
snapref
zstd
README.mdbitreader.gobitwriter.goblockdec.goblockenc.goblocktype_string.gobytebuf.gobytereader.godecodeheader.godecoder.godecoder_options.godict.goenc_base.goenc_best.goenc_better.goenc_dfast.goenc_fast.goencoder.goencoder_options.goframedec.goframeenc.gofse_decoder.gofse_decoder_amd64.gofse_decoder_amd64.sfse_decoder_generic.gofse_encoder.gofse_predefined.gohash.gohistory.gomatchlen_amd64.gomatchlen_amd64.smatchlen_generic.goseqdec.goseqdec_amd64.goseqdec_amd64.sseqdec_generic.goseqenc.gosnappy.gozip.gozstd.go
internal
xxhash
lucasb-eyer
go-colorful
mattn
go-colorable
LICENSEREADME.mdcolorable_appengine.gocolorable_others.gocolorable_windows.gogo.test.shnoncolorable.go
go-isatty
LICENSEREADME.mddoc.gogo.test.shisatty_bsd.goisatty_others.goisatty_plan9.goisatty_solaris.goisatty_tcgets.goisatty_windows.go
go-runewidth
mgutz
miekg
pkcs11
mitchellh
colorstring
moby
docker-image-spec
patternmatcher
sys
sequential
signal
LICENSEsignal.gosignal_darwin.gosignal_freebsd.gosignal_linux.gosignal_linux_mipsx.gosignal_unix.gosignal_unsupported.gosignal_windows.go
user
term
morikuni
muesli
termenv
.gitignore.golangci-soft.yml.golangci.ymlLICENSEREADME.mdansi_compat.mdansicolors.gocolor.goconstants_linux.goconstants_solaris.goconstants_unix.gocopy.gohyperlink.gonotification.gooutput.goprofile.goscreen.gostyle.gotemplatehelper.gotermenv.gotermenv_other.gotermenv_posix.gotermenv_solaris.gotermenv_unix.gotermenv_windows.go
munnerz
goautoneg
opencontainers
go-digest
.mailmap.pullapprove.yml.travis.ymlCONTRIBUTING.mdLICENSELICENSE.docsMAINTAINERSREADME.mdalgorithm.godigest.godigester.godoc.goverifiers.go
image-spec
pjbgf
sha1cd
pkg
pmezard
go-difflib
prometheus
client_golang
LICENSENOTICE
prometheus
.gitignoreREADME.mdbuild_info_collector.gocollector.gocounter.godesc.godoc.goexpvar_collector.gofnv.gogauge.goget_pid.goget_pid_gopherjs.gogo_collector.gogo_collector_go116.gogo_collector_latest.gohistogram.go
internal
labels.gometric.gonum_threads.gonum_threads_gopherjs.goobserver.goprocess_collector.goprocess_collector_js.goprocess_collector_other.goprocess_collector_wasip1.goprocess_collector_windows.gopromhttp
registry.gosummary.gotimer.gountyped.govalue.govec.govnext.gowrap.goclient_model
common
procfs
.gitignore.golangci.ymlCODE_OF_CONDUCT.mdCONTRIBUTING.mdLICENSEMAINTAINERS.mdMakefileMakefile.commonNOTICEREADME.mdSECURITY.mdarp.gobuddyinfo.gocmdline.gocpuinfo.gocpuinfo_armx.gocpuinfo_loong64.gocpuinfo_mipsx.gocpuinfo_others.gocpuinfo_ppcx.gocpuinfo_riscvx.gocpuinfo_s390x.gocpuinfo_x86.gocrypto.godoc.gofs.gofs_statfs_notype.gofs_statfs_type.gofscache.go
internal
ipvs.gokernel_random.goloadavg.gomdstat.gomeminfo.gomountinfo.gomountstats.gonet_conntrackstat.gonet_dev.gonet_ip_socket.gonet_protocols.gonet_route.gonet_sockstat.gonet_softnet.gonet_tcp.gonet_tls_stat.gonet_udp.gonet_unix.gonet_wireless.gonet_xfrm.gonetstat.goproc.goproc_cgroup.goproc_cgroups.goproc_environ.goproc_fdinfo.goproc_interrupts.goproc_io.goproc_limits.goproc_maps.goproc_netstat.goproc_ns.goproc_psi.goproc_smaps.goproc_snmp.goproc_snmp6.goproc_stat.goproc_status.goproc_sys.goschedstat.goslab.gosoftirqs.gostat.goswaps.gothread.gottarvm.gozoneinfo.gorivo
uniseg
russross
blackfriday
schollz
sergi
go-diff
sirupsen
logrus
.gitignore.golangci.yml.travis.ymlCHANGELOG.mdLICENSEREADME.mdalt_exit.goappveyor.ymlbuffer_pool.godoc.goentry.goexported.goformatter.gohooks.gojson_formatter.gologger.gologrus.goterminal_check_appengine.goterminal_check_bsd.goterminal_check_js.goterminal_check_no_terminal.goterminal_check_notappengine.goterminal_check_solaris.goterminal_check_unix.goterminal_check_windows.gotext_formatter.gowriter.go
skeema
knownhosts
spf13
cobra
.gitignore.golangci.yml.mailmapCONDUCT.mdCONTRIBUTING.mdLICENSE.txtMAINTAINERSMakefileREADME.mdactive_help.goargs.gobash_completions.gobash_completionsV2.gocobra.gocommand.gocommand_notwin.gocommand_win.gocompletions.gofish_completions.goflag_groups.gopowershell_completions.goshell_completions.gozsh_completions.go
pflag
.gitignore.travis.ymlLICENSEREADME.mdbool.gobool_slice.gobytes.gocount.goduration.goduration_slice.goflag.gofloat32.gofloat32_slice.gofloat64.gofloat64_slice.gogolangflag.goint.goint16.goint32.goint32_slice.goint64.goint64_slice.goint8.goint_slice.goip.goip_slice.goipmask.goipnet.gostring.gostring_array.gostring_slice.gostring_to_int.gostring_to_int64.gostring_to_string.gouint.gouint16.gouint32.gouint64.gouint8.gouint_slice.go
stretchr
testify
theupdateframework
notary
.gitignoreCHANGELOG.mdCODE_OF_CONDUCT.mdCONTRIBUTING.mdCONTRIBUTORSDockerfileJenkinsfileLICENSEMAINTAINERSMAINTAINERS.ALUMNIMAINTAINERS_RULES.mdMakefileNOTARY_VERSIONREADME.mdcodecov.ymlconst.goconst_nowindows.goconst_windows.gocross.Dockerfile
client
changelist
client.godelegations.goerrors.gohelpers.gointerface.goreader.gorepo.gorepo_pkcs11.gotufclient.gowitness.gocryptoservice
development.mysql.ymldevelopment.postgresql.ymldevelopment.rethink.ymldocker-compose.postgresql.ymldocker-compose.rethink.ymldocker-compose.ymlescrow.Dockerfilefips.gonotary.gopassphrase
server.Dockerfileserver.minimal.Dockerfilesigner.Dockerfilesigner.minimal.Dockerfilestorage
trustmanager
trustpinning
tuf
urfave
cli
.flake8.gitignoreCODE_OF_CONDUCT.mdLICENSEREADME.mdapp.gocategory.gocli.gocommand.gocontext.godocs.goerrors.gofish.goflag.goflag_bool.goflag_bool_t.goflag_duration.goflag_float64.goflag_generic.goflag_int.goflag_int64.goflag_int64_slice.goflag_int_slice.goflag_string.goflag_string_slice.goflag_uint.goflag_uint64.gofuncs.gohelp.goparse.gosort.gotemplate.go
xanzy
xeipuuv
gojsonpointer
gojsonreference
gojsonschema
go.opentelemetry.io
contrib
instrumentation
net
http
otel
.codespellignore.codespellrc.gitattributes.gitignore.golangci.yml.lycheeignore.markdownlint.yamlCHANGELOG.mdCODEOWNERSCONTRIBUTING.mdLICENSEMakefileREADME.mdRELEASING.mdVERSIONING.mdget_main_pkgs.shhandler.gotrace.go
attribute
baggage
codes
doc.goerror_handler.goexporters
otlp
otlpmetric
otlpmetricgrpc
otlptrace
LICENSEREADME.mdclients.godoc.goexporter.go
internal
tracetransform
otlptracegrpc
LICENSEREADME.mdclient.godoc.goexporter.go
version.gointernal
options.gointernal
internal_logging.gometric.gometric
LICENSEREADME.mdasyncfloat64.goasyncint64.goconfig.godoc.go
propagation.goembedded
instrument.gometer.gonoop
syncfloat64.gosyncint64.gopropagation
renovate.jsonrequirements.txtsdk
LICENSEREADME.md
instrumentation
internal
metric
LICENSEREADME.mdaggregation.gocache.goconfig.godoc.goenv.goexemplar.goexporter.goinstrument.goinstrumentkind_string.go
internal
manual_reader.gometer.gometricdata
periodic_reader.gopipeline.goprovider.goreader.goversion.goview.goresource
README.mdauto.gobuiltin.goconfig.gocontainer.godoc.goenv.gohost_id.gohost_id_bsd.gohost_id_darwin.gohost_id_exec.gohost_id_linux.gohost_id_readfile.gohost_id_unsupported.gohost_id_windows.goos.goos_release_darwin.goos_release_unix.goos_unix.goos_unsupported.goos_windows.goprocess.goresource.go
trace
README.mdbatch_span_processor.godoc.goevent.goevictedqueue.goid_generator.golink.goprovider.gosampler_env.gosampling.gosimple_span_processor.gosnapshot.gospan.gospan_exporter.gospan_limits.gospan_processor.gotracer.goversion.go
version.gosemconv
v1.20.0
v1.21.0
v1.24.0
v1.26.0
trace
verify_examples.shverify_readmes.shversion.goversions.yamlproto
otlp
golang.org
x
crypto
LICENSEPATENTS
argon2
blake2b
blake2b.goblake2bAVX2_amd64.goblake2bAVX2_amd64.sblake2b_amd64.sblake2b_generic.goblake2b_ref.goblake2x.goregister.go
blowfish
cast5
chacha20
chacha_arm64.gochacha_arm64.schacha_generic.gochacha_noasm.gochacha_ppc64le.gochacha_ppc64le.schacha_s390x.gochacha_s390x.sxor.go
curve25519
ed25519
hkdf
internal
alias
poly1305
pbkdf2
sha3
doc.gohashes.gohashes_noasm.gokeccakf.gokeccakf_amd64.gokeccakf_amd64.ssha3.gosha3_s390x.gosha3_s390x.sshake.goshake_noasm.goxor.go
ssh
exp
net
LICENSEPATENTS
context
http
httpguts
http2
.gitignoreascii.gociphers.goclient_conn_pool.godatabuffer.goerrors.goflow.goframe.gogotrack.goheadermap.go
hpack
http2.gopipe.goserver.gotimer.gotransport.gowrite.gowritesched.gowritesched_priority.gowritesched_random.gowritesched_roundrobin.goidna
go118.goidna10.0.0.goidna9.0.0.gopre_go118.gopunycode.gotables10.0.0.gotables11.0.0.gotables12.0.0.gotables13.0.0.gotables15.0.0.gotables9.0.0.gotrie.gotrie12.0.0.gotrie13.0.0.gotrieval.go
internal
proxy
trace
sync
sys
LICENSEPATENTS
cpu
asm_aix_ppc64.sbyteorder.gocpu.gocpu_aix.gocpu_arm.gocpu_arm64.gocpu_arm64.scpu_gc_arm64.gocpu_gc_s390x.gocpu_gc_x86.gocpu_gccgo_arm64.gocpu_gccgo_s390x.gocpu_gccgo_x86.ccpu_gccgo_x86.gocpu_linux.gocpu_linux_arm.gocpu_linux_arm64.gocpu_linux_mips64x.gocpu_linux_noinit.gocpu_linux_ppc64x.gocpu_linux_s390x.gocpu_loong64.gocpu_mips64x.gocpu_mipsx.gocpu_netbsd_arm64.gocpu_openbsd_arm64.gocpu_openbsd_arm64.scpu_other_arm.gocpu_other_arm64.gocpu_other_mips64x.gocpu_other_ppc64x.gocpu_other_riscv64.gocpu_ppc64x.gocpu_riscv64.gocpu_s390x.gocpu_s390x.scpu_wasm.gocpu_x86.gocpu_x86.scpu_zos.gocpu_zos_s390x.goendian_big.goendian_little.gohwcap_linux.goparse.goproc_cpuinfo_linux.goruntime_auxv.goruntime_auxv_go121.gosyscall_aix_gccgo.gosyscall_aix_ppc64_gc.go
execabs
plan9
asm.sasm_plan9_386.sasm_plan9_amd64.sasm_plan9_arm.sconst_plan9.godir_plan9.goenv_plan9.goerrors_plan9.gomkall.shmkerrors.shmksysnum_plan9.shpwd_go15_plan9.gopwd_plan9.gorace.gorace0.gostr.gosyscall.gosyscall_plan9.gozsyscall_plan9_386.gozsyscall_plan9_amd64.gozsyscall_plan9_arm.gozsysnum_plan9.go
unix
.gitignoreREADME.mdaffinity_linux.goaliases.goasm_aix_ppc64.sasm_bsd_386.sasm_bsd_amd64.sasm_bsd_arm.sasm_bsd_arm64.sasm_bsd_ppc64.sasm_bsd_riscv64.sasm_linux_386.sasm_linux_amd64.sasm_linux_arm.sasm_linux_arm64.sasm_linux_loong64.sasm_linux_mips64x.sasm_linux_mipsx.sasm_linux_ppc64x.sasm_linux_riscv64.sasm_linux_s390x.sasm_openbsd_mips64.sasm_solaris_amd64.sasm_zos_s390x.sbluetooth_linux.gobpxsvc_zos.gobpxsvc_zos.scap_freebsd.goconstants.godev_aix_ppc.godev_aix_ppc64.godev_darwin.godev_dragonfly.godev_freebsd.godev_linux.godev_netbsd.godev_openbsd.godev_zos.godirent.goendian_big.goendian_little.goenv_unix.gofcntl.gofcntl_darwin.gofcntl_linux_32bit.gofdset.gogccgo.gogccgo_c.cgccgo_linux_amd64.goifreq_linux.goioctl_linux.goioctl_signed.goioctl_unsigned.goioctl_zos.gomkall.shmkerrors.shmmap_nomremap.gomremap.gopagesize_unix.gopledge_openbsd.goptrace_darwin.goptrace_ios.gorace.gorace0.goreaddirent_getdents.goreaddirent_getdirentries.gosockcmsg_dragonfly.gosockcmsg_linux.gosockcmsg_unix.gosockcmsg_unix_other.gosockcmsg_zos.gosymaddr_zos_s390x.ssyscall.gosyscall_aix.gosyscall_aix_ppc.gosyscall_aix_ppc64.gosyscall_bsd.gosyscall_darwin.gosyscall_darwin_amd64.gosyscall_darwin_arm64.gosyscall_darwin_libSystem.gosyscall_dragonfly.gosyscall_dragonfly_amd64.gosyscall_freebsd.gosyscall_freebsd_386.gosyscall_freebsd_amd64.gosyscall_freebsd_arm.gosyscall_freebsd_arm64.gosyscall_freebsd_riscv64.gosyscall_hurd.gosyscall_hurd_386.gosyscall_illumos.gosyscall_linux.gosyscall_linux_386.gosyscall_linux_alarm.gosyscall_linux_amd64.gosyscall_linux_amd64_gc.gosyscall_linux_arm.gosyscall_linux_arm64.gosyscall_linux_gc.gosyscall_linux_gc_386.gosyscall_linux_gc_arm.gosyscall_linux_gccgo_386.gosyscall_linux_gccgo_arm.gosyscall_linux_loong64.gosyscall_linux_mips64x.gosyscall_linux_mipsx.gosyscall_linux_ppc.gosyscall_linux_ppc64x.gosyscall_linux_riscv64.gosyscall_linux_s390x.gosyscall_linux_sparc64.gosyscall_netbsd.gosyscall_netbsd_386.gosyscall_netbsd_amd64.gosyscall_netbsd_arm.gosyscall_netbsd_arm64.gosyscall_openbsd.gosyscall_openbsd_386.gosyscall_openbsd_amd64.gosyscall_openbsd_arm.gosyscall_openbsd_arm64.gosyscall_openbsd_libc.gosyscall_openbsd_mips64.gosyscall_openbsd_ppc64.gosyscall_openbsd_riscv64.gosyscall_solaris.gosyscall_solaris_amd64.gosyscall_unix.gosyscall_unix_gc.gosyscall_unix_gc_ppc64x.gosyscall_zos_s390x.gosysvshm_linux.gosysvshm_unix.gosysvshm_unix_other.gotimestruct.gounveil_openbsd.goxattr_bsd.gozerrors_aix_ppc.gozerrors_aix_ppc64.gozerrors_darwin_amd64.gozerrors_darwin_arm64.gozerrors_dragonfly_amd64.gozerrors_freebsd_386.gozerrors_freebsd_amd64.gozerrors_freebsd_arm.gozerrors_freebsd_arm64.gozerrors_freebsd_riscv64.gozerrors_linux.gozerrors_linux_386.gozerrors_linux_amd64.gozerrors_linux_arm.gozerrors_linux_arm64.gozerrors_linux_loong64.gozerrors_linux_mips.gozerrors_linux_mips64.gozerrors_linux_mips64le.gozerrors_linux_mipsle.gozerrors_linux_ppc.gozerrors_linux_ppc64.gozerrors_linux_ppc64le.gozerrors_linux_riscv64.gozerrors_linux_s390x.gozerrors_linux_sparc64.gozerrors_netbsd_386.gozerrors_netbsd_amd64.gozerrors_netbsd_arm.gozerrors_netbsd_arm64.gozerrors_openbsd_386.gozerrors_openbsd_amd64.gozerrors_openbsd_arm.gozerrors_openbsd_arm64.gozerrors_openbsd_mips64.gozerrors_openbsd_ppc64.gozerrors_openbsd_riscv64.gozerrors_solaris_amd64.gozerrors_zos_s390x.gozptrace_armnn_linux.gozptrace_linux_arm64.gozptrace_mipsnn_linux.gozptrace_mipsnnle_linux.gozptrace_x86_linux.gozsymaddr_zos_s390x.szsyscall_aix_ppc.gozsyscall_aix_ppc64.gozsyscall_aix_ppc64_gc.gozsyscall_aix_ppc64_gccgo.gozsyscall_darwin_amd64.gozsyscall_darwin_amd64.szsyscall_darwin_arm64.gozsyscall_darwin_arm64.szsyscall_dragonfly_amd64.gozsyscall_freebsd_386.gozsyscall_freebsd_amd64.gozsyscall_freebsd_arm.gozsyscall_freebsd_arm64.gozsyscall_freebsd_riscv64.gozsyscall_illumos_amd64.gozsyscall_linux.gozsyscall_linux_386.gozsyscall_linux_amd64.gozsyscall_linux_arm.gozsyscall_linux_arm64.gozsyscall_linux_loong64.gozsyscall_linux_mips.gozsyscall_linux_mips64.gozsyscall_linux_mips64le.gozsyscall_linux_mipsle.gozsyscall_linux_ppc.gozsyscall_linux_ppc64.gozsyscall_linux_ppc64le.gozsyscall_linux_riscv64.gozsyscall_linux_s390x.gozsyscall_linux_sparc64.gozsyscall_netbsd_386.gozsyscall_netbsd_amd64.gozsyscall_netbsd_arm.gozsyscall_netbsd_arm64.gozsyscall_openbsd_386.gozsyscall_openbsd_386.szsyscall_openbsd_amd64.gozsyscall_openbsd_amd64.szsyscall_openbsd_arm.gozsyscall_openbsd_arm.szsyscall_openbsd_arm64.gozsyscall_openbsd_arm64.szsyscall_openbsd_mips64.gozsyscall_openbsd_mips64.szsyscall_openbsd_ppc64.gozsyscall_openbsd_ppc64.szsyscall_openbsd_riscv64.gozsyscall_openbsd_riscv64.szsyscall_solaris_amd64.gozsyscall_zos_s390x.gozsysctl_openbsd_386.gozsysctl_openbsd_amd64.gozsysctl_openbsd_arm.gozsysctl_openbsd_arm64.gozsysctl_openbsd_mips64.gozsysctl_openbsd_ppc64.gozsysctl_openbsd_riscv64.gozsysnum_darwin_amd64.gozsysnum_darwin_arm64.gozsysnum_dragonfly_amd64.gozsysnum_freebsd_386.gozsysnum_freebsd_amd64.gozsysnum_freebsd_arm.gozsysnum_freebsd_arm64.gozsysnum_freebsd_riscv64.gozsysnum_linux_386.gozsysnum_linux_amd64.gozsysnum_linux_arm.gozsysnum_linux_arm64.gozsysnum_linux_loong64.gozsysnum_linux_mips.gozsysnum_linux_mips64.gozsysnum_linux_mips64le.gozsysnum_linux_mipsle.gozsysnum_linux_ppc.gozsysnum_linux_ppc64.gozsysnum_linux_ppc64le.gozsysnum_linux_riscv64.gozsysnum_linux_s390x.gozsysnum_linux_sparc64.gozsysnum_netbsd_386.gozsysnum_netbsd_amd64.gozsysnum_netbsd_arm.gozsysnum_netbsd_arm64.gozsysnum_openbsd_386.gozsysnum_openbsd_amd64.gozsysnum_openbsd_arm.gozsysnum_openbsd_arm64.gozsysnum_openbsd_mips64.gozsysnum_openbsd_ppc64.gozsysnum_openbsd_riscv64.gozsysnum_zos_s390x.goztypes_aix_ppc.goztypes_aix_ppc64.goztypes_darwin_amd64.goztypes_darwin_arm64.goztypes_dragonfly_amd64.goztypes_freebsd_386.goztypes_freebsd_amd64.goztypes_freebsd_arm.goztypes_freebsd_arm64.goztypes_freebsd_riscv64.goztypes_linux.goztypes_linux_386.goztypes_linux_amd64.goztypes_linux_arm.goztypes_linux_arm64.goztypes_linux_loong64.goztypes_linux_mips.goztypes_linux_mips64.goztypes_linux_mips64le.goztypes_linux_mipsle.goztypes_linux_ppc.goztypes_linux_ppc64.goztypes_linux_ppc64le.goztypes_linux_riscv64.goztypes_linux_s390x.goztypes_linux_sparc64.goztypes_netbsd_386.goztypes_netbsd_amd64.goztypes_netbsd_arm.goztypes_netbsd_arm64.goztypes_openbsd_386.goztypes_openbsd_amd64.goztypes_openbsd_arm.goztypes_openbsd_arm64.goztypes_openbsd_mips64.goztypes_openbsd_ppc64.goztypes_openbsd_riscv64.goztypes_solaris_amd64.goztypes_zos_s390x.go
windows
aliases.godll_windows.goenv_windows.goeventlog.goexec_windows.gomemory_windows.gomkerrors.bashmkknownfolderids.bashmksyscall.gorace.gorace0.go
registry
security_windows.goservice.gosetupapi_windows.gostr.gosyscall.gosyscall_windows.gotypes_windows.gotypes_windows_386.gotypes_windows_amd64.gotypes_windows_arm.gotypes_windows_arm64.gozerrors_windows.gozknownfolderids_windows.gozsyscall_windows.goterm
CONTRIBUTING.mdLICENSEPATENTSREADME.mdcodereview.cfgterm.goterm_plan9.goterm_unix.goterm_unix_bsd.goterm_unix_other.goterm_unsupported.goterm_windows.goterminal.go
text
LICENSEPATENTS
cases
cases.gocontext.gofold.goicu.goinfo.gomap.gotables10.0.0.gotables11.0.0.gotables12.0.0.gotables13.0.0.gotables15.0.0.gotables9.0.0.gotrieval.go
internal
internal.go
language
match.gotag
language
secure
transform
unicode
bidi
bidi.gobracket.gocore.goprop.gotables10.0.0.gotables11.0.0.gotables12.0.0.gotables13.0.0.gotables15.0.0.gotables9.0.0.gotrieval.go
norm
width
time
google.golang.org
genproto
googleapis
grpc
AUTHORSCODE-OF-CONDUCT.mdCONTRIBUTING.mdGOVERNANCE.mdLICENSEMAINTAINERS.mdMakefileNOTICE.txtREADME.mdSECURITY.mdcall.gointerceptor.go
attributes
backoff.gobackoff
balancer
balancer_wrapper.gobinarylog
grpc_binarylog_v1
channelz
clientconn.gocodec.gocodes
connectivity
credentials
dialoptions.godoc.goencoding
grpclog
health
grpc_health_v1
internal
backoff
balancer
gracefulswitch
balancerload
binarylog
buffer
channelz
channel.gochannelmap.gofuncs.gologging.goserver.gosocket.gosubchannel.gosyscall_linux.gosyscall_nonlinux.gotrace.go
credentials
envconfig
experimental.gogrpclog
grpcsync
grpcutil
idle
internal.gometadata
pretty
resolver
serviceconfig
status
syscall
tcp_keepalive_others.gotcp_keepalive_unix.gotcp_keepalive_windows.gotransport
keepalive
metadata
peer
picker_wrapper.gopreloader.goregenerate.shresolver
resolver_wrapper.gorpc_util.goserver.goservice_config.goserviceconfig
shared_buffer_pool.gostats
status
stream.gostream_interfaces.gotap
trace.gotrace_notrace.gotrace_withtrace.goversion.goprotobuf
LICENSEPATENTS
encoding
protodelim
protojson
prototext
protowire
internal
descfmt
descopts
detrand
editiondefaults
encoding
defval
json
messageset
tag
text
errors
filedesc
filetype
flags
genid
any_gen.goapi_gen.godescriptor_gen.godoc.goduration_gen.goempty_gen.gofield_mask_gen.gogo_features_gen.gogoname.gomap_entry.gosource_context_gen.gostruct_gen.gotimestamp_gen.gotype_gen.gowrappers.gowrappers_gen.go
impl
api_export.gocheckinit.gocodec_extension.gocodec_field.gocodec_gen.gocodec_map.gocodec_map_go111.gocodec_map_go112.gocodec_message.gocodec_messageset.gocodec_reflect.gocodec_tables.gocodec_unsafe.goconvert.goconvert_list.goconvert_map.godecode.goencode.goenum.goextension.golegacy_enum.golegacy_export.golegacy_extension.golegacy_file.golegacy_message.gomerge.gomerge_gen.gomessage.gomessage_reflect.gomessage_reflect_field.gomessage_reflect_gen.gopointer_reflect.gopointer_unsafe.govalidate.goweak.go
order
pragma
set
strs
version
proto
checkinit.godecode.godecode_gen.godoc.goencode.goencode_gen.goequal.goextension.gomerge.gomessageset.goproto.goproto_methods.goproto_reflect.goreset.gosize.gosize_gen.gowrappers.go
protoadapt
reflect
protoreflect
methods.goproto.gosource.gosource_gen.gotype.govalue.govalue_equal.govalue_pure.govalue_union.govalue_unsafe_go120.govalue_unsafe_go121.go
protoregistry
runtime
types
known
anypb
durationpb
fieldmaskpb
structpb
timestamppb
wrapperspb
gopkg.in
warnings.v0
yaml.v2
.travis.ymlLICENSELICENSE.libyamlNOTICEREADME.mdapic.godecode.goemitterc.goencode.goparserc.goreaderc.goresolve.goscannerc.gosorter.gowriterc.goyaml.goyamlh.goyamlprivateh.go
yaml.v3
gotest.tools
v3
79
vendor/github.com/klauspost/compress/fse/README.md
generated
vendored
Normal file
79
vendor/github.com/klauspost/compress/fse/README.md
generated
vendored
Normal file
@ -0,0 +1,79 @@
|
||||
# Finite State Entropy
|
||||
|
||||
This package provides Finite State Entropy encoding and decoding.
|
||||
|
||||
Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS))
|
||||
encoding provides a fast near-optimal symbol encoding/decoding
|
||||
for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
|
||||
|
||||
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
|
||||
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
|
||||
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
|
||||
|
||||
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
|
||||
|
||||
## News
|
||||
|
||||
* Feb 2018: First implementation released. Consider this beta software for now.
|
||||
|
||||
# Usage
|
||||
|
||||
This package provides a low level interface that allows to compress single independent blocks.
|
||||
|
||||
Each block is separate, and there is no built in integrity checks.
|
||||
This means that the caller should keep track of block sizes and also do checksums if needed.
|
||||
|
||||
Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
|
||||
You must provide input and will receive the output and maybe an error.
|
||||
|
||||
These error values can be returned:
|
||||
|
||||
| Error | Description |
|
||||
|---------------------|-----------------------------------------------------------------------------|
|
||||
| `<nil>` | Everything ok, output is returned |
|
||||
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
|
||||
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
|
||||
| `(error)` | An internal error occurred. |
|
||||
|
||||
As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
|
||||
|
||||
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object
|
||||
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
|
||||
object can be used for both.
|
||||
|
||||
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
|
||||
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
|
||||
|
||||
Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
|
||||
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
|
||||
your input was likely corrupted.
|
||||
|
||||
It is important to note that a successful decoding does *not* mean your output matches your original input.
|
||||
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
|
||||
|
||||
For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
|
||||
|
||||
# Performance
|
||||
|
||||
A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.
|
||||
All compression functions are currently only running on the calling goroutine so only one core will be used per block.
|
||||
|
||||
The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
|
||||
is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be
|
||||
beneficial to transpose all your input values down by 64.
|
||||
|
||||
With moderate block sizes around 64k speed are typically 200MB/s per core for compression and
|
||||
around 300MB/s decompression speed.
|
||||
|
||||
The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s.
|
||||
|
||||
# Plans
|
||||
|
||||
At one point, more internals will be exposed to facilitate more "expert" usage of the components.
|
||||
|
||||
A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).
|
||||
|
||||
# Contributing
|
||||
|
||||
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
|
||||
changes will likely not be accepted. If in doubt open an issue before writing the PR.
|
122
vendor/github.com/klauspost/compress/fse/bitreader.go
generated
vendored
Normal file
122
vendor/github.com/klauspost/compress/fse/bitreader.go
generated
vendored
Normal file
@ -0,0 +1,122 @@
|
||||
// Copyright 2018 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
// Based on work Copyright (c) 2013, Yann Collet, released under BSD License.
|
||||
|
||||
package fse
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// bitReader reads a bitstream in reverse.
|
||||
// The last set bit indicates the start of the stream and is used
|
||||
// for aligning the input.
|
||||
type bitReader struct {
|
||||
in []byte
|
||||
off uint // next byte to read is at in[off - 1]
|
||||
value uint64
|
||||
bitsRead uint8
|
||||
}
|
||||
|
||||
// init initializes and resets the bit reader.
|
||||
func (b *bitReader) init(in []byte) error {
|
||||
if len(in) < 1 {
|
||||
return errors.New("corrupt stream: too short")
|
||||
}
|
||||
b.in = in
|
||||
b.off = uint(len(in))
|
||||
// The highest bit of the last byte indicates where to start
|
||||
v := in[len(in)-1]
|
||||
if v == 0 {
|
||||
return errors.New("corrupt stream, did not find end of stream")
|
||||
}
|
||||
b.bitsRead = 64
|
||||
b.value = 0
|
||||
if len(in) >= 8 {
|
||||
b.fillFastStart()
|
||||
} else {
|
||||
b.fill()
|
||||
b.fill()
|
||||
}
|
||||
b.bitsRead += 8 - uint8(highBits(uint32(v)))
|
||||
return nil
|
||||
}
|
||||
|
||||
// getBits will return n bits. n can be 0.
|
||||
func (b *bitReader) getBits(n uint8) uint16 {
|
||||
if n == 0 || b.bitsRead >= 64 {
|
||||
return 0
|
||||
}
|
||||
return b.getBitsFast(n)
|
||||
}
|
||||
|
||||
// getBitsFast requires that at least one bit is requested every time.
|
||||
// There are no checks if the buffer is filled.
|
||||
func (b *bitReader) getBitsFast(n uint8) uint16 {
|
||||
const regMask = 64 - 1
|
||||
v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask))
|
||||
b.bitsRead += n
|
||||
return v
|
||||
}
|
||||
|
||||
// fillFast() will make sure at least 32 bits are available.
|
||||
// There must be at least 4 bytes available.
|
||||
func (b *bitReader) fillFast() {
|
||||
if b.bitsRead < 32 {
|
||||
return
|
||||
}
|
||||
// 2 bounds checks.
|
||||
v := b.in[b.off-4:]
|
||||
v = v[:4]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
b.value = (b.value << 32) | uint64(low)
|
||||
b.bitsRead -= 32
|
||||
b.off -= 4
|
||||
}
|
||||
|
||||
// fill() will make sure at least 32 bits are available.
|
||||
func (b *bitReader) fill() {
|
||||
if b.bitsRead < 32 {
|
||||
return
|
||||
}
|
||||
if b.off > 4 {
|
||||
v := b.in[b.off-4:]
|
||||
v = v[:4]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
b.value = (b.value << 32) | uint64(low)
|
||||
b.bitsRead -= 32
|
||||
b.off -= 4
|
||||
return
|
||||
}
|
||||
for b.off > 0 {
|
||||
b.value = (b.value << 8) | uint64(b.in[b.off-1])
|
||||
b.bitsRead -= 8
|
||||
b.off--
|
||||
}
|
||||
}
|
||||
|
||||
// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read.
|
||||
func (b *bitReader) fillFastStart() {
|
||||
// Do single re-slice to avoid bounds checks.
|
||||
b.value = binary.LittleEndian.Uint64(b.in[b.off-8:])
|
||||
b.bitsRead = 0
|
||||
b.off -= 8
|
||||
}
|
||||
|
||||
// finished returns true if all bits have been read from the bit stream.
|
||||
func (b *bitReader) finished() bool {
|
||||
return b.bitsRead >= 64 && b.off == 0
|
||||
}
|
||||
|
||||
// close the bitstream and returns an error if out-of-buffer reads occurred.
|
||||
func (b *bitReader) close() error {
|
||||
// Release reference.
|
||||
b.in = nil
|
||||
if b.bitsRead > 64 {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
167
vendor/github.com/klauspost/compress/fse/bitwriter.go
generated
vendored
Normal file
167
vendor/github.com/klauspost/compress/fse/bitwriter.go
generated
vendored
Normal file
@ -0,0 +1,167 @@
|
||||
// Copyright 2018 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
// Based on work Copyright (c) 2013, Yann Collet, released under BSD License.
|
||||
|
||||
package fse
|
||||
|
||||
import "fmt"
|
||||
|
||||
// bitWriter will write bits.
|
||||
// First bit will be LSB of the first byte of output.
|
||||
type bitWriter struct {
|
||||
bitContainer uint64
|
||||
nBits uint8
|
||||
out []byte
|
||||
}
|
||||
|
||||
// bitMask16 is bitmasks. Has extra to avoid bounds check.
|
||||
var bitMask16 = [32]uint16{
|
||||
0, 1, 3, 7, 0xF, 0x1F,
|
||||
0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF,
|
||||
0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0xFFFF,
|
||||
0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF,
|
||||
0xFFFF, 0xFFFF} /* up to 16 bits */
|
||||
|
||||
// addBits16NC will add up to 16 bits.
|
||||
// It will not check if there is space for them,
|
||||
// so the caller must ensure that it has flushed recently.
|
||||
func (b *bitWriter) addBits16NC(value uint16, bits uint8) {
|
||||
b.bitContainer |= uint64(value&bitMask16[bits&31]) << (b.nBits & 63)
|
||||
b.nBits += bits
|
||||
}
|
||||
|
||||
// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated.
|
||||
// It will not check if there is space for them, so the caller must ensure that it has flushed recently.
|
||||
func (b *bitWriter) addBits16Clean(value uint16, bits uint8) {
|
||||
b.bitContainer |= uint64(value) << (b.nBits & 63)
|
||||
b.nBits += bits
|
||||
}
|
||||
|
||||
// addBits16ZeroNC will add up to 16 bits.
|
||||
// It will not check if there is space for them,
|
||||
// so the caller must ensure that it has flushed recently.
|
||||
// This is fastest if bits can be zero.
|
||||
func (b *bitWriter) addBits16ZeroNC(value uint16, bits uint8) {
|
||||
if bits == 0 {
|
||||
return
|
||||
}
|
||||
value <<= (16 - bits) & 15
|
||||
value >>= (16 - bits) & 15
|
||||
b.bitContainer |= uint64(value) << (b.nBits & 63)
|
||||
b.nBits += bits
|
||||
}
|
||||
|
||||
// flush will flush all pending full bytes.
|
||||
// There will be at least 56 bits available for writing when this has been called.
|
||||
// Using flush32 is faster, but leaves less space for writing.
|
||||
func (b *bitWriter) flush() {
|
||||
v := b.nBits >> 3
|
||||
switch v {
|
||||
case 0:
|
||||
case 1:
|
||||
b.out = append(b.out,
|
||||
byte(b.bitContainer),
|
||||
)
|
||||
case 2:
|
||||
b.out = append(b.out,
|
||||
byte(b.bitContainer),
|
||||
byte(b.bitContainer>>8),
|
||||
)
|
||||
case 3:
|
||||
b.out = append(b.out,
|
||||
byte(b.bitContainer),
|
||||
byte(b.bitContainer>>8),
|
||||
byte(b.bitContainer>>16),
|
||||
)
|
||||
case 4:
|
||||
b.out = append(b.out,
|
||||
byte(b.bitContainer),
|
||||
byte(b.bitContainer>>8),
|
||||
byte(b.bitContainer>>16),
|
||||
byte(b.bitContainer>>24),
|
||||
)
|
||||
case 5:
|
||||
b.out = append(b.out,
|
||||
byte(b.bitContainer),
|
||||
byte(b.bitContainer>>8),
|
||||
byte(b.bitContainer>>16),
|
||||
byte(b.bitContainer>>24),
|
||||
byte(b.bitContainer>>32),
|
||||
)
|
||||
case 6:
|
||||
b.out = append(b.out,
|
||||
byte(b.bitContainer),
|
||||
byte(b.bitContainer>>8),
|
||||
byte(b.bitContainer>>16),
|
||||
byte(b.bitContainer>>24),
|
||||
byte(b.bitContainer>>32),
|
||||
byte(b.bitContainer>>40),
|
||||
)
|
||||
case 7:
|
||||
b.out = append(b.out,
|
||||
byte(b.bitContainer),
|
||||
byte(b.bitContainer>>8),
|
||||
byte(b.bitContainer>>16),
|
||||
byte(b.bitContainer>>24),
|
||||
byte(b.bitContainer>>32),
|
||||
byte(b.bitContainer>>40),
|
||||
byte(b.bitContainer>>48),
|
||||
)
|
||||
case 8:
|
||||
b.out = append(b.out,
|
||||
byte(b.bitContainer),
|
||||
byte(b.bitContainer>>8),
|
||||
byte(b.bitContainer>>16),
|
||||
byte(b.bitContainer>>24),
|
||||
byte(b.bitContainer>>32),
|
||||
byte(b.bitContainer>>40),
|
||||
byte(b.bitContainer>>48),
|
||||
byte(b.bitContainer>>56),
|
||||
)
|
||||
default:
|
||||
panic(fmt.Errorf("bits (%d) > 64", b.nBits))
|
||||
}
|
||||
b.bitContainer >>= v << 3
|
||||
b.nBits &= 7
|
||||
}
|
||||
|
||||
// flush32 will flush out, so there are at least 32 bits available for writing.
|
||||
func (b *bitWriter) flush32() {
|
||||
if b.nBits < 32 {
|
||||
return
|
||||
}
|
||||
b.out = append(b.out,
|
||||
byte(b.bitContainer),
|
||||
byte(b.bitContainer>>8),
|
||||
byte(b.bitContainer>>16),
|
||||
byte(b.bitContainer>>24))
|
||||
b.nBits -= 32
|
||||
b.bitContainer >>= 32
|
||||
}
|
||||
|
||||
// flushAlign will flush remaining full bytes and align to next byte boundary.
|
||||
func (b *bitWriter) flushAlign() {
|
||||
nbBytes := (b.nBits + 7) >> 3
|
||||
for i := uint8(0); i < nbBytes; i++ {
|
||||
b.out = append(b.out, byte(b.bitContainer>>(i*8)))
|
||||
}
|
||||
b.nBits = 0
|
||||
b.bitContainer = 0
|
||||
}
|
||||
|
||||
// close will write the alignment bit and write the final byte(s)
|
||||
// to the output.
|
||||
func (b *bitWriter) close() {
|
||||
// End mark
|
||||
b.addBits16Clean(1, 1)
|
||||
// flush until next byte.
|
||||
b.flushAlign()
|
||||
}
|
||||
|
||||
// reset and continue writing by appending to out.
|
||||
func (b *bitWriter) reset(out []byte) {
|
||||
b.bitContainer = 0
|
||||
b.nBits = 0
|
||||
b.out = out
|
||||
}
|
47
vendor/github.com/klauspost/compress/fse/bytereader.go
generated
vendored
Normal file
47
vendor/github.com/klauspost/compress/fse/bytereader.go
generated
vendored
Normal file
@ -0,0 +1,47 @@
|
||||
// Copyright 2018 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
// Based on work Copyright (c) 2013, Yann Collet, released under BSD License.
|
||||
|
||||
package fse
|
||||
|
||||
// byteReader provides a byte reader that reads
|
||||
// little endian values from a byte stream.
|
||||
// The input stream is manually advanced.
|
||||
// The reader performs no bounds checks.
|
||||
type byteReader struct {
|
||||
b []byte
|
||||
off int
|
||||
}
|
||||
|
||||
// init will initialize the reader and set the input.
|
||||
func (b *byteReader) init(in []byte) {
|
||||
b.b = in
|
||||
b.off = 0
|
||||
}
|
||||
|
||||
// advance the stream b n bytes.
|
||||
func (b *byteReader) advance(n uint) {
|
||||
b.off += int(n)
|
||||
}
|
||||
|
||||
// Uint32 returns a little endian uint32 starting at current offset.
|
||||
func (b byteReader) Uint32() uint32 {
|
||||
b2 := b.b[b.off:]
|
||||
b2 = b2[:4]
|
||||
v3 := uint32(b2[3])
|
||||
v2 := uint32(b2[2])
|
||||
v1 := uint32(b2[1])
|
||||
v0 := uint32(b2[0])
|
||||
return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24)
|
||||
}
|
||||
|
||||
// unread returns the unread portion of the input.
|
||||
func (b byteReader) unread() []byte {
|
||||
return b.b[b.off:]
|
||||
}
|
||||
|
||||
// remain will return the number of bytes remaining.
|
||||
func (b byteReader) remain() int {
|
||||
return len(b.b) - b.off
|
||||
}
|
683
vendor/github.com/klauspost/compress/fse/compress.go
generated
vendored
Normal file
683
vendor/github.com/klauspost/compress/fse/compress.go
generated
vendored
Normal file
@ -0,0 +1,683 @@
|
||||
// Copyright 2018 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
// Based on work Copyright (c) 2013, Yann Collet, released under BSD License.
|
||||
|
||||
package fse
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Compress the input bytes. Input must be < 2GB.
|
||||
// Provide a Scratch buffer to avoid memory allocations.
|
||||
// Note that the output is also kept in the scratch buffer.
|
||||
// If input is too hard to compress, ErrIncompressible is returned.
|
||||
// If input is a single byte value repeated ErrUseRLE is returned.
|
||||
func Compress(in []byte, s *Scratch) ([]byte, error) {
|
||||
if len(in) <= 1 {
|
||||
return nil, ErrIncompressible
|
||||
}
|
||||
if len(in) > (2<<30)-1 {
|
||||
return nil, errors.New("input too big, must be < 2GB")
|
||||
}
|
||||
s, err := s.prepare(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create histogram, if none was provided.
|
||||
maxCount := s.maxCount
|
||||
if maxCount == 0 {
|
||||
maxCount = s.countSimple(in)
|
||||
}
|
||||
// Reset for next run.
|
||||
s.clearCount = true
|
||||
s.maxCount = 0
|
||||
if maxCount == len(in) {
|
||||
// One symbol, use RLE
|
||||
return nil, ErrUseRLE
|
||||
}
|
||||
if maxCount == 1 || maxCount < (len(in)>>7) {
|
||||
// Each symbol present maximum once or too well distributed.
|
||||
return nil, ErrIncompressible
|
||||
}
|
||||
s.optimalTableLog()
|
||||
err = s.normalizeCount()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = s.writeCount()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if false {
|
||||
err = s.validateNorm()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err = s.buildCTable()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = s.compress(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Out = s.bw.out
|
||||
// Check if we compressed.
|
||||
if len(s.Out) >= len(in) {
|
||||
return nil, ErrIncompressible
|
||||
}
|
||||
return s.Out, nil
|
||||
}
|
||||
|
||||
// cState contains the compression state of a stream.
|
||||
type cState struct {
|
||||
bw *bitWriter
|
||||
stateTable []uint16
|
||||
state uint16
|
||||
}
|
||||
|
||||
// init will initialize the compression state to the first symbol of the stream.
|
||||
func (c *cState) init(bw *bitWriter, ct *cTable, tableLog uint8, first symbolTransform) {
|
||||
c.bw = bw
|
||||
c.stateTable = ct.stateTable
|
||||
|
||||
nbBitsOut := (first.deltaNbBits + (1 << 15)) >> 16
|
||||
im := int32((nbBitsOut << 16) - first.deltaNbBits)
|
||||
lu := (im >> nbBitsOut) + first.deltaFindState
|
||||
c.state = c.stateTable[lu]
|
||||
}
|
||||
|
||||
// encode the output symbol provided and write it to the bitstream.
|
||||
func (c *cState) encode(symbolTT symbolTransform) {
|
||||
nbBitsOut := (uint32(c.state) + symbolTT.deltaNbBits) >> 16
|
||||
dstState := int32(c.state>>(nbBitsOut&15)) + symbolTT.deltaFindState
|
||||
c.bw.addBits16NC(c.state, uint8(nbBitsOut))
|
||||
c.state = c.stateTable[dstState]
|
||||
}
|
||||
|
||||
// encode the output symbol provided and write it to the bitstream.
|
||||
func (c *cState) encodeZero(symbolTT symbolTransform) {
|
||||
nbBitsOut := (uint32(c.state) + symbolTT.deltaNbBits) >> 16
|
||||
dstState := int32(c.state>>(nbBitsOut&15)) + symbolTT.deltaFindState
|
||||
c.bw.addBits16ZeroNC(c.state, uint8(nbBitsOut))
|
||||
c.state = c.stateTable[dstState]
|
||||
}
|
||||
|
||||
// flush will write the tablelog to the output and flush the remaining full bytes.
|
||||
func (c *cState) flush(tableLog uint8) {
|
||||
c.bw.flush32()
|
||||
c.bw.addBits16NC(c.state, tableLog)
|
||||
c.bw.flush()
|
||||
}
|
||||
|
||||
// compress is the main compression loop that will encode the input from the last byte to the first.
|
||||
func (s *Scratch) compress(src []byte) error {
|
||||
if len(src) <= 2 {
|
||||
return errors.New("compress: src too small")
|
||||
}
|
||||
tt := s.ct.symbolTT[:256]
|
||||
s.bw.reset(s.Out)
|
||||
|
||||
// Our two states each encodes every second byte.
|
||||
// Last byte encoded (first byte decoded) will always be encoded by c1.
|
||||
var c1, c2 cState
|
||||
|
||||
// Encode so remaining size is divisible by 4.
|
||||
ip := len(src)
|
||||
if ip&1 == 1 {
|
||||
c1.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-1]])
|
||||
c2.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-2]])
|
||||
c1.encodeZero(tt[src[ip-3]])
|
||||
ip -= 3
|
||||
} else {
|
||||
c2.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-1]])
|
||||
c1.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-2]])
|
||||
ip -= 2
|
||||
}
|
||||
if ip&2 != 0 {
|
||||
c2.encodeZero(tt[src[ip-1]])
|
||||
c1.encodeZero(tt[src[ip-2]])
|
||||
ip -= 2
|
||||
}
|
||||
src = src[:ip]
|
||||
|
||||
// Main compression loop.
|
||||
switch {
|
||||
case !s.zeroBits && s.actualTableLog <= 8:
|
||||
// We can encode 4 symbols without requiring a flush.
|
||||
// We do not need to check if any output is 0 bits.
|
||||
for ; len(src) >= 4; src = src[:len(src)-4] {
|
||||
s.bw.flush32()
|
||||
v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
|
||||
c2.encode(tt[v0])
|
||||
c1.encode(tt[v1])
|
||||
c2.encode(tt[v2])
|
||||
c1.encode(tt[v3])
|
||||
}
|
||||
case !s.zeroBits:
|
||||
// We do not need to check if any output is 0 bits.
|
||||
for ; len(src) >= 4; src = src[:len(src)-4] {
|
||||
s.bw.flush32()
|
||||
v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
|
||||
c2.encode(tt[v0])
|
||||
c1.encode(tt[v1])
|
||||
s.bw.flush32()
|
||||
c2.encode(tt[v2])
|
||||
c1.encode(tt[v3])
|
||||
}
|
||||
case s.actualTableLog <= 8:
|
||||
// We can encode 4 symbols without requiring a flush
|
||||
for ; len(src) >= 4; src = src[:len(src)-4] {
|
||||
s.bw.flush32()
|
||||
v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
|
||||
c2.encodeZero(tt[v0])
|
||||
c1.encodeZero(tt[v1])
|
||||
c2.encodeZero(tt[v2])
|
||||
c1.encodeZero(tt[v3])
|
||||
}
|
||||
default:
|
||||
for ; len(src) >= 4; src = src[:len(src)-4] {
|
||||
s.bw.flush32()
|
||||
v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
|
||||
c2.encodeZero(tt[v0])
|
||||
c1.encodeZero(tt[v1])
|
||||
s.bw.flush32()
|
||||
c2.encodeZero(tt[v2])
|
||||
c1.encodeZero(tt[v3])
|
||||
}
|
||||
}
|
||||
|
||||
// Flush final state.
|
||||
// Used to initialize state when decoding.
|
||||
c2.flush(s.actualTableLog)
|
||||
c1.flush(s.actualTableLog)
|
||||
|
||||
s.bw.close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeCount will write the normalized histogram count to header.
|
||||
// This is read back by readNCount.
|
||||
func (s *Scratch) writeCount() error {
|
||||
var (
|
||||
tableLog = s.actualTableLog
|
||||
tableSize = 1 << tableLog
|
||||
previous0 bool
|
||||
charnum uint16
|
||||
|
||||
maxHeaderSize = ((int(s.symbolLen)*int(tableLog) + 4 + 2) >> 3) + 3
|
||||
|
||||
// Write Table Size
|
||||
bitStream = uint32(tableLog - minTablelog)
|
||||
bitCount = uint(4)
|
||||
remaining = int16(tableSize + 1) /* +1 for extra accuracy */
|
||||
threshold = int16(tableSize)
|
||||
nbBits = uint(tableLog + 1)
|
||||
)
|
||||
if cap(s.Out) < maxHeaderSize {
|
||||
s.Out = make([]byte, 0, s.br.remain()+maxHeaderSize)
|
||||
}
|
||||
outP := uint(0)
|
||||
out := s.Out[:maxHeaderSize]
|
||||
|
||||
// stops at 1
|
||||
for remaining > 1 {
|
||||
if previous0 {
|
||||
start := charnum
|
||||
for s.norm[charnum] == 0 {
|
||||
charnum++
|
||||
}
|
||||
for charnum >= start+24 {
|
||||
start += 24
|
||||
bitStream += uint32(0xFFFF) << bitCount
|
||||
out[outP] = byte(bitStream)
|
||||
out[outP+1] = byte(bitStream >> 8)
|
||||
outP += 2
|
||||
bitStream >>= 16
|
||||
}
|
||||
for charnum >= start+3 {
|
||||
start += 3
|
||||
bitStream += 3 << bitCount
|
||||
bitCount += 2
|
||||
}
|
||||
bitStream += uint32(charnum-start) << bitCount
|
||||
bitCount += 2
|
||||
if bitCount > 16 {
|
||||
out[outP] = byte(bitStream)
|
||||
out[outP+1] = byte(bitStream >> 8)
|
||||
outP += 2
|
||||
bitStream >>= 16
|
||||
bitCount -= 16
|
||||
}
|
||||
}
|
||||
|
||||
count := s.norm[charnum]
|
||||
charnum++
|
||||
max := (2*threshold - 1) - remaining
|
||||
if count < 0 {
|
||||
remaining += count
|
||||
} else {
|
||||
remaining -= count
|
||||
}
|
||||
count++ // +1 for extra accuracy
|
||||
if count >= threshold {
|
||||
count += max // [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[
|
||||
}
|
||||
bitStream += uint32(count) << bitCount
|
||||
bitCount += nbBits
|
||||
if count < max {
|
||||
bitCount--
|
||||
}
|
||||
|
||||
previous0 = count == 1
|
||||
if remaining < 1 {
|
||||
return errors.New("internal error: remaining<1")
|
||||
}
|
||||
for remaining < threshold {
|
||||
nbBits--
|
||||
threshold >>= 1
|
||||
}
|
||||
|
||||
if bitCount > 16 {
|
||||
out[outP] = byte(bitStream)
|
||||
out[outP+1] = byte(bitStream >> 8)
|
||||
outP += 2
|
||||
bitStream >>= 16
|
||||
bitCount -= 16
|
||||
}
|
||||
}
|
||||
|
||||
out[outP] = byte(bitStream)
|
||||
out[outP+1] = byte(bitStream >> 8)
|
||||
outP += (bitCount + 7) / 8
|
||||
|
||||
if charnum > s.symbolLen {
|
||||
return errors.New("internal error: charnum > s.symbolLen")
|
||||
}
|
||||
s.Out = out[:outP]
|
||||
return nil
|
||||
}
|
||||
|
||||
// symbolTransform contains the state transform for a symbol.
|
||||
type symbolTransform struct {
|
||||
deltaFindState int32
|
||||
deltaNbBits uint32
|
||||
}
|
||||
|
||||
// String prints values as a human readable string.
|
||||
func (s symbolTransform) String() string {
|
||||
return fmt.Sprintf("dnbits: %08x, fs:%d", s.deltaNbBits, s.deltaFindState)
|
||||
}
|
||||
|
||||
// cTable contains tables used for compression.
|
||||
type cTable struct {
|
||||
tableSymbol []byte
|
||||
stateTable []uint16
|
||||
symbolTT []symbolTransform
|
||||
}
|
||||
|
||||
// allocCtable will allocate tables needed for compression.
|
||||
// If existing tables a re big enough, they are simply re-used.
|
||||
func (s *Scratch) allocCtable() {
|
||||
tableSize := 1 << s.actualTableLog
|
||||
// get tableSymbol that is big enough.
|
||||
if cap(s.ct.tableSymbol) < tableSize {
|
||||
s.ct.tableSymbol = make([]byte, tableSize)
|
||||
}
|
||||
s.ct.tableSymbol = s.ct.tableSymbol[:tableSize]
|
||||
|
||||
ctSize := tableSize
|
||||
if cap(s.ct.stateTable) < ctSize {
|
||||
s.ct.stateTable = make([]uint16, ctSize)
|
||||
}
|
||||
s.ct.stateTable = s.ct.stateTable[:ctSize]
|
||||
|
||||
if cap(s.ct.symbolTT) < 256 {
|
||||
s.ct.symbolTT = make([]symbolTransform, 256)
|
||||
}
|
||||
s.ct.symbolTT = s.ct.symbolTT[:256]
|
||||
}
|
||||
|
||||
// buildCTable will populate the compression table so it is ready to be used.
|
||||
func (s *Scratch) buildCTable() error {
|
||||
tableSize := uint32(1 << s.actualTableLog)
|
||||
highThreshold := tableSize - 1
|
||||
var cumul [maxSymbolValue + 2]int16
|
||||
|
||||
s.allocCtable()
|
||||
tableSymbol := s.ct.tableSymbol[:tableSize]
|
||||
// symbol start positions
|
||||
{
|
||||
cumul[0] = 0
|
||||
for ui, v := range s.norm[:s.symbolLen-1] {
|
||||
u := byte(ui) // one less than reference
|
||||
if v == -1 {
|
||||
// Low proba symbol
|
||||
cumul[u+1] = cumul[u] + 1
|
||||
tableSymbol[highThreshold] = u
|
||||
highThreshold--
|
||||
} else {
|
||||
cumul[u+1] = cumul[u] + v
|
||||
}
|
||||
}
|
||||
// Encode last symbol separately to avoid overflowing u
|
||||
u := int(s.symbolLen - 1)
|
||||
v := s.norm[s.symbolLen-1]
|
||||
if v == -1 {
|
||||
// Low proba symbol
|
||||
cumul[u+1] = cumul[u] + 1
|
||||
tableSymbol[highThreshold] = byte(u)
|
||||
highThreshold--
|
||||
} else {
|
||||
cumul[u+1] = cumul[u] + v
|
||||
}
|
||||
if uint32(cumul[s.symbolLen]) != tableSize {
|
||||
return fmt.Errorf("internal error: expected cumul[s.symbolLen] (%d) == tableSize (%d)", cumul[s.symbolLen], tableSize)
|
||||
}
|
||||
cumul[s.symbolLen] = int16(tableSize) + 1
|
||||
}
|
||||
// Spread symbols
|
||||
s.zeroBits = false
|
||||
{
|
||||
step := tableStep(tableSize)
|
||||
tableMask := tableSize - 1
|
||||
var position uint32
|
||||
// if any symbol > largeLimit, we may have 0 bits output.
|
||||
largeLimit := int16(1 << (s.actualTableLog - 1))
|
||||
for ui, v := range s.norm[:s.symbolLen] {
|
||||
symbol := byte(ui)
|
||||
if v > largeLimit {
|
||||
s.zeroBits = true
|
||||
}
|
||||
for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ {
|
||||
tableSymbol[position] = symbol
|
||||
position = (position + step) & tableMask
|
||||
for position > highThreshold {
|
||||
position = (position + step) & tableMask
|
||||
} /* Low proba area */
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we have gone through all positions
|
||||
if position != 0 {
|
||||
return errors.New("position!=0")
|
||||
}
|
||||
}
|
||||
|
||||
// Build table
|
||||
table := s.ct.stateTable
|
||||
{
|
||||
tsi := int(tableSize)
|
||||
for u, v := range tableSymbol {
|
||||
// TableU16 : sorted by symbol order; gives next state value
|
||||
table[cumul[v]] = uint16(tsi + u)
|
||||
cumul[v]++
|
||||
}
|
||||
}
|
||||
|
||||
// Build Symbol Transformation Table
|
||||
{
|
||||
total := int16(0)
|
||||
symbolTT := s.ct.symbolTT[:s.symbolLen]
|
||||
tableLog := s.actualTableLog
|
||||
tl := (uint32(tableLog) << 16) - (1 << tableLog)
|
||||
for i, v := range s.norm[:s.symbolLen] {
|
||||
switch v {
|
||||
case 0:
|
||||
case -1, 1:
|
||||
symbolTT[i].deltaNbBits = tl
|
||||
symbolTT[i].deltaFindState = int32(total - 1)
|
||||
total++
|
||||
default:
|
||||
maxBitsOut := uint32(tableLog) - highBits(uint32(v-1))
|
||||
minStatePlus := uint32(v) << maxBitsOut
|
||||
symbolTT[i].deltaNbBits = (maxBitsOut << 16) - minStatePlus
|
||||
symbolTT[i].deltaFindState = int32(total - v)
|
||||
total += v
|
||||
}
|
||||
}
|
||||
if total != int16(tableSize) {
|
||||
return fmt.Errorf("total mismatch %d (got) != %d (want)", total, tableSize)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// countSimple will create a simple histogram in s.count.
|
||||
// Returns the biggest count.
|
||||
// Does not update s.clearCount.
|
||||
func (s *Scratch) countSimple(in []byte) (max int) {
|
||||
for _, v := range in {
|
||||
s.count[v]++
|
||||
}
|
||||
m, symlen := uint32(0), s.symbolLen
|
||||
for i, v := range s.count[:] {
|
||||
if v == 0 {
|
||||
continue
|
||||
}
|
||||
if v > m {
|
||||
m = v
|
||||
}
|
||||
symlen = uint16(i) + 1
|
||||
}
|
||||
s.symbolLen = symlen
|
||||
return int(m)
|
||||
}
|
||||
|
||||
// minTableLog provides the minimum logSize to safely represent a distribution.
|
||||
func (s *Scratch) minTableLog() uint8 {
|
||||
minBitsSrc := highBits(uint32(s.br.remain()-1)) + 1
|
||||
minBitsSymbols := highBits(uint32(s.symbolLen-1)) + 2
|
||||
if minBitsSrc < minBitsSymbols {
|
||||
return uint8(minBitsSrc)
|
||||
}
|
||||
return uint8(minBitsSymbols)
|
||||
}
|
||||
|
||||
// optimalTableLog calculates and sets the optimal tableLog in s.actualTableLog
|
||||
func (s *Scratch) optimalTableLog() {
|
||||
tableLog := s.TableLog
|
||||
minBits := s.minTableLog()
|
||||
maxBitsSrc := uint8(highBits(uint32(s.br.remain()-1))) - 2
|
||||
if maxBitsSrc < tableLog {
|
||||
// Accuracy can be reduced
|
||||
tableLog = maxBitsSrc
|
||||
}
|
||||
if minBits > tableLog {
|
||||
tableLog = minBits
|
||||
}
|
||||
// Need a minimum to safely represent all symbol values
|
||||
if tableLog < minTablelog {
|
||||
tableLog = minTablelog
|
||||
}
|
||||
if tableLog > maxTableLog {
|
||||
tableLog = maxTableLog
|
||||
}
|
||||
s.actualTableLog = tableLog
|
||||
}
|
||||
|
||||
var rtbTable = [...]uint32{0, 473195, 504333, 520860, 550000, 700000, 750000, 830000}
|
||||
|
||||
// normalizeCount will normalize the count of the symbols so
|
||||
// the total is equal to the table size.
|
||||
func (s *Scratch) normalizeCount() error {
|
||||
var (
|
||||
tableLog = s.actualTableLog
|
||||
scale = 62 - uint64(tableLog)
|
||||
step = (1 << 62) / uint64(s.br.remain())
|
||||
vStep = uint64(1) << (scale - 20)
|
||||
stillToDistribute = int16(1 << tableLog)
|
||||
largest int
|
||||
largestP int16
|
||||
lowThreshold = (uint32)(s.br.remain() >> tableLog)
|
||||
)
|
||||
|
||||
for i, cnt := range s.count[:s.symbolLen] {
|
||||
// already handled
|
||||
// if (count[s] == s.length) return 0; /* rle special case */
|
||||
|
||||
if cnt == 0 {
|
||||
s.norm[i] = 0
|
||||
continue
|
||||
}
|
||||
if cnt <= lowThreshold {
|
||||
s.norm[i] = -1
|
||||
stillToDistribute--
|
||||
} else {
|
||||
proba := (int16)((uint64(cnt) * step) >> scale)
|
||||
if proba < 8 {
|
||||
restToBeat := vStep * uint64(rtbTable[proba])
|
||||
v := uint64(cnt)*step - (uint64(proba) << scale)
|
||||
if v > restToBeat {
|
||||
proba++
|
||||
}
|
||||
}
|
||||
if proba > largestP {
|
||||
largestP = proba
|
||||
largest = i
|
||||
}
|
||||
s.norm[i] = proba
|
||||
stillToDistribute -= proba
|
||||
}
|
||||
}
|
||||
|
||||
if -stillToDistribute >= (s.norm[largest] >> 1) {
|
||||
// corner case, need another normalization method
|
||||
return s.normalizeCount2()
|
||||
}
|
||||
s.norm[largest] += stillToDistribute
|
||||
return nil
|
||||
}
|
||||
|
||||
// Secondary normalization method.
|
||||
// To be used when primary method fails.
|
||||
func (s *Scratch) normalizeCount2() error {
|
||||
const notYetAssigned = -2
|
||||
var (
|
||||
distributed uint32
|
||||
total = uint32(s.br.remain())
|
||||
tableLog = s.actualTableLog
|
||||
lowThreshold = total >> tableLog
|
||||
lowOne = (total * 3) >> (tableLog + 1)
|
||||
)
|
||||
for i, cnt := range s.count[:s.symbolLen] {
|
||||
if cnt == 0 {
|
||||
s.norm[i] = 0
|
||||
continue
|
||||
}
|
||||
if cnt <= lowThreshold {
|
||||
s.norm[i] = -1
|
||||
distributed++
|
||||
total -= cnt
|
||||
continue
|
||||
}
|
||||
if cnt <= lowOne {
|
||||
s.norm[i] = 1
|
||||
distributed++
|
||||
total -= cnt
|
||||
continue
|
||||
}
|
||||
s.norm[i] = notYetAssigned
|
||||
}
|
||||
toDistribute := (1 << tableLog) - distributed
|
||||
|
||||
if (total / toDistribute) > lowOne {
|
||||
// risk of rounding to zero
|
||||
lowOne = (total * 3) / (toDistribute * 2)
|
||||
for i, cnt := range s.count[:s.symbolLen] {
|
||||
if (s.norm[i] == notYetAssigned) && (cnt <= lowOne) {
|
||||
s.norm[i] = 1
|
||||
distributed++
|
||||
total -= cnt
|
||||
continue
|
||||
}
|
||||
}
|
||||
toDistribute = (1 << tableLog) - distributed
|
||||
}
|
||||
if distributed == uint32(s.symbolLen)+1 {
|
||||
// all values are pretty poor;
|
||||
// probably incompressible data (should have already been detected);
|
||||
// find max, then give all remaining points to max
|
||||
var maxV int
|
||||
var maxC uint32
|
||||
for i, cnt := range s.count[:s.symbolLen] {
|
||||
if cnt > maxC {
|
||||
maxV = i
|
||||
maxC = cnt
|
||||
}
|
||||
}
|
||||
s.norm[maxV] += int16(toDistribute)
|
||||
return nil
|
||||
}
|
||||
|
||||
if total == 0 {
|
||||
// all of the symbols were low enough for the lowOne or lowThreshold
|
||||
for i := uint32(0); toDistribute > 0; i = (i + 1) % (uint32(s.symbolLen)) {
|
||||
if s.norm[i] > 0 {
|
||||
toDistribute--
|
||||
s.norm[i]++
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
vStepLog = 62 - uint64(tableLog)
|
||||
mid = uint64((1 << (vStepLog - 1)) - 1)
|
||||
rStep = (((1 << vStepLog) * uint64(toDistribute)) + mid) / uint64(total) // scale on remaining
|
||||
tmpTotal = mid
|
||||
)
|
||||
for i, cnt := range s.count[:s.symbolLen] {
|
||||
if s.norm[i] == notYetAssigned {
|
||||
var (
|
||||
end = tmpTotal + uint64(cnt)*rStep
|
||||
sStart = uint32(tmpTotal >> vStepLog)
|
||||
sEnd = uint32(end >> vStepLog)
|
||||
weight = sEnd - sStart
|
||||
)
|
||||
if weight < 1 {
|
||||
return errors.New("weight < 1")
|
||||
}
|
||||
s.norm[i] = int16(weight)
|
||||
tmpTotal = end
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateNorm validates the normalized histogram table.
|
||||
func (s *Scratch) validateNorm() (err error) {
|
||||
var total int
|
||||
for _, v := range s.norm[:s.symbolLen] {
|
||||
if v >= 0 {
|
||||
total += int(v)
|
||||
} else {
|
||||
total -= int(v)
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
fmt.Printf("selected TableLog: %d, Symbol length: %d\n", s.actualTableLog, s.symbolLen)
|
||||
for i, v := range s.norm[:s.symbolLen] {
|
||||
fmt.Printf("%3d: %5d -> %4d \n", i, s.count[i], v)
|
||||
}
|
||||
}()
|
||||
if total != (1 << s.actualTableLog) {
|
||||
return fmt.Errorf("warning: Total == %d != %d", total, 1<<s.actualTableLog)
|
||||
}
|
||||
for i, v := range s.count[s.symbolLen:] {
|
||||
if v != 0 {
|
||||
return fmt.Errorf("warning: Found symbol out of range, %d after cut", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
376
vendor/github.com/klauspost/compress/fse/decompress.go
generated
vendored
Normal file
376
vendor/github.com/klauspost/compress/fse/decompress.go
generated
vendored
Normal file
@ -0,0 +1,376 @@
|
||||
package fse
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
tablelogAbsoluteMax = 15
|
||||
)
|
||||
|
||||
// Decompress a block of data.
|
||||
// You can provide a scratch buffer to avoid allocations.
|
||||
// If nil is provided a temporary one will be allocated.
|
||||
// It is possible, but by no way guaranteed that corrupt data will
|
||||
// return an error.
|
||||
// It is up to the caller to verify integrity of the returned data.
|
||||
// Use a predefined Scrach to set maximum acceptable output size.
|
||||
func Decompress(b []byte, s *Scratch) ([]byte, error) {
|
||||
s, err := s.prepare(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Out = s.Out[:0]
|
||||
err = s.readNCount()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = s.buildDtable()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = s.decompress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s.Out, nil
|
||||
}
|
||||
|
||||
// readNCount will read the symbol distribution so decoding tables can be constructed.
|
||||
func (s *Scratch) readNCount() error {
|
||||
var (
|
||||
charnum uint16
|
||||
previous0 bool
|
||||
b = &s.br
|
||||
)
|
||||
iend := b.remain()
|
||||
if iend < 4 {
|
||||
return errors.New("input too small")
|
||||
}
|
||||
bitStream := b.Uint32()
|
||||
nbBits := uint((bitStream & 0xF) + minTablelog) // extract tableLog
|
||||
if nbBits > tablelogAbsoluteMax {
|
||||
return errors.New("tableLog too large")
|
||||
}
|
||||
bitStream >>= 4
|
||||
bitCount := uint(4)
|
||||
|
||||
s.actualTableLog = uint8(nbBits)
|
||||
remaining := int32((1 << nbBits) + 1)
|
||||
threshold := int32(1 << nbBits)
|
||||
gotTotal := int32(0)
|
||||
nbBits++
|
||||
|
||||
for remaining > 1 {
|
||||
if previous0 {
|
||||
n0 := charnum
|
||||
for (bitStream & 0xFFFF) == 0xFFFF {
|
||||
n0 += 24
|
||||
if b.off < iend-5 {
|
||||
b.advance(2)
|
||||
bitStream = b.Uint32() >> bitCount
|
||||
} else {
|
||||
bitStream >>= 16
|
||||
bitCount += 16
|
||||
}
|
||||
}
|
||||
for (bitStream & 3) == 3 {
|
||||
n0 += 3
|
||||
bitStream >>= 2
|
||||
bitCount += 2
|
||||
}
|
||||
n0 += uint16(bitStream & 3)
|
||||
bitCount += 2
|
||||
if n0 > maxSymbolValue {
|
||||
return errors.New("maxSymbolValue too small")
|
||||
}
|
||||
for charnum < n0 {
|
||||
s.norm[charnum&0xff] = 0
|
||||
charnum++
|
||||
}
|
||||
|
||||
if b.off <= iend-7 || b.off+int(bitCount>>3) <= iend-4 {
|
||||
b.advance(bitCount >> 3)
|
||||
bitCount &= 7
|
||||
bitStream = b.Uint32() >> bitCount
|
||||
} else {
|
||||
bitStream >>= 2
|
||||
}
|
||||
}
|
||||
|
||||
max := (2*(threshold) - 1) - (remaining)
|
||||
var count int32
|
||||
|
||||
if (int32(bitStream) & (threshold - 1)) < max {
|
||||
count = int32(bitStream) & (threshold - 1)
|
||||
bitCount += nbBits - 1
|
||||
} else {
|
||||
count = int32(bitStream) & (2*threshold - 1)
|
||||
if count >= threshold {
|
||||
count -= max
|
||||
}
|
||||
bitCount += nbBits
|
||||
}
|
||||
|
||||
count-- // extra accuracy
|
||||
if count < 0 {
|
||||
// -1 means +1
|
||||
remaining += count
|
||||
gotTotal -= count
|
||||
} else {
|
||||
remaining -= count
|
||||
gotTotal += count
|
||||
}
|
||||
s.norm[charnum&0xff] = int16(count)
|
||||
charnum++
|
||||
previous0 = count == 0
|
||||
for remaining < threshold {
|
||||
nbBits--
|
||||
threshold >>= 1
|
||||
}
|
||||
if b.off <= iend-7 || b.off+int(bitCount>>3) <= iend-4 {
|
||||
b.advance(bitCount >> 3)
|
||||
bitCount &= 7
|
||||
} else {
|
||||
bitCount -= (uint)(8 * (len(b.b) - 4 - b.off))
|
||||
b.off = len(b.b) - 4
|
||||
}
|
||||
bitStream = b.Uint32() >> (bitCount & 31)
|
||||
}
|
||||
s.symbolLen = charnum
|
||||
|
||||
if s.symbolLen <= 1 {
|
||||
return fmt.Errorf("symbolLen (%d) too small", s.symbolLen)
|
||||
}
|
||||
if s.symbolLen > maxSymbolValue+1 {
|
||||
return fmt.Errorf("symbolLen (%d) too big", s.symbolLen)
|
||||
}
|
||||
if remaining != 1 {
|
||||
return fmt.Errorf("corruption detected (remaining %d != 1)", remaining)
|
||||
}
|
||||
if bitCount > 32 {
|
||||
return fmt.Errorf("corruption detected (bitCount %d > 32)", bitCount)
|
||||
}
|
||||
if gotTotal != 1<<s.actualTableLog {
|
||||
return fmt.Errorf("corruption detected (total %d != %d)", gotTotal, 1<<s.actualTableLog)
|
||||
}
|
||||
b.advance((bitCount + 7) >> 3)
|
||||
return nil
|
||||
}
|
||||
|
||||
// decSymbol contains information about a state entry,
|
||||
// Including the state offset base, the output symbol and
|
||||
// the number of bits to read for the low part of the destination state.
|
||||
type decSymbol struct {
|
||||
newState uint16
|
||||
symbol uint8
|
||||
nbBits uint8
|
||||
}
|
||||
|
||||
// allocDtable will allocate decoding tables if they are not big enough.
|
||||
func (s *Scratch) allocDtable() {
|
||||
tableSize := 1 << s.actualTableLog
|
||||
if cap(s.decTable) < tableSize {
|
||||
s.decTable = make([]decSymbol, tableSize)
|
||||
}
|
||||
s.decTable = s.decTable[:tableSize]
|
||||
|
||||
if cap(s.ct.tableSymbol) < 256 {
|
||||
s.ct.tableSymbol = make([]byte, 256)
|
||||
}
|
||||
s.ct.tableSymbol = s.ct.tableSymbol[:256]
|
||||
|
||||
if cap(s.ct.stateTable) < 256 {
|
||||
s.ct.stateTable = make([]uint16, 256)
|
||||
}
|
||||
s.ct.stateTable = s.ct.stateTable[:256]
|
||||
}
|
||||
|
||||
// buildDtable will build the decoding table.
|
||||
func (s *Scratch) buildDtable() error {
|
||||
tableSize := uint32(1 << s.actualTableLog)
|
||||
highThreshold := tableSize - 1
|
||||
s.allocDtable()
|
||||
symbolNext := s.ct.stateTable[:256]
|
||||
|
||||
// Init, lay down lowprob symbols
|
||||
s.zeroBits = false
|
||||
{
|
||||
largeLimit := int16(1 << (s.actualTableLog - 1))
|
||||
for i, v := range s.norm[:s.symbolLen] {
|
||||
if v == -1 {
|
||||
s.decTable[highThreshold].symbol = uint8(i)
|
||||
highThreshold--
|
||||
symbolNext[i] = 1
|
||||
} else {
|
||||
if v >= largeLimit {
|
||||
s.zeroBits = true
|
||||
}
|
||||
symbolNext[i] = uint16(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Spread symbols
|
||||
{
|
||||
tableMask := tableSize - 1
|
||||
step := tableStep(tableSize)
|
||||
position := uint32(0)
|
||||
for ss, v := range s.norm[:s.symbolLen] {
|
||||
for i := 0; i < int(v); i++ {
|
||||
s.decTable[position].symbol = uint8(ss)
|
||||
position = (position + step) & tableMask
|
||||
for position > highThreshold {
|
||||
// lowprob area
|
||||
position = (position + step) & tableMask
|
||||
}
|
||||
}
|
||||
}
|
||||
if position != 0 {
|
||||
// position must reach all cells once, otherwise normalizedCounter is incorrect
|
||||
return errors.New("corrupted input (position != 0)")
|
||||
}
|
||||
}
|
||||
|
||||
// Build Decoding table
|
||||
{
|
||||
tableSize := uint16(1 << s.actualTableLog)
|
||||
for u, v := range s.decTable {
|
||||
symbol := v.symbol
|
||||
nextState := symbolNext[symbol]
|
||||
symbolNext[symbol] = nextState + 1
|
||||
nBits := s.actualTableLog - byte(highBits(uint32(nextState)))
|
||||
s.decTable[u].nbBits = nBits
|
||||
newState := (nextState << nBits) - tableSize
|
||||
if newState >= tableSize {
|
||||
return fmt.Errorf("newState (%d) outside table size (%d)", newState, tableSize)
|
||||
}
|
||||
if newState == uint16(u) && nBits == 0 {
|
||||
// Seems weird that this is possible with nbits > 0.
|
||||
return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, u)
|
||||
}
|
||||
s.decTable[u].newState = newState
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// decompress will decompress the bitstream.
|
||||
// If the buffer is over-read an error is returned.
|
||||
func (s *Scratch) decompress() error {
|
||||
br := &s.bits
|
||||
if err := br.init(s.br.unread()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var s1, s2 decoder
|
||||
// Initialize and decode first state and symbol.
|
||||
s1.init(br, s.decTable, s.actualTableLog)
|
||||
s2.init(br, s.decTable, s.actualTableLog)
|
||||
|
||||
// Use temp table to avoid bound checks/append penalty.
|
||||
var tmp = s.ct.tableSymbol[:256]
|
||||
var off uint8
|
||||
|
||||
// Main part
|
||||
if !s.zeroBits {
|
||||
for br.off >= 8 {
|
||||
br.fillFast()
|
||||
tmp[off+0] = s1.nextFast()
|
||||
tmp[off+1] = s2.nextFast()
|
||||
br.fillFast()
|
||||
tmp[off+2] = s1.nextFast()
|
||||
tmp[off+3] = s2.nextFast()
|
||||
off += 4
|
||||
// When off is 0, we have overflowed and should write.
|
||||
if off == 0 {
|
||||
s.Out = append(s.Out, tmp...)
|
||||
if len(s.Out) >= s.DecompressLimit {
|
||||
return fmt.Errorf("output size (%d) > DecompressLimit (%d)", len(s.Out), s.DecompressLimit)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for br.off >= 8 {
|
||||
br.fillFast()
|
||||
tmp[off+0] = s1.next()
|
||||
tmp[off+1] = s2.next()
|
||||
br.fillFast()
|
||||
tmp[off+2] = s1.next()
|
||||
tmp[off+3] = s2.next()
|
||||
off += 4
|
||||
if off == 0 {
|
||||
s.Out = append(s.Out, tmp...)
|
||||
// When off is 0, we have overflowed and should write.
|
||||
if len(s.Out) >= s.DecompressLimit {
|
||||
return fmt.Errorf("output size (%d) > DecompressLimit (%d)", len(s.Out), s.DecompressLimit)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
s.Out = append(s.Out, tmp[:off]...)
|
||||
|
||||
// Final bits, a bit more expensive check
|
||||
for {
|
||||
if s1.finished() {
|
||||
s.Out = append(s.Out, s1.final(), s2.final())
|
||||
break
|
||||
}
|
||||
br.fill()
|
||||
s.Out = append(s.Out, s1.next())
|
||||
if s2.finished() {
|
||||
s.Out = append(s.Out, s2.final(), s1.final())
|
||||
break
|
||||
}
|
||||
s.Out = append(s.Out, s2.next())
|
||||
if len(s.Out) >= s.DecompressLimit {
|
||||
return fmt.Errorf("output size (%d) > DecompressLimit (%d)", len(s.Out), s.DecompressLimit)
|
||||
}
|
||||
}
|
||||
return br.close()
|
||||
}
|
||||
|
||||
// decoder keeps track of the current state and updates it from the bitstream.
|
||||
type decoder struct {
|
||||
state uint16
|
||||
br *bitReader
|
||||
dt []decSymbol
|
||||
}
|
||||
|
||||
// init will initialize the decoder and read the first state from the stream.
|
||||
func (d *decoder) init(in *bitReader, dt []decSymbol, tableLog uint8) {
|
||||
d.dt = dt
|
||||
d.br = in
|
||||
d.state = in.getBits(tableLog)
|
||||
}
|
||||
|
||||
// next returns the next symbol and sets the next state.
|
||||
// At least tablelog bits must be available in the bit reader.
|
||||
func (d *decoder) next() uint8 {
|
||||
n := &d.dt[d.state]
|
||||
lowBits := d.br.getBits(n.nbBits)
|
||||
d.state = n.newState + lowBits
|
||||
return n.symbol
|
||||
}
|
||||
|
||||
// finished returns true if all bits have been read from the bitstream
|
||||
// and the next state would require reading bits from the input.
|
||||
func (d *decoder) finished() bool {
|
||||
return d.br.finished() && d.dt[d.state].nbBits > 0
|
||||
}
|
||||
|
||||
// final returns the current state symbol without decoding the next.
|
||||
func (d *decoder) final() uint8 {
|
||||
return d.dt[d.state].symbol
|
||||
}
|
||||
|
||||
// nextFast returns the next symbol and sets the next state.
|
||||
// This can only be used if no symbols are 0 bits.
|
||||
// At least tablelog bits must be available in the bit reader.
|
||||
func (d *decoder) nextFast() uint8 {
|
||||
n := d.dt[d.state]
|
||||
lowBits := d.br.getBitsFast(n.nbBits)
|
||||
d.state = n.newState + lowBits
|
||||
return n.symbol
|
||||
}
|
144
vendor/github.com/klauspost/compress/fse/fse.go
generated
vendored
Normal file
144
vendor/github.com/klauspost/compress/fse/fse.go
generated
vendored
Normal file
@ -0,0 +1,144 @@
|
||||
// Copyright 2018 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
// Based on work Copyright (c) 2013, Yann Collet, released under BSD License.
|
||||
|
||||
// Package fse provides Finite State Entropy encoding and decoding.
|
||||
//
|
||||
// Finite State Entropy encoding provides a fast near-optimal symbol encoding/decoding
|
||||
// for byte blocks as implemented in zstd.
|
||||
//
|
||||
// See https://github.com/klauspost/compress/tree/master/fse for more information.
|
||||
package fse
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
const (
|
||||
/*!MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
||||
maxMemoryUsage = 14
|
||||
defaultMemoryUsage = 13
|
||||
|
||||
maxTableLog = maxMemoryUsage - 2
|
||||
maxTablesize = 1 << maxTableLog
|
||||
defaultTablelog = defaultMemoryUsage - 2
|
||||
minTablelog = 5
|
||||
maxSymbolValue = 255
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrIncompressible is returned when input is judged to be too hard to compress.
|
||||
ErrIncompressible = errors.New("input is not compressible")
|
||||
|
||||
// ErrUseRLE is returned from the compressor when the input is a single byte value repeated.
|
||||
ErrUseRLE = errors.New("input is single value repeated")
|
||||
)
|
||||
|
||||
// Scratch provides temporary storage for compression and decompression.
|
||||
type Scratch struct {
|
||||
// Private
|
||||
count [maxSymbolValue + 1]uint32
|
||||
norm [maxSymbolValue + 1]int16
|
||||
br byteReader
|
||||
bits bitReader
|
||||
bw bitWriter
|
||||
ct cTable // Compression tables.
|
||||
decTable []decSymbol // Decompression table.
|
||||
maxCount int // count of the most probable symbol
|
||||
|
||||
// Per block parameters.
|
||||
// These can be used to override compression parameters of the block.
|
||||
// Do not touch, unless you know what you are doing.
|
||||
|
||||
// Out is output buffer.
|
||||
// If the scratch is re-used before the caller is done processing the output,
|
||||
// set this field to nil.
|
||||
// Otherwise the output buffer will be re-used for next Compression/Decompression step
|
||||
// and allocation will be avoided.
|
||||
Out []byte
|
||||
|
||||
// DecompressLimit limits the maximum decoded size acceptable.
|
||||
// If > 0 decompression will stop when approximately this many bytes
|
||||
// has been decoded.
|
||||
// If 0, maximum size will be 2GB.
|
||||
DecompressLimit int
|
||||
|
||||
symbolLen uint16 // Length of active part of the symbol table.
|
||||
actualTableLog uint8 // Selected tablelog.
|
||||
zeroBits bool // no bits has prob > 50%.
|
||||
clearCount bool // clear count
|
||||
|
||||
// MaxSymbolValue will override the maximum symbol value of the next block.
|
||||
MaxSymbolValue uint8
|
||||
|
||||
// TableLog will attempt to override the tablelog for the next block.
|
||||
TableLog uint8
|
||||
}
|
||||
|
||||
// Histogram allows to populate the histogram and skip that step in the compression,
|
||||
// It otherwise allows to inspect the histogram when compression is done.
|
||||
// To indicate that you have populated the histogram call HistogramFinished
|
||||
// with the value of the highest populated symbol, as well as the number of entries
|
||||
// in the most populated entry. These are accepted at face value.
|
||||
// The returned slice will always be length 256.
|
||||
func (s *Scratch) Histogram() []uint32 {
|
||||
return s.count[:]
|
||||
}
|
||||
|
||||
// HistogramFinished can be called to indicate that the histogram has been populated.
|
||||
// maxSymbol is the index of the highest set symbol of the next data segment.
|
||||
// maxCount is the number of entries in the most populated entry.
|
||||
// These are accepted at face value.
|
||||
func (s *Scratch) HistogramFinished(maxSymbol uint8, maxCount int) {
|
||||
s.maxCount = maxCount
|
||||
s.symbolLen = uint16(maxSymbol) + 1
|
||||
s.clearCount = maxCount != 0
|
||||
}
|
||||
|
||||
// prepare will prepare and allocate scratch tables used for both compression and decompression.
|
||||
func (s *Scratch) prepare(in []byte) (*Scratch, error) {
|
||||
if s == nil {
|
||||
s = &Scratch{}
|
||||
}
|
||||
if s.MaxSymbolValue == 0 {
|
||||
s.MaxSymbolValue = 255
|
||||
}
|
||||
if s.TableLog == 0 {
|
||||
s.TableLog = defaultTablelog
|
||||
}
|
||||
if s.TableLog > maxTableLog {
|
||||
return nil, fmt.Errorf("tableLog (%d) > maxTableLog (%d)", s.TableLog, maxTableLog)
|
||||
}
|
||||
if cap(s.Out) == 0 {
|
||||
s.Out = make([]byte, 0, len(in))
|
||||
}
|
||||
if s.clearCount && s.maxCount == 0 {
|
||||
for i := range s.count {
|
||||
s.count[i] = 0
|
||||
}
|
||||
s.clearCount = false
|
||||
}
|
||||
s.br.init(in)
|
||||
if s.DecompressLimit == 0 {
|
||||
// Max size 2GB.
|
||||
s.DecompressLimit = (2 << 30) - 1
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// tableStep returns the next table index.
|
||||
func tableStep(tableSize uint32) uint32 {
|
||||
return (tableSize >> 1) + (tableSize >> 3) + 3
|
||||
}
|
||||
|
||||
func highBits(val uint32) (n uint32) {
|
||||
return uint32(bits.Len32(val) - 1)
|
||||
}
|
Reference in New Issue
Block a user