Compare commits

...

4 Commits

Author SHA1 Message Date
sixsmith 9b7952bdf4 add --pindigests flag 2026-07-27 06:05:14 +00:00
sixsmith 8839ad17ba add regclient to fetch image manifest/digest
* also run `go mod vendor`
2026-07-27 06:05:14 +00:00
sixsmith abe3d4d4dd chore: mark vendor directory as generated 2026-07-27 06:05:14 +00:00
sixsmith 57ce169160 handle image digests in compose.yml 2026-07-27 05:45:37 +00:00
237 changed files with 34324 additions and 25825 deletions
+1
View File
@@ -0,0 +1 @@
vendor/** linguist-generated=true
+1
View File
@@ -19,5 +19,6 @@ var (
Minor bool Minor bool
NoDomainChecks bool NoDomainChecks bool
Patch bool Patch bool
PinDigests bool
ShowUnchanged bool ShowUnchanged bool
) )
+1 -1
View File
@@ -98,7 +98,7 @@ func GetMainAppImage(recipe recipe.Recipe) (string, error) {
} }
for _, service := range config.Services { for _, service := range config.Services {
if service.Name == "app" { if service.Name == "app" {
img, err := reference.ParseNormalizedNamed(service.Image) img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
if err != nil { if err != nil {
return "", err return "", err
} }
+1 -1
View File
@@ -322,7 +322,7 @@ func GetImageVersions(recipe recipePkg.Recipe) (map[string]string, error) {
continue continue
} }
img, err := reference.ParseNormalizedNamed(service.Image) img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
if err != nil { if err != nil {
return services, err return services, err
} }
+30 -3
View File
@@ -130,7 +130,7 @@ interface.`),
} }
for _, service := range config.Services { for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(service.Image) img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@@ -293,12 +293,31 @@ interface.`),
} }
} }
if upgradeTag != "skip" { if upgradeTag != "skip" {
ok, err := recipe.UpdateTag(image, upgradeTag) var ok bool
var err error
var resolvedDigest string
if internal.PinDigests {
digest, dErr := client.GetImageDigest(img, upgradeTag)
if dErr != nil {
log.Fatal(i18n.G("failed to resolve digest for %s:%s: %s", image, upgradeTag, dErr.Error()))
}
resolvedDigest = digest
ok, err = recipe.UpdatePinnedTag(image, upgradeTag, digest)
} else {
ok, err = recipe.UpdateTag(image, upgradeTag)
}
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
if ok { if ok {
log.Info(i18n.G("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image)) if internal.PinDigests {
log.Info(i18n.G("tag upgraded and pinned from %s to %s@%s for %s", tag.String(), upgradeTag, resolvedDigest, image))
} else {
log.Info(i18n.G("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image))
}
} }
} else { } else {
if !internal.NoInput { if !internal.NoInput {
@@ -417,4 +436,12 @@ func init() {
false, false,
i18n.G("commit changes"), i18n.G("commit changes"),
) )
RecipeUpgradeCommand.Flags().BoolVarP(
&internal.PinDigests,
i18n.G("pindigests"),
i18n.GC("p", "pin container image by digest"),
false,
i18n.G("pin the container image version by manifest digest (image:tag@sha256:...)"),
)
} }
+7 -5
View File
@@ -19,6 +19,7 @@ require (
github.com/moby/sys/signal v0.7.1 github.com/moby/sys/signal v0.7.1
github.com/moby/term v0.5.2 github.com/moby/term v0.5.2
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/regclient/regclient v0.11.5
github.com/schollz/progressbar/v3 v3.19.1 github.com/schollz/progressbar/v3 v3.19.1
golang.org/x/term v0.45.0 golang.org/x/term v0.45.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
@@ -73,7 +74,7 @@ require (
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kevinburke/ssh_config v1.6.0 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect
github.com/klauspost/compress v1.18.5 // indirect github.com/klauspost/compress v1.18.6 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-colorable v0.1.14 // indirect
@@ -106,6 +107,7 @@ require (
github.com/sirupsen/logrus v1.9.4 // indirect github.com/sirupsen/logrus v1.9.4 // indirect
github.com/skeema/knownhosts v1.3.2 // indirect github.com/skeema/knownhosts v1.3.2 // indirect
github.com/spf13/pflag v1.0.10 // indirect github.com/spf13/pflag v1.0.10 // indirect
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@@ -124,10 +126,10 @@ require (
go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.50.0 // indirect golang.org/x/crypto v0.52.0 // indirect
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
golang.org/x/net v0.53.0 // indirect golang.org/x/net v0.54.0 // indirect
golang.org/x/text v0.36.0 // indirect golang.org/x/text v0.37.0 // indirect
golang.org/x/time v0.15.0 // indirect golang.org/x/time v0.15.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
@@ -151,7 +153,7 @@ require (
github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect
github.com/sergi/go-diff v1.4.0 // indirect github.com/sergi/go-diff v1.4.0 // indirect
github.com/spf13/cobra v1.10.1 github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.11.1 github.com/stretchr/testify v1.11.1
github.com/theupdateframework/notary v0.7.0 // indirect github.com/theupdateframework/notary v0.7.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+18 -8
View File
@@ -427,6 +427,8 @@ github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
@@ -584,8 +586,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -699,6 +701,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/olareg/olareg v0.2.1 h1:RPHGIaqlVWPbKAsOYUj7e2WfEEW5M7F8I6hKMrwd4jU=
github.com/olareg/olareg v0.2.1/go.mod h1:dhr8QetC7U7jJ2m93oxDhEEOKCRbPgOK1oGyKfB4QNo=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -796,6 +800,8 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
github.com/regclient/regclient v0.11.5 h1:OHRsXO0F3qHGfa4HEUv+EkMH9NXNcCTBKjNzyC/UhIA=
github.com/regclient/regclient v0.11.5/go.mod h1:DZUOfIT14WFTK2Pj4vjd93avy9O4Fdpjrf9ir23TbRE=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -859,6 +865,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/sudo-bmitch/oci-digest v0.1.2 h1:are0qzWTsFZGZ3Uvdi9OSztJszSWaab6iqquMEEB7rw=
github.com/sudo-bmitch/oci-digest v0.1.2/go.mod h1:SH6l5OIe0islKBZBedjiPOeET/0QwGL+/oYfQt51uQo=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
@@ -870,6 +878,8 @@ github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnH
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@@ -966,8 +976,8 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1043,8 +1053,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1155,8 +1165,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+24
View File
@@ -9,8 +9,12 @@ import (
"github.com/containers/image/docker" "github.com/containers/image/docker"
"github.com/containers/image/types" "github.com/containers/image/types"
"github.com/distribution/reference" "github.com/distribution/reference"
"github.com/regclient/regclient"
"github.com/regclient/regclient/types/ref"
) )
var rc = regclient.New()
// GetRegistryTags retrieves all tags of an image from a container registry. // GetRegistryTags retrieves all tags of an image from a container registry.
func GetRegistryTags(img reference.Named) ([]string, error) { func GetRegistryTags(img reference.Named) ([]string, error) {
var tags []string var tags []string
@@ -28,3 +32,23 @@ func GetRegistryTags(img reference.Named) ([]string, error) {
return tags, nil return tags, nil
} }
// GetImageDigest resolves the content digest (sha256:...) for a specific
// image:tag reference by querying the registry manifest. This gives us a
// digest to pin to while keeping support for multi-arch deployments: when
// the tag points at a manifest list/OCI index, the digest returned is the
// digest of that top-level index, so the pin still resolves correctly
// regardless of the deploying host's architecture.
func GetImageDigest(img reference.Named, tag string) (string, error) {
r, err := ref.New(fmt.Sprintf("%s:%s", img.Name(), tag))
if err != nil {
return "", fmt.Errorf("parsing reference %s:%s: %w", img.Name(), tag, err)
}
m, err := rc.ManifestHead(context.Background(), r)
if err != nil {
return "", fmt.Errorf("fetching manifest for %s:%s: %w", img.Name(), tag, err)
}
return m.GetDescriptor().Digest.String(), nil
}
+2 -2
View File
@@ -158,7 +158,7 @@ func GetImagesForStack(cl *dockerClient.Client, app appPkg.App) (map[string]stri
if service.Spec.TaskTemplate.ContainerSpec != nil { if service.Spec.TaskTemplate.ContainerSpec != nil {
imageName := service.Spec.TaskTemplate.ContainerSpec.Image imageName := service.Spec.TaskTemplate.ContainerSpec.Image
imageParsed, err := reference.ParseNormalizedNamed(imageName) imageParsed, err := reference.ParseNormalizedNamed(formatter.StripDigest(imageName))
if err != nil { if err != nil {
log.Warn(err) log.Warn(err)
continue continue
@@ -281,7 +281,7 @@ func GatherImagesForDeploy(cl *dockerClient.Client, app appPkg.App, compose *com
newImages := make(map[string]string) newImages := make(map[string]string)
for _, service := range compose.Services { for _, service := range compose.Services {
imageParsed, err := reference.ParseNormalizedNamed(service.Image) imageParsed, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
if err != nil { if err != nil {
log.Warn(err) log.Warn(err)
continue continue
+5
View File
@@ -34,6 +34,11 @@ func SmallSHA(hash string) string {
return hash[:8] return hash[:8]
} }
// StripDigest removes the digest suffix from an image reference (e.g., "image:tag@sha256:..." -> "image:tag").
func StripDigest(image string) string {
return strings.Split(image, "@")[0]
}
// RemoveSha remove image sha from a string that are added in some docker outputs // RemoveSha remove image sha from a string that are added in some docker outputs
func RemoveSha(str string) string { func RemoveSha(str string) string {
return strings.Split(str, "@")[0] return strings.Split(str, "@")[0]
+7
View File
@@ -6,6 +6,13 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func TestStripDigest(t *testing.T) {
assert.Equal(t, "ubuntu:latest", StripDigest("ubuntu:latest@sha256:1234567890abcdef"))
assert.Equal(t, "ubuntu:latest", StripDigest("ubuntu:latest"))
assert.Equal(t, "my-repo/my-image:v1", StripDigest("my-repo/my-image:v1@sha256:abcdef1234567890"))
assert.Equal(t, "my-repo/my-image", StripDigest("my-repo/my-image@sha256:abcdef1234567890"))
}
func TestBoldDirtyDefault(t *testing.T) { func TestBoldDirtyDefault(t *testing.T) {
assert.Equal(t, "foo", BoldDirtyDefault("foo")) assert.Equal(t, "foo", BoldDirtyDefault("foo"))
} }
+4 -3
View File
@@ -7,6 +7,7 @@ import (
"os" "os"
"path" "path"
"coopcloud.tech/abra/pkg/formatter"
"coopcloud.tech/abra/pkg/i18n" "coopcloud.tech/abra/pkg/i18n"
"coopcloud.tech/abra/pkg/log" "coopcloud.tech/abra/pkg/log"
"coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/recipe"
@@ -321,7 +322,7 @@ func LintAllImagesTagged(recipe recipe.Recipe) (bool, error) {
return false, err return false, err
} }
for _, service := range config.Services { for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(service.Image) img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
if err != nil { if err != nil {
return false, err return false, err
} }
@@ -339,7 +340,7 @@ func LintNoUnstableTags(recipe recipe.Recipe) (bool, error) {
return false, err return false, err
} }
for _, service := range config.Services { for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(service.Image) img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
if err != nil { if err != nil {
return false, err return false, err
} }
@@ -366,7 +367,7 @@ func LintSemverLikeTags(recipe recipe.Recipe) (bool, error) {
return false, err return false, err
} }
for _, service := range config.Services { for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(service.Image) img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
if err != nil { if err != nil {
return false, err return false, err
} }
+68 -2
View File
@@ -122,6 +122,7 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
log.Debug(i18n.G("considering %s config(s) for tag update", strings.Join(composeFiles, ", "))) log.Debug(i18n.G("considering %s config(s) for tag update", strings.Join(composeFiles, ", ")))
updated := false
for _, composeFile := range composeFiles { for _, composeFile := range composeFiles {
opts := stack.Deploy{Composefiles: []string{composeFile}} opts := stack.Deploy{Composefiles: []string{composeFile}}
@@ -140,7 +141,7 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
continue // may be a compose.$optional.yml file continue // may be a compose.$optional.yml file
} }
img, _ := reference.ParseNormalizedNamed(service.Image) img, _ := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
if err != nil { if err != nil {
return false, err return false, err
} }
@@ -166,6 +167,10 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
old := fmt.Sprintf("%s:%s", composeImage, composeTag) old := fmt.Sprintf("%s:%s", composeImage, composeTag)
new := fmt.Sprintf("%s:%s", composeImage, tag) new := fmt.Sprintf("%s:%s", composeImage, tag)
if old == new {
continue
}
replacedBytes := strings.Replace(string(bytes), old, new, -1) replacedBytes := strings.Replace(string(bytes), old, new, -1)
log.Debug(i18n.G("updating %s to %s in %s", old, new, compose.Filename)) log.Debug(i18n.G("updating %s to %s in %s", old, new, compose.Filename))
@@ -173,11 +178,72 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
if err := os.WriteFile(compose.Filename, []byte(replacedBytes), 0o764); err != nil { if err := os.WriteFile(compose.Filename, []byte(replacedBytes), 0o764); err != nil {
return false, err return false, err
} }
updated = true
} }
} }
} }
return false, nil return updated, nil
}
// UpdatePinnedTag updates an image reference to a pinned version (image:tag@sha256:...) in-place on local compose files.
func (r Recipe) UpdatePinnedTag(image, tag, digest string) (bool, error) {
glob := fmt.Sprintf("%s/compose**yml", r.Dir)
image = formatter.StripTagMeta(image)
composeFiles, err := filepath.Glob(glob)
if err != nil {
return false, err
}
updated := false
for _, composeFile := range composeFiles {
opts := stack.Deploy{Composefiles: []string{composeFile}}
sampleEnv, err := r.SampleEnv()
if err != nil {
return false, err
}
compose, err := loader.LoadComposefile(opts, sampleEnv)
if err != nil {
return false, err
}
for _, service := range compose.Services {
if service.Image == "" {
continue
}
img, _ := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
composeImage := formatter.StripTagMeta(reference.Path(img))
if image == composeImage {
bytes, err := ioutil.ReadFile(composeFile)
if err != nil {
return false, err
}
old := service.Image
new := fmt.Sprintf("%s:%s@%s", composeImage, tag, digest)
if old == new {
continue
}
replacedBytes := strings.Replace(string(bytes), old, new, -1)
log.Debug(i18n.G("updating pinned reference %s to %s in %s", old, new, compose.Filename))
if err := os.WriteFile(compose.Filename, []byte(replacedBytes), 0o764); err != nil {
return false, err
}
updated = true
}
}
}
return updated, nil
} }
// UpdateLabel updates a label in-place on file system local compose files. // UpdateLabel updates a label in-place on file system local compose files.
+1 -1
View File
@@ -426,7 +426,7 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, []string, error) {
versionMeta := make(map[string]ServiceMeta) versionMeta := make(map[string]ServiceMeta)
for _, service := range config.Services { for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(service.Image) img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
if err != nil { if err != nil {
log.Debug(i18n.G("failed to parse image for %s in %s: %s", service.Name, tag, err)) log.Debug(i18n.G("failed to parse image for %s in %s: %s", service.Name, tag, err))
warnMsg = append(warnMsg, i18n.G("skipping tag %s: invalid image reference in service %s: %s", tag, service.Name, err)) warnMsg = append(warnMsg, i18n.G("skipping tag %s: invalid image reference in service %s: %s", tag, service.Name, err))
+1
View File
@@ -1,2 +1,3 @@
* -text * -text
*.bin -text -diff *.bin -text -diff
*.md text eol=lf
+700 -700
View File
File diff suppressed because it is too large Load Diff
+78 -78
View File
@@ -1,79 +1,79 @@
# Finite State Entropy # Finite State Entropy
This package provides Finite State Entropy encoding and decoding. This package provides Finite State Entropy encoding and decoding.
Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS))
encoding provides a fast near-optimal symbol encoding/decoding encoding provides a fast near-optimal symbol encoding/decoding
for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
This can be used for compressing input with a lot of similar input values to the smallest number of bytes. This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) * [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
## News ## News
* Feb 2018: First implementation released. Consider this beta software for now. * Feb 2018: First implementation released. Consider this beta software for now.
# Usage # Usage
This package provides a low level interface that allows to compress single independent blocks. This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks. Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed. This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
You must provide input and will receive the output and maybe an error. You must provide input and will receive the output and maybe an error.
These error values can be returned: These error values can be returned:
| Error | Description | | Error | Description |
|---------------------|-----------------------------------------------------------------------------| |---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned | | `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress | | `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | | `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `(error)` | An internal error occurred. | | `(error)` | An internal error occurred. |
As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both. object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted. your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input. It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
# Performance # Performance
A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.
All compression functions are currently only running on the calling goroutine so only one core will be used per block. All compression functions are currently only running on the calling goroutine so only one core will be used per block.
The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be
beneficial to transpose all your input values down by 64. beneficial to transpose all your input values down by 64.
With moderate block sizes around 64k speed are typically 200MB/s per core for compression and With moderate block sizes around 64k speed are typically 200MB/s per core for compression and
around 300MB/s decompression speed. around 300MB/s decompression speed.
The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s.
# Plans # Plans
At one point, more internals will be exposed to facilitate more "expert" usage of the components. At one point, more internals will be exposed to facilitate more "expert" usage of the components.
A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).
# Contributing # Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
changes will likely not be accepted. If in doubt open an issue before writing the PR. changes will likely not be accepted. If in doubt open an issue before writing the PR.
+89 -89
View File
@@ -1,89 +1,89 @@
# Huff0 entropy compression # Huff0 entropy compression
This package provides Huff0 encoding and decoding as used in zstd. This package provides Huff0 encoding and decoding as used in zstd.
[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), [Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders),
a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU
(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. (Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
This can be used for compressing input with a lot of similar input values to the smallest number of bytes. This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) * [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
## News ## News
This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
This ensures that most functionality is well tested. This ensures that most functionality is well tested.
# Usage # Usage
This package provides a low level interface that allows to compress single independent blocks. This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks. Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed. This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and
[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. [`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
You must provide input and will receive the output and maybe an error. You must provide input and will receive the output and maybe an error.
These error values can be returned: These error values can be returned:
| Error | Description | | Error | Description |
|---------------------|-----------------------------------------------------------------------------| |---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned | | `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress | | `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | | `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | | `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) |
| `(error)` | An internal error occurred. | | `(error)` | An internal error occurred. |
As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both. object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.
## Tables and re-use ## Tables and re-use
Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results.
The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy)
that controls this behaviour. See the documentation for details. This can be altered between each block. that controls this behaviour. See the documentation for details. This can be altered between each block.
Do however note that this information is *not* stored in the output block and it is up to the users of the package to Do however note that this information is *not* stored in the output block and it is up to the users of the package to
record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
based on the boolean reported back from the CompressXX call. based on the boolean reported back from the CompressXX call.
If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the
[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
## Decompressing ## Decompressing
The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
This will initialize the decoding tables. This will initialize the decoding tables.
You can supply the complete block to `ReadTable` and it will return the data part of the block You can supply the complete block to `ReadTable` and it will return the data part of the block
which can be given to the decompressor. which can be given to the decompressor.
Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X)
or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted. your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input. It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
# Contributing # Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
changes will likely not be accepted. If in doubt open an issue before writing the PR. changes will likely not be accepted. If in doubt open an issue before writing the PR.
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.26@sha256:b900de91b15b2e2953d930ece1d0ecff0a1590ab2006088d20dcf0f56f1e979f FROM golang:1.25@sha256:31c1e53dfc1cc2d269deec9c83f58729fa3c53dc9a576f6426109d1e319e9e9a
ENV GOOS=linux ENV GOOS=linux
ENV GOARCH=arm ENV GOARCH=arm
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.26@sha256:b900de91b15b2e2953d930ece1d0ecff0a1590ab2006088d20dcf0f56f1e979f FROM golang:1.25@sha256:31c1e53dfc1cc2d269deec9c83f58729fa3c53dc9a576f6426109d1e319e9e9a
ENV GOOS=linux ENV GOOS=linux
ENV GOARCH=arm64 ENV GOARCH=arm64
+15
View File
@@ -0,0 +1,15 @@
*
!.git/
!build/root.tgz
!cmd/
!config/
!internal/
!mod/
!pkg/
!regclient/
!scheme/
!types/
!vendor/
!go.*
!*.go
!Makefile
+5
View File
@@ -0,0 +1,5 @@
artifacts/
bin/
output/
vendor/
.regctl_conf_ci.json
+19
View File
@@ -0,0 +1,19 @@
# all lists use a `-`
MD004:
style: dash
# allow tabs in code blocks (for Go)
MD010:
code_blocks: false
# disable line length, prefer one sentence per line for PRs
MD013: false
# emphasis with underscore (`_emphasis_`)
MD049:
style: "underscore"
# bold with asterisk (`**bold**`)
MD050:
style: "asterisk"
+1
View File
@@ -0,0 +1 @@
GoVersionOverride = "1.26.3"
+53
View File
@@ -0,0 +1,53 @@
{"name":"docker-arg-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
{"name":"docker-arg-alpine-tag","key":"docker.io/library/alpine","version":"3.23.4"}
{"name":"docker-arg-ecr","key":"https://github.com/awslabs/amazon-ecr-credential-helper.git","version":"v0.12.0"}
{"name":"docker-arg-gcr","key":"https://github.com/GoogleCloudPlatform/docker-credential-gcr.git","version":"v2.1.32"}
{"name":"docker-arg-go-digest","key":"docker.io/library/golang:1.26.3-alpine","version":"sha256:91eda9776261207ea25fd06b5b7fed8d397dd2c0a283e77f2ab6e91bfa71079d"}
{"name":"docker-arg-go-tag","key":"docker.io/library/golang","version":"1.26.3"}
{"name":"docker-arg-lunajson","key":"https://github.com/grafi-tt/lunajson.git:master","version":"e3a9666eb1275741e887e29926b144f8daee3bef"}
{"name":"docker-arg-semver","key":"https://github.com/kikito/semver.lua.git:master","version":"a4b708ba243208d46e575da870af969dca46a94d"}
{"name":"gha-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
{"name":"gha-alpine-tag-base","key":"docker.io/library/alpine","version":"3"}
{"name":"gha-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.23.4"}
{"name":"gha-cosign-version","key":"https://github.com/sigstore/cosign.git","version":"v3.0.6"}
{"name":"gha-golang-matrix","key":"golang-matrix","version":"[\"1.25\", \"1.26\"]"}
{"name":"gha-golang-release","key":"golang-latest","version":"1.26"}
{"name":"gha-syft-version","key":"docker.io/anchore/syft","version":"v1.44.0"}
{"name":"gha-uses-commit","key":"https://github.com/actions/checkout.git:v6.0.2","version":"de0fac2e4500dabe0009e67214ff5f5447ce83dd"}
{"name":"gha-uses-commit","key":"https://github.com/actions/setup-go.git:v6.4.0","version":"4a3601121dd01d1626a1e23e37211e3254c1c06c"}
{"name":"gha-uses-commit","key":"https://github.com/actions/stale.git:v10.3.0","version":"eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899"}
{"name":"gha-uses-commit","key":"https://github.com/actions/upload-artifact.git:v7.0.1","version":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"}
{"name":"gha-uses-commit","key":"https://github.com/anchore/sbom-action.git:v0.24.0","version":"e22c389904149dbc22b58101806040fa8d37a610"}
{"name":"gha-uses-commit","key":"https://github.com/docker/build-push-action.git:v7.2.0","version":"f9f3042f7e2789586610d6e8b85c8f03e5195baf"}
{"name":"gha-uses-commit","key":"https://github.com/docker/login-action.git:v4.2.0","version":"650006c6eb7dba73a995cc03b0b2d7f5ca915bee"}
{"name":"gha-uses-commit","key":"https://github.com/docker/setup-buildx-action.git:v4.1.0","version":"d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5"}
{"name":"gha-uses-commit","key":"https://github.com/regclient/actions.git:main","version":"c70ad64367908075211b10dcd2ab9fad4bfa1816"}
{"name":"gha-uses-commit","key":"https://github.com/sigstore/cosign-installer.git:v4.1.2","version":"6f9f17788090df1f26f669e9d70d6ae9567deba6"}
{"name":"gha-uses-commit","key":"https://github.com/softprops/action-gh-release.git:v3.0.0","version":"b4309332981a82ec1c5618f44dd2e27cc8bfbfda"}
{"name":"gha-uses-semver","key":"https://github.com/actions/checkout.git","version":"v6.0.2"}
{"name":"gha-uses-semver","key":"https://github.com/actions/setup-go.git","version":"v6.4.0"}
{"name":"gha-uses-semver","key":"https://github.com/actions/stale.git","version":"v10.3.0"}
{"name":"gha-uses-semver","key":"https://github.com/actions/upload-artifact.git","version":"v7.0.1"}
{"name":"gha-uses-semver","key":"https://github.com/anchore/sbom-action.git","version":"v0.24.0"}
{"name":"gha-uses-semver","key":"https://github.com/docker/build-push-action.git","version":"v7.2.0"}
{"name":"gha-uses-semver","key":"https://github.com/docker/login-action.git","version":"v4.2.0"}
{"name":"gha-uses-semver","key":"https://github.com/docker/setup-buildx-action.git","version":"v4.1.0"}
{"name":"gha-uses-semver","key":"https://github.com/sigstore/cosign-installer.git","version":"v4.1.2"}
{"name":"gha-uses-semver","key":"https://github.com/softprops/action-gh-release.git","version":"v3.0.0"}
{"name":"go-mod-golang-release","key":"golang-oldest","version":"1.25.0"}
{"name":"makefile-ci-distribution","key":"docker.io/library/registry","version":"3.1.1"}
{"name":"makefile-ci-zot","key":"ghcr.io/project-zot/zot-linux-amd64","version":"v2.1.17"}
{"name":"makefile-go-vulncheck","key":"https://go.googlesource.com/vuln.git","version":"v1.3.0"}
{"name":"makefile-gofumpt","key":"https://github.com/mvdan/gofumpt.git","version":"v0.10.0"}
{"name":"makefile-gomajor","key":"https://github.com/icholy/gomajor.git","version":"v0.15.0"}
{"name":"makefile-gosec","key":"https://github.com/securego/gosec.git","version":"v2.26.1"}
{"name":"makefile-markdown-lint","key":"docker.io/davidanson/markdownlint-cli2","version":"v0.22.1"}
{"name":"makefile-osv-scanner","key":"https://github.com/google/osv-scanner.git","version":"v2.3.8"}
{"name":"makefile-staticcheck","key":"https://github.com/dominikh/go-tools.git","version":"v0.7.0"}
{"name":"makefile-syft-container-digest","key":"anchore/syft:v1.44.0","version":"sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6"}
{"name":"makefile-syft-container-tag","key":"anchore/syft","version":"v1.44.0"}
{"name":"makefile-syft-version","key":"docker.io/anchore/syft","version":"v1.44.0"}
{"name":"osv-golang-release","key":"docker.io/library/golang","version":"1.26.3"}
{"name":"shell-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
{"name":"shell-alpine-tag-base","key":"docker.io/library/alpine","version":"3"}
{"name":"shell-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.23.4"}
+346
View File
@@ -0,0 +1,346 @@
files:
"build/Dockerfile*":
processors:
- docker-arg-alpine-tag
- docker-arg-alpine-digest
- docker-arg-go-tag
- docker-arg-go-digest
- docker-arg-ecr
- docker-arg-gcr
- docker-arg-lunajson
- docker-arg-semver
"build/oci-image.sh":
processors:
- shell-alpine-tag-base
- shell-alpine-tag-comment
- shell-alpine-digest
".github/workflows/*.yml":
processors:
- gha-golang-matrix
- gha-golang-release
- gha-uses-vx
- gha-uses-semver
- gha-uses-commit
- gha-syft-version
- gha-cosign-version
- gha-alpine-tag-base
- gha-alpine-tag-comment
- gha-alpine-digest
"Makefile":
processors:
- makefile-gofumpt
- makefile-gomajor
- makefile-go-vulncheck
- makefile-markdown-lint
- makefile-gosec
- makefile-osv-scanner
- makefile-staticcheck
- makefile-syft-version
- makefile-syft-container-tag
- makefile-syft-container-digest
- makefile-ci-distribution
- makefile-ci-zot
"go.mod":
processors:
- go-mod-golang-release
".osv-scanner.toml":
processors:
- osv-golang-release
x-processor-tmpl:
git-commit: &git-commit
key: "{{ .SourceArgs.url }}:{{ .SourceArgs.ref }}"
scan: "regexp"
source: "git-commit"
filter:
expr: "^{{ .SourceArgs.ref }}$"
git-tag-semver: &git-tag-semver
key: "{{ .SourceArgs.url }}"
scan: "regexp"
source: "git-tag"
filter:
expr: '^v?\d+\.\d+\.\d+$'
sort:
method: "semver"
registry-digest: &registry-digest
key: "{{ .SourceArgs.image }}"
scan: "regexp"
source: "registry-digest"
registry-tag-semver: &registry-tag-semver
key: "{{ .SourceArgs.repo }}"
scan: "regexp"
source: "registry-tag"
filter:
expr: '^v?\d+\.\d+\.\d+$'
sort:
method: "semver"
processors:
docker-arg-alpine-tag:
<<: *registry-tag-semver
scanArgs:
regexp: '^ARG ALPINE_VER=(?P<Version>v?\d+\.\d+\.\d+)@(?P<SHA>sha256:[0-9a-f]+)\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
docker-arg-alpine-digest:
<<: *registry-digest
scanArgs:
regexp: '^ARG ALPINE_VER=(?P<Tag>v?\d+\.\d+\.\d+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
sourceArgs:
image: "docker.io/library/alpine:{{.ScanMatch.Tag}}"
docker-arg-go-tag:
<<: *registry-tag-semver
scanArgs:
regexp: '^ARG GO_VER=(?P<Version>[a-z0-9\-\.]+)-alpine@(?P<SHA>sha256:[0-9a-f]+)\s*$'
sourceArgs:
repo: "docker.io/library/golang"
docker-arg-go-digest:
<<: *registry-digest
scanArgs:
regexp: '^ARG GO_VER=(?P<Tag>[a-z0-9\-\.]+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
sourceArgs:
image: "docker.io/library/golang:{{.ScanMatch.Tag}}"
docker-arg-ecr:
<<: *git-tag-semver
scanArgs:
regexp: '^ARG ECR_HELPER_VER=(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
url: "https://github.com/awslabs/amazon-ecr-credential-helper.git"
# get the version for the ecr-login nested package in the repo
filter:
expr: '^ecr-login/v?\d+\.\d+\.\d+$'
# sort and output only the version number without the package name prefix
sort:
method: "semver"
template: '{{ index (split . "/") 1 }}'
template: '{{ index (split .Version "/") 1 }}'
docker-arg-gcr:
<<: *git-tag-semver
scanArgs:
regexp: '^ARG GCR_HELPER_VER=(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
url: "https://github.com/GoogleCloudPlatform/docker-credential-gcr.git"
docker-arg-lunajson:
<<: *git-commit
scanArgs:
regexp: '^ARG LUNAJSON_COMMIT=(?P<Version>[0-9a-f]+)\s*$'
sourceArgs:
url: "https://github.com/grafi-tt/lunajson.git"
ref: master
docker-arg-semver:
<<: *git-commit
scanArgs:
regexp: '^ARG SEMVER_COMMIT=(?P<Version>[0-9a-f]+)\s*$'
sourceArgs:
url: "https://github.com/kikito/semver.lua.git"
ref: master
gha-alpine-digest:
<<: *registry-digest
scanArgs:
regexp: '^\s*ALPINE_DIGEST: "(?P<Version>sha256:[0-9a-f]+)"\s*#\s*(?P<Tag>\d+\.\d+\.\d+)\s*$'
sourceArgs:
image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}"
gha-alpine-tag-base:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_NAME: "alpine:(?P<Version>v?\d+)"\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
# only return the major version number in the tag to support detecting a change in the base image
template: '{{ index ( split .Version "." ) 0 }}'
gha-alpine-tag-comment:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_DIGEST: "(?P<Digest>sha256:[0-9a-f]+)"\s*#\s*(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
gha-cosign-version:
<<: *git-tag-semver
scanArgs:
regexp: '^\s*cosign-release: "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
url: "https://github.com/sigstore/cosign.git"
filter:
expr: '^v?3\.\d+\.\d+$' # pin to v3, v4 will remove support for older clients
gha-golang-matrix:
<<: *registry-tag-semver
key: "golang-matrix"
scanArgs:
regexp: '^\s*gover: (?P<Version>\[["0-9, \.]+\])\s*$'
sourceArgs:
repo: "docker.io/library/golang"
filter:
expr: '^v?\d+\.\d+$'
template: '["{{ index .VerMap ( index .VerList 1 ) }}", "{{ index .VerMap ( index .VerList 0 ) }}"]'
gha-golang-release:
<<: *registry-tag-semver
key: "golang-latest"
scanArgs:
regexp: '^\s*RELEASE_GO_VER: "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
repo: "docker.io/library/golang"
filter:
expr: '^v?\d+\.\d+$'
gha-syft-version:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*syft-version: "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
repo: "docker.io/anchore/syft"
gha-uses-vx:
<<: *git-tag-semver
scanArgs:
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Commit>[0-9a-f]+)\s+#\s+(?P<Version>v?\d+)\s*$'
sourceArgs:
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
filter:
expr: '^v?\d+$'
gha-uses-semver:
<<: *git-tag-semver
scanArgs:
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Commit>[0-9a-f]+)\s+#\s+(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
gha-uses-commit:
<<: *git-commit
scanArgs:
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Version>[0-9a-f]+)\s+#\s+(?P<Ref>[\w\d\.]+)\s*$'
sourceArgs:
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
ref: "{{ .ScanMatch.Ref }}"
go-mod-golang-release:
<<: *registry-tag-semver
key: "golang-oldest"
scanArgs:
regexp: '^go (?P<Version>[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/library/golang"
filter:
expr: '^\d+\.\d+$'
template: '{{ index .VerMap ( index .VerList 1 ) }}.0'
makefile-ci-distribution:
<<: *registry-tag-semver
scanArgs:
regexp: '^CI_DISTRIBUTION_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/library/registry"
makefile-ci-zot:
<<: *registry-tag-semver
scanArgs:
regexp: '^CI_ZOT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
repo: "ghcr.io/project-zot/zot-linux-amd64"
makefile-gofumpt:
<<: *git-tag-semver
scanArgs:
regexp: '^GOFUMPT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/mvdan/gofumpt.git"
makefile-gomajor:
<<: *git-tag-semver
scanArgs:
regexp: '^GOMAJOR_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/icholy/gomajor.git"
makefile-gosec:
<<: *git-tag-semver
scanArgs:
regexp: '^GOSEC_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/securego/gosec.git"
makefile-go-vulncheck:
<<: *git-tag-semver
scanArgs:
regexp: '^GO_VULNCHECK_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://go.googlesource.com/vuln.git"
makefile-markdown-lint:
<<: *registry-tag-semver
scanArgs:
regexp: '^MARKDOWN_LINT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/davidanson/markdownlint-cli2"
makefile-osv-scanner:
<<: *git-tag-semver
scanArgs:
regexp: '^OSV_SCANNER_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/google/osv-scanner.git"
makefile-staticcheck:
<<: *git-tag-semver
scanArgs:
regexp: '^STATICCHECK_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/dominikh/go-tools.git"
filter:
# repo also has dated tags, ignore versions without a preceding "v"
expr: '^v\d+\.\d+\.\d+$'
makefile-syft-container-tag:
<<: *registry-tag-semver
scanArgs:
regexp: '^SYFT_CONTAINER\?=(?P<Repo>[^:]*):(?P<Version>v?[0-9\.]+)@(?P<Digest>sha256:[0-9a-f]+)\s*$'
sourceArgs:
repo: "{{ .ScanMatch.Repo }}"
makefile-syft-container-digest:
<<: *registry-digest
scanArgs:
regexp: '^SYFT_CONTAINER\?=(?P<Image>[^:]*):(?P<Tag>v?[0-9\.]+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
sourceArgs:
image: "{{ .ScanMatch.Image }}:{{.ScanMatch.Tag}}"
makefile-syft-version:
<<: *registry-tag-semver
scanArgs:
regexp: '^SYFT_VERSION\?=(?P<Version>v[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/anchore/syft"
osv-golang-release:
<<: *registry-tag-semver
scanArgs:
regexp: '^GoVersionOverride = "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
repo: "docker.io/library/golang"
shell-alpine-tag-base:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_NAME="alpine:(?P<Version>v?\d+)"\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
# only return the major version number in the tag to support detecting a change in the base image
template: '{{ index ( split .Version "." ) 0 }}'
shell-alpine-tag-comment:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_DIGEST="(?P<Digest>sha256:[0-9a-f]+)"\s*#\s*(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
shell-alpine-digest:
<<: *registry-digest
scanArgs:
regexp: '^\s*ALPINE_DIGEST="(?P<Version>sha256:[0-9a-f]+)"\s*#\s*(?P<Tag>\d+\.\d+\.\d+)\s*$'
sourceArgs:
image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}"
scans:
regexp:
type: "regexp"
sources:
git-commit:
type: "git"
args:
type: "commit"
git-tag:
type: "git"
args:
type: "tag"
registry-digest:
type: "registry"
registry-tag:
type: "registry"
args:
type: "tag"
+134
View File
@@ -0,0 +1,134 @@
# Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
- Demonstrating empathy and kindness toward other people
- Being respectful of differing opinions, viewpoints, and experiences
- Giving and gracefully accepting constructive feedback
- Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
- Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior include:
- The use of sexualized language or imagery, and sexual attention or advances of
any kind
- Trolling, insulting or derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or email address,
without their explicit permission
- Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
<git@bmitch.net> or slack (I'm found on the CNCF, Docker, OCI, and OpenSSF
slacks).
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
[homepage]: https://www.contributor-covenant.org
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[Mozilla CoC]: https://github.com/mozilla/diversity
[FAQ]: https://www.contributor-covenant.org/faq
[translations]: https://www.contributor-covenant.org/translations
+80
View File
@@ -0,0 +1,80 @@
# Contributing
## Reporting security issues
Please see [SECURITY.md](security.md) for the process to report security issues.
## Reporting other issues
Please search for similar issues and if none are seen, report an issue at [github.com/regclient/regclient/issues](https://github.com/regclient/regclient/issues).
## Code style
This project attempts to follow these principles:
- Code is canonical Go, following styles and patterns commonly used by the Go community.
- Dependencies outside of the Go standard library should be minimized.
- Dependencies should be pinned to a specific digest and tracked by Go or version-check.
- Unit tests are strongly encouraged with a focus on test coverage of the successful path and common errors.
- Linters and other style formatting tools are used, please run `make all` before committing any changes.
## LLM Policy
This project expects all contributions to be developed by a human or created with a reproducible tool.
Developers using an AI/LLM tool to generate their contribution are expected to fully understand the entire contribution and the logic behind its design.
Contributions that appear to have been generated by an AI/LLM without a human review may result in a ban from future contributions to the project.
## Pull requests
PRs are welcome following the below guides:
- For anything beyond a minor fix, opening an issue is suggested to discuss possible solutions.
- Changes should be rebased on the main branch.
- Changes should be squashed to a single commit per logical change.
All changes must be signed (`git commit -s`) to indicate you agree to the [Developer Certificate or Origin](https://developercertificate.org/):
```text
Developer Certificate of Origin
Version 1.1
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
Everyone is permitted to copy and distribute verbatim copies of this
license document, but changing it is not allowed.
Developer's Certificate of Origin 1.1
By making a contribution to this project, I certify that:
(a) The contribution was created in whole or in part by me and I
have the right to submit it under the open source license
indicated in the file; or
(b) The contribution is based upon previous work that, to the best
of my knowledge, is covered under an appropriate open source
license and I have the right under that license to submit that
work with modifications, whether created in whole or in part
by me, under the same open source license (unless I am
permitted to submit under a different license), as indicated
in the file; or
(c) The contribution was provided directly to me by some other
person who certified (a), (b) or (c) and I have not modified
it.
(d) I understand and agree that this project and the contribution
are public and that a record of the contribution (including all
personal information I submit with it, including my sign-off) is
maintained indefinitely and may be redistributed consistent with
this project or the open source license(s) involved.
```
The sign-off will include the following message in your commit:
```text
Signed-off-by: Your Name <your-email@example.org>
```
This needs to be your real name, no aliases please.
+191
View File
@@ -0,0 +1,191 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
Copyright 2020 The regclient Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+285
View File
@@ -0,0 +1,285 @@
COMMANDS?=regctl regsync regbot
BINARIES?=$(addprefix bin/,$(COMMANDS))
IMAGES?=$(addprefix docker-,$(COMMANDS))
ARTIFACT_PLATFORMS?=linux-amd64 linux-arm64 linux-ppc64le linux-s390x linux-riscv64 darwin-amd64 darwin-arm64 windows-amd64.exe freebsd-amd64
ARTIFACTS?=$(foreach cmd,$(addprefix artifacts/,$(COMMANDS)),$(addprefix $(cmd)-,$(ARTIFACT_PLATFORMS)))
IMAGE_PLATFORMS?=linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/riscv64
VCS_REPO?="https://github.com/regclient/regclient.git"
VCS_REF?=$(shell git rev-list -1 HEAD)
ifneq ($(shell git status --porcelain 2>/dev/null),)
VCS_REF := $(VCS_REF)-dirty
endif
VCS_VERSION?=$(shell vcs_describe="$$(git describe --all)"; \
vcs_version="(devel)"; \
if [ "$${vcs_describe}" != "$${vcs_describe#tags/}" ]; then \
vcs_version="$${vcs_describe#tags/}"; \
elif [ "$${vcs_describe}" != "$${vcs_describe#heads/}" ]; then \
vcs_version="$${vcs_describe#heads/}"; \
if [ "main" = "$${vcs_version}" ]; then vcs_version=edge; fi; \
fi; \
echo "$${vcs_version}" | sed -r 's#/+#-#g')
VCS_TAG?=$(shell git describe --tags --abbrev=0 2>/dev/null || true)
VCS_SEC?=$(shell git log -1 --format=%ct)
VCS_DATE?=$(shell date -d "@$(VCS_SEC)" +%Y-%m-%dT%H:%M:%SZ --utc)
LD_FLAGS?=-s -w -extldflags -static -buildid= -X \"github.com/regclient/regclient/internal/version.vcsTag=$(VCS_TAG)\"
GO_BUILD_FLAGS?=-trimpath -ldflags "$(LD_FLAGS)"
DOCKERFILE_EXT?=$(shell if docker build --help 2>/dev/null | grep -q -- '--progress'; then echo ".buildkit"; fi)
DOCKER_ARGS?=--build-arg "VCS_REF=$(VCS_REF)" --build-arg "VCS_VERSION=$(VCS_VERSION)" --build-arg "SOURCE_DATE_EPOCH=$(VCS_SEC)" --build-arg "BUILD_DATE=$(VCS_DATE)"
GOPATH?=$(shell go env GOPATH)
PWD:=$(shell pwd)
VER_BUMP?=$(shell command -v version-bump 2>/dev/null)
VER_BUMP_CONTAINER?=sudobmitch/version-bump:edge
ifeq "$(strip $(VER_BUMP))" ''
VER_BUMP=docker run --rm \
-v "$(shell pwd)/:$(shell pwd)/" -w "$(shell pwd)" \
-u "$(shell id -u):$(shell id -g)" \
$(VER_BUMP_CONTAINER)
endif
MARKDOWN_LINT_VER?=v0.22.1
GOFUMPT_VER?=v0.10.0
GOMAJOR_VER?=v0.15.0
GOSEC_VER?=v2.26.1
GO_VULNCHECK_VER?=v1.3.0
OSV_SCANNER_VER?=v2.3.8
SYFT?=$(shell command -v syft 2>/dev/null)
SYFT_CMD_VER:=$(shell [ -x "$(SYFT)" ] && echo "v$$($(SYFT) version | awk '/^Version: / {print $$2}')" || echo "0")
SYFT_VERSION?=v1.44.0
SYFT_CONTAINER?=anchore/syft:v1.44.0@sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6
ifneq "$(SYFT_CMD_VER)" "$(SYFT_VERSION)"
SYFT=docker run --rm \
-v "$(shell pwd)/:$(shell pwd)/" -w "$(shell pwd)" \
-u "$(shell id -u):$(shell id -g)" \
$(SYFT_CONTAINER)
endif
STATICCHECK_VER?=v0.7.0
CI_DISTRIBUTION_VER?=3.1.1
CI_ZOT_VER?=v2.1.17
.PHONY: .FORCE
.FORCE:
.PHONY: all
all: fmt gofumpt gofix goimports vet test lint binaries ## Full build of Go binaries (including fmt, vet, test, and lint)
.PHONY: fmt
fmt: ## go fmt
go fmt ./...
.PHONY: gofumpt
gofumpt: $(GOPATH)/bin/gofumpt ## gofumpt is a stricter alternative to go fmt
gofumpt -l -w .
.PHONY: gofix
gofix: ## go fix
go fix ./...
goimports: $(GOPATH)/bin/goimports
$(GOPATH)/bin/goimports -w -format-only -local github.com/regclient .
.PHONY: vet
vet: ## go vet
go vet ./...
.PHONY: test
test: ## go test
go test -cover -race ./...
.PHONY: lint
lint: lint-go lint-goimports lint-md lint-gosec ## Run all linting
.PHONY: lint-go
lint-go: $(GOPATH)/bin/gofumpt $(GOPATH)/bin/staticcheck .FORCE ## Run linting for Go
$(GOPATH)/bin/staticcheck -checks all ./...
$(GOPATH)/bin/gofumpt -l -d .
errors=$$(go fix -diff ./...); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi
lint-goimports: $(GOPATH)/bin/goimports
@if [ -n "$$($(GOPATH)/bin/goimports -l -format-only -local github.com/regclient .)" ]; then \
echo $(GOPATH)/bin/goimports -d -format-only -local github.com/regclient .; \
$(GOPATH)/bin/goimports -d -format-only -local github.com/regclient .; \
exit 1; \
fi
# excluding types/platform pending resultion to https://github.com/securego/gosec/issues/1116
.PHONY: lint-gosec
lint-gosec: $(GOPATH)/bin/gosec .FORCE ## Run gosec
$(GOPATH)/bin/gosec -terse -exclude-dir types/platform ./...
.PHONY: lint-md
lint-md: .FORCE ## Run linting for markdown
docker run --rm -v "$(PWD):/workdir:ro" davidanson/markdownlint-cli2:$(MARKDOWN_LINT_VER) \
"**/*.md" "#vendor"
.PHONY: vulnerability-scan
vulnerability-scan: osv-scanner vulncheck-go ## Run all vulnerability scanners
.PHONY: osv-scanner
osv-scanner: $(GOPATH)/bin/osv-scanner .FORCE ## Run OSV Scanner
$(GOPATH)/bin/osv-scanner scan --config .osv-scanner.toml -r --licenses="Apache-2.0,BSD-3-Clause,MIT,CC-BY-SA-4.0,UNKNOWN" .
.PHONY: vulncheck-go
vulncheck-go: $(GOPATH)/bin/govulncheck .FORCE ## Run govulncheck
$(GOPATH)/bin/govulncheck ./...
.PHONY: vendor
vendor: ## Vendor Go modules
go mod vendor
.PHONY: binaries
binaries: $(BINARIES) ## Build Go binaries
bin/%: .FORCE
CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o bin/$* ./cmd/$*
.PHONY: docker
docker: $(IMAGES) ## Build Docker images
docker-%: .FORCE
docker build -t regclient/$* -f build/Dockerfile.$*$(DOCKERFILE_EXT) $(DOCKER_ARGS) .
docker build -t regclient/$*:alpine -f build/Dockerfile.$*$(DOCKERFILE_EXT) --target release-alpine $(DOCKER_ARGS) .
.PHONY: oci-image
oci-image: $(addprefix oci-image-,$(COMMANDS)) ## Build reproducible images to an OCI Layout
oci-image-%: bin/regctl .FORCE
PATH="$(PWD)/bin:$(PATH)" build/oci-image.sh -r scratch -i "$*" -p "$(IMAGE_PLATFORMS)"
PATH="$(PWD)/bin:$(PATH)" build/oci-image.sh -r alpine -i "$*" -p "$(IMAGE_PLATFORMS)" -b "alpine:3"
.PHONY: test-docker
test-docker: $(addprefix test-docker-,$(COMMANDS)) ## Build multi-platform docker images (but do not tag)
test-docker-%:
docker buildx build --platform="$(IMAGE_PLATFORMS)" -f build/Dockerfile.$*.buildkit .
docker buildx build --platform="$(IMAGE_PLATFORMS)" -f build/Dockerfile.$*.buildkit --target release-alpine .
.PHONY: ci
ci: ci-distribution ci-zot ## Run CI tests against self hosted registries
.PHONY: ci-distribution
ci-distribution:
docker run --rm -d -p 5000 \
--label regclient-ci=true --name regclient-ci-distribution \
-e "REGISTRY_STORAGE_DELETE_ENABLED=true" \
docker.io/library/registry:$(CI_DISTRIBUTION_VER)
./build/ci-test.sh -t localhost:$$(docker port regclient-ci-distribution 5000 | head -1 | cut -f2 -d:)/test-ci
docker stop regclient-ci-distribution
.PHONY: ci-zot
ci-zot:
docker run --rm -d -p 5000 \
--label regclient-ci=true --name regclient-ci-zot \
-v "$$(pwd)/build/zot-config.json:/etc/zot/config.json:ro" \
ghcr.io/project-zot/zot-linux-amd64:$(CI_ZOT_VER)
./build/ci-test.sh -t localhost:$$(docker port regclient-ci-zot 5000 | head -1 | cut -f2 -d:)/test-ci
docker stop regclient-ci-zot
.PHONY: artifacts
artifacts: $(ARTIFACTS) ## Generate artifacts
.PHONY: artifact-pre
artifact-pre:
mkdir -p artifacts
artifacts/%: artifact-pre .FORCE
@set -e; \
target="$*"; \
command="$${target%%-*}"; \
platform_ext="$${target#*-}"; \
platform="$${platform_ext%.*}"; \
export GOOS="$${platform%%-*}"; \
export GOARCH="$${platform#*-}"; \
echo export GOOS=$${GOOS}; \
echo export GOARCH=$${GOARCH}; \
echo go build ${GO_BUILD_FLAGS} -o "$@" ./cmd/$${command}/; \
CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o "$@" ./cmd/$${command}/; \
$(SYFT) scan -q "file:$@" --source-name "$${command}" -o cyclonedx-json >"artifacts/$${command}-$${platform}.cyclonedx.json"; \
$(SYFT) scan -q "file:$@" --source-name "$${command}" -o spdx-json >"artifacts/$${command}-$${platform}.spdx.json"
.PHONY: clean
clean: ## delete generated content
[ ! -d artifacts ] || rm -r artifacts
[ ! -d bin ] || rm -r bin
[ ! -d output ] || rm -r output
[ ! -d vendor ] || rm -r vendor
.PHONY: plugin-user
plugin-user:
mkdir -p ${HOME}/.docker/cli-plugins/
cp docker-plugin/docker-regclient ${HOME}/.docker/cli-plugins/docker-regctl
.PHONY: plugin-host
plugin-host:
sudo cp docker-plugin/docker-regclient /usr/libexec/docker/cli-plugins/docker-regctl
.PHONY: util-golang-major
util-golang-major: $(GOPATH)/bin/gomajor ## check for major dependency updates
$(GOPATH)/bin/gomajor list
.PHONY: util-golang-update
util-golang-update: ## update go module versions
go get -u -t ./...
go mod tidy
[ ! -d vendor ] || go mod vendor
.PHONY: util-release-preview
util-release-preview: $(GOPATH)/bin/gorelease ## preview changes for next release
git checkout main
./.github/release.sh -d
gorelease
.PHONY: util-release-run
util-release-run: ## generate a new release
git checkout main
./.github/release.sh
.PHONY: util-version-check
util-version-check: ## check all dependencies for updates
$(VER_BUMP) check
.PHONY: util-version-update
util-version-update: ## update versions on all dependencies
$(VER_BUMP) update
$(GOPATH)/bin/gofumpt: .FORCE
@[ -f "$(GOPATH)/bin/gofumpt" ] \
&& [ "$$($(GOPATH)/bin/gofumpt -version | cut -f 1 -d ' ')" = "$(GOFUMPT_VER)" ] \
|| go install mvdan.cc/gofumpt@$(GOFUMPT_VER)
$(GOPATH)/bin/gomajor: .FORCE
@[ -f "$(GOPATH)/bin/gomajor" ] \
&& [ "$$($(GOPATH)/bin/gomajor version | grep '^version' | cut -f 2 -d ' ')" = "$(GOMAJOR_VER)" ] \
|| go install github.com/icholy/gomajor@$(GOMAJOR_VER)
$(GOPATH)/bin/goimports: .FORCE
@[ -f "$(GOPATH)/bin/goimports" ] && [ "$$(go version | cut -f3 -d' ')" = "$$(go version $(GOPATH)/bin/goimports | cut -f2 -d' ')" ] \
|| go install golang.org/x/tools/cmd/goimports@latest
$(GOPATH)/bin/gorelease: .FORCE
@[ -f "$(GOPATH)/bin/gorelease" ] && [ "$$(go version | cut -f3 -d' ')" = "$$(go version $(GOPATH)/bin/gorelease | cut -f2 -d' ')" ] \
|| go install golang.org/x/exp/cmd/gorelease@latest
$(GOPATH)/bin/gosec: .FORCE
@[ -f $(GOPATH)/bin/gosec ] \
&& [ "$$($(GOPATH)/bin/gosec -version | grep '^Version' | cut -f 2 -d ' ')" = "$(GOSEC_VER)" ] \
|| go install -ldflags '-X main.Version=$(GOSEC_VER) -X main.GitTag=$(GOSEC_VER)' \
github.com/securego/gosec/v2/cmd/gosec@$(GOSEC_VER)
$(GOPATH)/bin/staticcheck: .FORCE
@[ -f $(GOPATH)/bin/staticcheck ] \
&& [ "$$($(GOPATH)/bin/staticcheck -version | cut -f 3 -d ' ' | tr -d '()')" = "$(STATICCHECK_VER)" ] \
|| go install "honnef.co/go/tools/cmd/staticcheck@$(STATICCHECK_VER)"
$(GOPATH)/bin/govulncheck: .FORCE
@[ -f $(GOPATH)/bin/govulncheck ] \
&& [ $$(go version -m $(GOPATH)/bin/govulncheck | \
awk -F ' ' '{ if ($$1 == "mod" && $$2 == "golang.org/x/vuln") { printf "%s\n", $$3 } }') = "$(GO_VULNCHECK_VER)" ] \
|| CGO_ENABLED=0 go install "golang.org/x/vuln/cmd/govulncheck@$(GO_VULNCHECK_VER)"
$(GOPATH)/bin/osv-scanner: .FORCE
@[ -f $(GOPATH)/bin/osv-scanner ] \
&& [ "$$(osv-scanner --version | awk -F ': ' '{ if ($$1 == "osv-scanner version") { printf "%s\n", $$2 } }')" = "$(OSV_SCANNER_VER)" ] \
|| CGO_ENABLED=0 go install "github.com/google/osv-scanner/v2/cmd/osv-scanner@$(OSV_SCANNER_VER)"
.PHONY: help
help: # Display help
@awk -F ':|##' '/^[^\t].+?:.*?##/ { printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF }' $(MAKEFILE_LIST)
+116
View File
@@ -0,0 +1,116 @@
# regclient
[![Go Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/go.yml?branch=main&label=Go%20build)](https://github.com/regclient/regclient/actions/workflows/go.yml)
[![Docker Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/docker.yml?branch=main&label=Docker%20build)](https://github.com/regclient/regclient/actions/workflows/docker.yml)
[![Dependency Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/version-check.yml?branch=main&label=Dependency%20check)](https://github.com/regclient/regclient/actions/workflows/version-check.yml)
[![Vulnerability Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/vulnscans.yml?branch=main&label=Vulnerability%20check)](https://github.com/regclient/regclient/actions/workflows/vulnscans.yml)
[![Go Reference](https://pkg.go.dev/badge/github.com/regclient/regclient.svg)](https://pkg.go.dev/github.com/regclient/regclient)
![License](https://img.shields.io/github/license/regclient/regclient)
[![Go Report Card](https://goreportcard.com/badge/github.com/regclient/regclient)](https://goreportcard.com/report/github.com/regclient/regclient)
[![GitHub Downloads](https://img.shields.io/github/downloads/regclient/regclient/total?label=GitHub%20downloads)](https://github.com/regclient/regclient/releases)
regclient is a client interface to OCI conformant registries and content shipped with the OCI Image Layout.
It includes a Go library and several CLI commands.
## regclient Go Library Features
- Runs without a container runtime and without privileged access to the local host.
- Querying for a tag listing, repository listing, and remotely inspecting the contents of images.
- Efficiently copying and retagging images, only pulling layers when required, and without changing the image digest.
- Support for multi-platform images.
- Support for querying, creating, and copying OCI Artifacts, allowing arbitrary data to be stored in an OCI registry.
- Support for packaging OCI Artifacts with an Index of multiple artifacts, which can be used for platform specific artifacts.
- Support for querying OCI referrers, copying referrers, and pushing content with an OCI subject field, associating artifacts with other content on the registry.
- Support for the “digest tags” used by projects like sigstore/cosign, allowing the content to be included when copying images.
- Efficiently query for an image digest.
- Efficiently query for pull rate limits used by Docker Hub.
- Import and export content into OCI Layouts and Docker formatted tar files.
- Support OCI Layouts in all commands as a local disk equivalent of a repository.
- Support for deleting tags, manifests, and blobs.
- Ability to mutate existing images, including:
- Settings annotations or labels
- Deleting content from layers
- Changing timestamps for reproducibility
- Converting between Docker and OCI media types
- Replacing the base image layers
- Add or remove volumes and exposed ports
- Change digest algorithms
- Support for registry warning headers, which may be used to notify users of issues with the server or content they are using.
- Automatically import logins from the docker CLI, and registry certificates from the docker engine.
- Automatic retry, and fallback to a chunked blob push, when network issues are encountered.
The full Go references is available on [pkg.go.dev](https://pkg.go.dev/github.com/regclient/regclient).
## regctl Features
`regctl` is a CLI interface to the `regclient` library.
In addition to the features listed for `regclient`, `regctl` adds the following abilities:
- Generating multi-platform manifests from multiple images that may have been separately built.
- Repackage a multi-platform image with only the requested platforms.
- Push and pull arbitrary OCI artifacts.
- Recursively list all content associated with an image.
- Extract files from a layer or image.
- Compare images, showing the differences between manifests, the config, and layers.
- Formatted output using Go templates.
The project website includes [usage instructions](https://regclient.org/usage/regctl/) and a [CLI reference](https://regclient.org/cli/regctl/).
## regsync features
`regsync` is an image mirroring tool.
It will copy images between two locations with the following additional features:
- Ability to run on a cron schedule, one time synchronization, or only report stale images.
- Uses a yaml configuration.
- Each source may be an entire registry (not recommended), a repository, or a single image, with the ability to filter repositories and tags.
- Support for multi-platform images, OCI referrers, “digest tags”, and copying to or from an OCI Layout (for maintaining a mirror over an air-gap).
- Ability to mirror multiple images concurrently.
- Support for copying a single platform from multi-platform images.
- Ability to backup an existing image before overwriting the tag.
- Ability to postpone mirror step when rate limit (used by Docker Hub) is below a threshold.
- Can use users docker configuration for user credentials and registry certificates.
The project website includes [usage instructions](https://regclient.org/usage/regsync/) and a [CLI reference](https://regclient.org/cli/regsync/).
## regbot features
`regbot` is a scripting tool on top of the `regclient` API with the following features:
- Ability to run on a cron schedule, one time execution, or test with a dry-run mode.
- Uses a yaml configuration.
- Scripts are written in Lua and executed directly in Go.
- Built-in functions include:
- Repository list
- Tag list
- Image manifest (either head or get, and optional resolving multi-platform reference)
- Image config (this includes the creation time, labels, and other details shown in a docker image inspect)
- Image rate limit and a wait function to delay the script when rate limit remaining is below a threshold
- Image copy
- Manifest delete
- Tag delete
The project website includes [usage instructions](https://regclient.org/usage/regbot/) and a [CLI reference](https://regclient.org/cli/regbot/).
## Development Status
This project is using v0 version numbers due to Go's backwards compatibility requirements of a v1 release.
The library and commands are stable for external use.
Minor version updates may contain breaking changes, however effort is made to first deprecate and provide warnings to give users time to move off of older APIs and commands.
## Installing
See the [installation instructions](https://regclient.org/install/) on the project website for the various ways to download or build CLI binaries.
## Usage
See the [project documentation](https://regclient.org/usage/).
## Contributors
<a href="https://github.com/regclient/regclient/graphs/contributors">
<img src="https://contrib.rocks/image?repo=regclient/regclient" alt="contributor list"/>
</a>
<!-- markdownlint-disable-file MD033 -->
+5
View File
@@ -0,0 +1,5 @@
# Reporting security issues
Please report security issues directly in GitHub at <https://github.com/regclient/regclient/security/advisories/new> or alternatively email <git@bmitch.net>.
We will typically respond within 7 working days of your report. If the issue is confirmed as a vulnerability, we will open a Security Advisory and acknowledge your contributions as part of it. This project follows a 90 day disclosure timeline.
+283
View File
@@ -0,0 +1,283 @@
package regclient
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"log/slog"
"time"
"github.com/regclient/regclient/internal/pqueue"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types"
"github.com/regclient/regclient/types/blob"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/warning"
)
const blobCBFreq = time.Millisecond * 100
type blobOpt struct {
callback func(kind types.CallbackKind, instance string, state types.CallbackState, cur, total int64)
readerHook func(*blob.BReader) (*blob.BReader, error)
}
// BlobOpts define options for the Image* commands.
type BlobOpts func(*blobOpt)
// BlobWithCallback provides progress data to a callback function.
func BlobWithCallback(callback func(kind types.CallbackKind, instance string, state types.CallbackState, cur, total int64)) BlobOpts {
return func(opts *blobOpt) {
opts.callback = callback
}
}
// BlobWithReaderHook is called in [RegClient.BlobCopy] with the blob source.
// The returned [blob.BReader] is pushed to the target.
// If the hook returns an error, the copy will fail.
func BlobWithReaderHook(hook func(*blob.BReader) (*blob.BReader, error)) BlobOpts {
return func(opts *blobOpt) {
opts.readerHook = hook
}
}
// BlobCopy copies a blob between two locations.
// If the blob already exists in the target, the copy is skipped.
// A server side cross repository blob mount is attempted.
func (rc *RegClient) BlobCopy(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor, opts ...BlobOpts) error {
if !refSrc.IsSetRepo() {
return fmt.Errorf("refSrc is not set: %s%.0w", refSrc.CommonName(), errs.ErrInvalidReference)
}
if !refTgt.IsSetRepo() {
return fmt.Errorf("refTgt is not set: %s%.0w", refTgt.CommonName(), errs.ErrInvalidReference)
}
var opt blobOpt
for _, optFn := range opts {
optFn(&opt)
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
tDesc := d
tDesc.URLs = []string{} // ignore URLs when pushing to target
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackStarted, 0, d.Size)
}
// for the same repository, there's nothing to copy
if ref.EqualRepository(refSrc, refTgt) {
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
}
rc.slog.Debug("Blob copy skipped, same repo",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("digest", string(d.Digest)))
return nil
}
// check if layer already exists
if _, err := rc.BlobHead(ctx, refTgt, tDesc); err == nil {
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
}
rc.slog.Debug("Blob copy skipped, already exists",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("digest", string(d.Digest)))
return nil
}
// acquire throttle for both src and tgt to avoid deadlocks
tList := []*pqueue.Queue[reqmeta.Data]{}
schemeSrcAPI, err := rc.schemeGet(refSrc.Scheme)
if err != nil {
return err
}
schemeTgtAPI, err := rc.schemeGet(refTgt.Scheme)
if err != nil {
return err
}
if tSrc, ok := schemeSrcAPI.(scheme.Throttler); ok {
tList = append(tList, tSrc.Throttle(refSrc, false)...)
}
if tTgt, ok := schemeTgtAPI.(scheme.Throttler); ok {
tList = append(tList, tTgt.Throttle(refTgt, true)...)
}
if len(tList) > 0 {
ctxMulti, done, err := pqueue.AcquireMulti[reqmeta.Data](ctx, reqmeta.Data{Kind: reqmeta.Blob, Size: d.Size}, tList...)
if err != nil {
return err
}
if done != nil {
defer done()
}
ctx = ctxMulti
}
// try mounting blob from the source repo is the registry is the same
if ref.EqualRegistry(refSrc, refTgt) {
err := rc.BlobMount(ctx, refSrc, refTgt, d)
if err == nil {
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
}
rc.slog.Debug("Blob copy performed server side with registry mount",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("digest", string(d.Digest)))
return nil
}
rc.slog.Warn("Failed to mount blob",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("err", err.Error()))
}
// fast options failed, download layer from source and push to target
blobIO, err := rc.BlobGet(ctx, refSrc, d)
if err != nil {
if !errors.Is(err, context.Canceled) {
rc.slog.Warn("Failed to retrieve blob",
slog.String("src", refSrc.Reference),
slog.String("digest", string(d.Digest)),
slog.String("err", err.Error()))
}
return err
}
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackStarted, 0, d.Size)
ticker := time.NewTicker(blobCBFreq)
done := make(chan bool)
defer func() {
close(done)
ticker.Stop()
if ctx.Err() == nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackFinished, d.Size, d.Size)
}
}()
go func() {
for {
select {
case <-done:
return
case <-ticker.C:
offset, err := blobIO.Seek(0, io.SeekCurrent)
if err == nil && offset > 0 {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackActive, offset, d.Size)
}
}
}
}()
}
if opt.readerHook != nil {
blobIO, err = opt.readerHook(blobIO)
if err != nil {
rc.slog.Warn("Failed to apply reader hook to blob",
slog.String("src", refSrc.Reference),
slog.String("err", err.Error()))
return err
}
}
defer blobIO.Close()
if _, err := rc.BlobPut(ctx, refTgt, blobIO.GetDescriptor(), blobIO); err != nil {
if !errors.Is(err, context.Canceled) {
rc.slog.Warn("Failed to push blob",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("err", err.Error()))
}
return err
}
return nil
}
// BlobDelete removes a blob from the registry.
// This method should only be used to repair a damaged registry.
// Typically a server side garbage collection should be used to purge unused blobs.
func (rc *RegClient) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error {
if !r.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return err
}
return schemeAPI.BlobDelete(ctx, r, d)
}
// BlobGet retrieves a blob, returning a reader.
// This reader must be closed to free up resources that limit concurrent pulls.
func (rc *RegClient) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
data, err := d.GetData()
if err == nil {
return blob.NewReader(blob.WithDesc(d), blob.WithRef(r), blob.WithReader(bytes.NewReader(data))), nil
}
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
return schemeAPI.BlobGet(ctx, r, d)
}
// BlobGetOCIConfig retrieves an OCI config from a blob, automatically extracting the JSON.
func (rc *RegClient) BlobGetOCIConfig(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.OCIConfig, error) {
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
b, err := rc.BlobGet(ctx, r, d)
if err != nil {
return nil, err
}
return b.ToOCIConfig()
}
// BlobHead is used to verify if a blob exists and is accessible.
func (rc *RegClient) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
return schemeAPI.BlobHead(ctx, r, d)
}
// BlobMount attempts to perform a server side copy/mount of the blob between repositories.
func (rc *RegClient) BlobMount(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor) error {
if !refSrc.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", refSrc.CommonName(), errs.ErrInvalidReference)
}
if !refTgt.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", refTgt.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(refSrc.Scheme)
if err != nil {
return err
}
return schemeAPI.BlobMount(ctx, refSrc, refTgt, d)
}
// BlobPut uploads a blob to a repository.
// Descriptor is optional, leave size and digest to zero value if unknown.
// Reader must also be an [io.Seeker] to support chunked upload fallback.
//
// This will attempt an anonymous blob mount first which some registries may support.
// It will then try doing a full put of the blob without chunking (most widely supported).
// If the full put fails, it will fall back to a chunked upload (useful for flaky networks).
func (rc *RegClient) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error) {
if !r.IsSetRepo() {
return descriptor.Descriptor{}, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return descriptor.Descriptor{}, err
}
return schemeAPI.BlobPut(ctx, r, d, rdr)
}
+100
View File
@@ -0,0 +1,100 @@
package config
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"strings"
)
// credHelper wraps a command that manages user credentials.
type credHelper struct {
prog string
env map[string]string
}
func newCredHelper(prog string, env map[string]string) *credHelper {
return &credHelper{prog: prog, env: env}
}
func (ch *credHelper) run(arg string, input io.Reader) ([]byte, error) {
//#nosec G204 only untrusted arg is a hostname which the executed command should not trust
cmd := exec.Command(ch.prog, arg)
cmd.Env = os.Environ()
if ch.env != nil {
for k, v := range ch.env {
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
}
}
cmd.Stderr = os.Stderr
cmd.Stdin = input
return cmd.Output()
}
type credStore struct {
ServerURL string `json:"ServerURL"`
Username string `json:"Username"`
Secret string `json:"Secret"` //#nosec G117 exported struct intentionally holds secrets
}
// get requests a credential from the helper for a given host.
func (ch *credHelper) get(host *Host) error {
hostname := host.Hostname
if host.CredHost != "" {
hostname = host.CredHost
}
hostIn := strings.NewReader(hostname)
credOut := credStore{
Username: host.User,
Secret: host.Pass,
}
outB, err := ch.run("get", hostIn)
if err != nil {
outS := strings.TrimSpace(string(outB))
return fmt.Errorf("error getting credentials, output: %s, error: %w", outS, err)
}
err = json.NewDecoder(bytes.NewReader(outB)).Decode(&credOut)
if err != nil {
return fmt.Errorf("error reading credentials: %w", err)
}
if credOut.Username == tokenUser {
host.User = ""
host.Pass = ""
host.Token = credOut.Secret
} else {
host.User = credOut.Username
host.Pass = credOut.Secret
host.Token = ""
}
return nil
}
// list returns a list of hosts supported by the credential helper.
func (ch *credHelper) list() ([]Host, error) {
credList := map[string]string{}
outB, err := ch.run("list", bytes.NewReader([]byte{}))
if err != nil {
outS := strings.TrimSpace(string(outB))
return nil, fmt.Errorf("error getting credential list, output: %s, error: %w", outS, err)
}
err = json.NewDecoder(bytes.NewReader(outB)).Decode(&credList)
if err != nil {
return nil, fmt.Errorf("error reading credential list: %w", err)
}
hostList := []Host{}
for host, user := range credList {
if !HostValidate(host) {
continue
}
h := HostNewName(host)
h.User = user
h.CredHelper = ch.prog
hostList = append(hostList, *h)
}
return hostList, nil
}
// TODO: store method not implemented
+210
View File
@@ -0,0 +1,210 @@
package config
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"strings"
"github.com/regclient/regclient/internal/conffile"
"github.com/regclient/regclient/types/errs"
)
const (
// dockerEnv is the environment variable used to look for Docker's config.json.
dockerEnv = "DOCKER_CONFIG"
// dockerEnvConfig is used to inject the config as an environment variable.
dockerEnvConfig = "DOCKER_AUTH_CONFIG"
// dockerDir is the directory name for Docker's config (inside the users home directory).
dockerDir = ".docker"
// dockerConfFile is the name of Docker's config file.
dockerConfFile = "config.json"
// dockerHelperPre is the prefix of docker credential helpers.
dockerHelperPre = "docker-credential-"
)
// dockerConfig is used to parse the ~/.docker/config.json
type dockerConfig struct {
AuthConfigs map[string]dockerAuthConfig `json:"auths"`
HTTPHeaders map[string]string `json:"HttpHeaders,omitempty"`
DetachKeys string `json:"detachKeys,omitempty"`
CredentialsStore string `json:"credsStore,omitempty"`
CredentialHelpers map[string]string `json:"credHelpers,omitempty"`
Proxies map[string]dockerProxyConfig `json:"proxies,omitempty"`
}
// dockerProxyConfig contains proxy configuration settings
type dockerProxyConfig struct {
HTTPProxy string `json:"httpProxy,omitempty"`
HTTPSProxy string `json:"httpsProxy,omitempty"`
NoProxy string `json:"noProxy,omitempty"`
FTPProxy string `json:"ftpProxy,omitempty"`
AllProxy string `json:"allProxy,omitempty"`
}
// dockerAuthConfig contains the auths
type dockerAuthConfig struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"` //#nosec G117 exported struct intentionally holds secrets
Auth string `json:"auth,omitempty"`
ServerAddress string `json:"serveraddress,omitempty"`
// IdentityToken is used to authenticate the user and get
// an access token for the registry.
IdentityToken string `json:"identitytoken,omitempty"`
// RegistryToken is a bearer token to be sent to a registry
RegistryToken string `json:"registrytoken,omitempty"`
}
// DockerLoad returns a slice of hosts from the users docker config.
// This will search for the config.json in either the DOCKER_CONFIG identified directory or the default .docker directory.
// It also includes hosts extracted from the DOCKER_AUTH_CONFIG variable.
// If the config file is missing and no value is injected using an environment variable, an empty list is returned.
func DockerLoad() ([]Host, error) {
hosts := []Host{}
errList := []error{}
// load from a file
cf := conffile.New(
conffile.WithHomeDir(dockerDir, dockerConfFile, true),
conffile.WithEnvDir(dockerEnv, dockerConfFile),
)
rdr, err := cf.Open()
if err != nil && !errors.Is(err, fs.ErrNotExist) {
errList = append(errList, err)
} else if err == nil {
defer rdr.Close()
hostsFile, err := dockerParse(rdr)
if err != nil {
errList = append(errList, err)
} else {
hosts = append(hosts, hostsFile...)
}
}
// load from an env var
hostsEnv, err := DockerLoadEnv(dockerEnvConfig)
if err != nil && !errors.Is(err, errs.ErrNotFound) {
errList = append(errList, err)
} else if err == nil {
hosts = append(hosts, hostsEnv...)
}
// return the concatenated result, only wrapping an error list if necessary
if len(errList) == 1 {
return hosts, errList[0]
} else {
return hosts, errors.Join(errList...)
}
}
// DockerLoadFile returns a slice of hosts from a named docker config file.
func DockerLoadFile(fname string) ([]Host, error) {
//#nosec G304 scoping file operations to a directory is not yet a feature of regclient.
rdr, err := os.Open(fname)
if err != nil && errors.Is(err, fs.ErrNotExist) {
return []Host{}, nil
} else if err != nil {
return nil, err
}
defer rdr.Close()
return dockerParse(rdr)
}
// DockerLoadEnv returns a slice of hosts extracted from the config injected in an environment variable.
func DockerLoadEnv(envName string) ([]Host, error) {
envVal := os.Getenv(envName)
if envVal == "" {
return []Host{}, errs.ErrNotFound
}
return dockerParse(strings.NewReader(envVal))
}
// dockerParse parses a docker config into a slice of Hosts.
func dockerParse(rdr io.Reader) ([]Host, error) {
dc := dockerConfig{}
if err := json.NewDecoder(rdr).Decode(&dc); err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
hosts := []Host{}
for name, auth := range dc.AuthConfigs {
if !HostValidate(name) {
continue
}
h, err := dockerAuthToHost(name, dc, auth)
if err != nil {
continue
}
hosts = append(hosts, h)
}
// also include default entries for credential helpers
for name, helper := range dc.CredentialHelpers {
if !HostValidate(name) {
continue
}
h := HostNewName(name)
h.CredHelper = dockerHelperPre + helper
if _, ok := dc.AuthConfigs[name]; ok {
continue // skip fields with auth config
}
hosts = append(hosts, *h)
}
// add credStore entries
if dc.CredentialsStore != "" {
ch := newCredHelper(dockerHelperPre+dc.CredentialsStore, map[string]string{})
csHosts, err := ch.list()
if err == nil {
hosts = append(hosts, csHosts...)
}
}
return hosts, nil
}
// dockerAuthToHost parses an auth entry from a docker config into a Host.
func dockerAuthToHost(name string, conf dockerConfig, auth dockerAuthConfig) (Host, error) {
helper := ""
if conf.CredentialHelpers != nil && conf.CredentialHelpers[name] != "" {
helper = dockerHelperPre + conf.CredentialHelpers[name]
}
// parse base64 auth into user/pass
if auth.Auth != "" {
var err error
auth.Username, auth.Password, err = decodeAuth(auth.Auth)
if err != nil {
return Host{}, err
}
}
if (auth.Username == "" || auth.Password == "") && auth.IdentityToken == "" && helper == "" {
return Host{}, fmt.Errorf("no credentials found for %s", name)
}
h := HostNewName(name)
// ignore unknown names
if h.Name != DockerRegistry && !strings.HasSuffix(strings.TrimSuffix(name, "/"), h.Name) {
return Host{}, fmt.Errorf("rejecting entry with repository: %s", name)
}
h.User = auth.Username
h.Pass = auth.Password
h.Token = auth.IdentityToken
h.CredHelper = helper
return *h, nil
}
// decodeAuth extracts a base64 encoded user:pass into the username and password.
func decodeAuth(authStr string) (string, string, error) {
if authStr == "" {
return "", "", nil
}
decoded, err := base64.StdEncoding.DecodeString(authStr)
if err != nil {
return "", "", err
}
userPass := strings.SplitN(string(decoded), ":", 2)
if len(userPass) != 2 {
return "", "", fmt.Errorf("invalid auth configuration file")
}
return userPass[0], strings.Trim(userPass[1], "\x00"), nil
}
+521
View File
@@ -0,0 +1,521 @@
// Package config is used for all regclient configuration settings.
package config
import (
"encoding/json"
"fmt"
"io"
"log/slog"
"maps"
"slices"
"strings"
"time"
"github.com/regclient/regclient/internal/timejson"
)
// TLSConf specifies whether TLS is enabled and verified for a host.
type TLSConf int
const (
// TLSUndefined indicates TLS is not passed, defaults to Enabled.
TLSUndefined TLSConf = iota
// TLSEnabled uses TLS (https) for the connection.
TLSEnabled
// TLSInsecure uses TLS but does not verify CA.
TLSInsecure
// TLSDisabled does not use TLS (http).
TLSDisabled
)
const (
// DockerRegistry is the name resolved in docker images on Hub.
DockerRegistry = "docker.io"
// DockerRegistryAuth is the name provided in docker's config for Hub.
DockerRegistryAuth = "https://index.docker.io/v1/"
// DockerRegistryDNS is the host to connect to for Hub.
DockerRegistryDNS = "registry-1.docker.io"
// defaultExpire is the default time to expire a credential and force re-authentication.
defaultExpire = time.Hour * 1
// defaultCredHelperRetry is the time to refresh a credential from a failed credential helper command.
defaultCredHelperRetry = time.Second * 5
// defaultConcurrent is the default number of concurrent registry connections.
defaultConcurrent = 3
// defaultReqPerSec is the default maximum frequency to send requests to a registry.
defaultReqPerSec = 0
// tokenUser is the username returned by credential helpers that indicates the password is an identity token.
tokenUser = "<token>"
)
// MarshalJSON converts TLSConf to a json string using MarshalText.
func (t TLSConf) MarshalJSON() ([]byte, error) {
s, err := t.MarshalText()
if err != nil {
return []byte(""), err
}
return json.Marshal(string(s))
}
// MarshalText converts TLSConf to a string.
func (t TLSConf) MarshalText() ([]byte, error) {
var s string
switch t {
default:
s = ""
case TLSEnabled:
s = "enabled"
case TLSInsecure:
s = "insecure"
case TLSDisabled:
s = "disabled"
}
return []byte(s), nil
}
// UnmarshalJSON converts TLSConf from a json string.
func (t *TLSConf) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err != nil {
return err
}
return t.UnmarshalText([]byte(s))
}
// UnmarshalText converts TLSConf from a string.
func (t *TLSConf) UnmarshalText(b []byte) error {
switch strings.ToLower(string(b)) {
default:
return fmt.Errorf("unknown TLS value \"%s\"", b)
case "":
*t = TLSUndefined
case "enabled":
*t = TLSEnabled
case "insecure":
*t = TLSInsecure
case "disabled":
*t = TLSDisabled
}
return nil
}
// Host defines settings for connecting to a registry.
type Host struct {
Name string `json:"-" yaml:"registry,omitempty"` // Name of the registry (required) (yaml configs pass this as a field, json provides this from the object key)
TLS TLSConf `json:"tls,omitempty" yaml:"tls"` // TLS setting: enabled (default), disabled, insecure
RegCert string `json:"regcert,omitempty" yaml:"regcert"` // public pem cert of registry
ClientCert string `json:"clientCert,omitempty" yaml:"clientCert"` // public pem cert for client (mTLS)
ClientKey string `json:"clientKey,omitempty" yaml:"clientKey"` //#nosec G117 private pem cert for client (mTLS)
Hostname string `json:"hostname,omitempty" yaml:"hostname"` // hostname of registry, default is the registry name
User string `json:"user,omitempty" yaml:"user"` // username, not used with credHelper
Pass string `json:"pass,omitempty" yaml:"pass"` //#nosec G117 password, not used with credHelper
Token string `json:"token,omitempty" yaml:"token"` // token, experimental for specific APIs
CredHelper string `json:"credHelper,omitempty" yaml:"credHelper"` // credential helper command for requesting logins
CredExpire timejson.Duration `json:"credExpire,omitempty" yaml:"credExpire"` // time until credential expires
CredHost string `json:"credHost,omitempty" yaml:"credHost"` // used when a helper hostname doesn't match Hostname
PathPrefix string `json:"pathPrefix,omitempty" yaml:"pathPrefix"` // used for mirrors defined within a repository namespace
Mirrors []string `json:"mirrors,omitempty" yaml:"mirrors"` // list of other Host Names to use as mirrors
Priority uint `json:"priority,omitempty" yaml:"priority"` // priority when sorting mirrors, higher priority attempted first
RepoAuth bool `json:"repoAuth,omitempty" yaml:"repoAuth"` // tracks a separate auth per repo
API string `json:"api,omitempty" yaml:"api"` // Deprecated: registry API to use
APIOpts map[string]string `json:"apiOpts,omitempty" yaml:"apiOpts"` // options for APIs
BlobChunk int64 `json:"blobChunk,omitempty" yaml:"blobChunk"` // size of each blob chunk
BlobMax int64 `json:"blobMax,omitempty" yaml:"blobMax"` // threshold to switch to chunked upload, -1 to disable, 0 for regclient.blobMaxPut
ReqPerSec float64 `json:"reqPerSec,omitempty" yaml:"reqPerSec"` // requests per second
ReqConcurrent int64 `json:"reqConcurrent,omitempty" yaml:"reqConcurrent"` // concurrent requests, default is defaultConcurrent(3)
Scheme string `json:"scheme,omitempty" yaml:"scheme"` // Deprecated: use TLS instead
credRefresh time.Time `json:"-" yaml:"-"` // internal use, when to refresh credentials
}
// Cred defines a user credential for accessing a registry.
type Cred struct {
User, Password, Token string //#nosec G117 exported struct intentionally holds secrets
}
// HostNew creates a default Host entry.
func HostNew() *Host {
h := Host{
TLS: TLSEnabled,
APIOpts: map[string]string{},
ReqConcurrent: int64(defaultConcurrent),
ReqPerSec: float64(defaultReqPerSec),
}
return &h
}
// HostNewName creates a default Host with a hostname.
func HostNewName(name string) *Host {
return HostNewDefName(nil, name)
}
// HostNewDefName creates a host using provided defaults and hostname.
func HostNewDefName(def *Host, name string) *Host {
var h Host
if def == nil {
h = *HostNew()
} else {
h = *def
// configure required defaults
if h.TLS == TLSUndefined {
h.TLS = TLSEnabled
}
if h.APIOpts == nil {
h.APIOpts = map[string]string{}
}
if h.ReqConcurrent == 0 {
h.ReqConcurrent = int64(defaultConcurrent)
}
if h.ReqPerSec == 0 {
h.ReqPerSec = float64(defaultReqPerSec)
}
// copy any fields that are not passed by value
if len(h.APIOpts) > 0 {
orig := h.APIOpts
h.APIOpts = map[string]string{}
maps.Copy(h.APIOpts, orig)
}
if h.Mirrors != nil {
orig := h.Mirrors
h.Mirrors = make([]string, len(orig))
copy(h.Mirrors, orig)
}
}
// configure host
scheme, registry, _ := parseName(name)
if scheme == "http" {
h.TLS = TLSDisabled
}
// Docker Hub is a special case
if registry == DockerRegistry {
h.Name = DockerRegistry
h.Hostname = DockerRegistryDNS
h.CredHost = DockerRegistryAuth
return &h
}
h.Name = registry
h.Hostname = registry
if name != registry {
h.CredHost = name
}
return &h
}
// HostValidate returns true if the scheme is missing or a known value, and the path is not set.
func HostValidate(name string) bool {
scheme, _, path := parseName(name)
return path == "" && (scheme == "https" || scheme == "http")
}
// GetCred returns the credential, fetching from a credential helper if needed.
func (host *Host) GetCred() Cred {
// refresh from credHelper if needed
if host.CredHelper != "" && (host.credRefresh.IsZero() || time.Now().After(host.credRefresh)) {
host.refreshHelper()
}
return Cred{User: host.User, Password: host.Pass, Token: host.Token}
}
func (host *Host) refreshHelper() {
if host.CredHelper == "" {
return
}
if host.CredExpire <= 0 {
host.CredExpire = timejson.Duration(defaultExpire)
}
// run a cred helper, calling get method
ch := newCredHelper(host.CredHelper, map[string]string{})
err := ch.get(host)
if err != nil {
host.credRefresh = time.Now().Add(defaultCredHelperRetry)
} else {
host.credRefresh = time.Now().Add(time.Duration(host.CredExpire))
}
}
// IsZero returns true if the struct is set to the zero value or the result of [HostNew].
func (host Host) IsZero() bool {
if (host.TLS != TLSUndefined && host.TLS != TLSEnabled) ||
host.RegCert != "" ||
host.ClientCert != "" ||
host.ClientKey != "" ||
(host.Hostname != "" && host.Hostname != host.Name) ||
host.User != "" ||
host.Pass != "" ||
host.Token != "" ||
host.CredHelper != "" ||
host.CredExpire != 0 ||
host.CredHost != "" ||
host.PathPrefix != "" ||
len(host.Mirrors) != 0 ||
host.Priority != 0 ||
host.RepoAuth ||
len(host.APIOpts) != 0 ||
host.BlobChunk != 0 ||
host.BlobMax != 0 ||
(host.ReqPerSec != 0 && host.ReqPerSec != float64(defaultReqPerSec)) ||
(host.ReqConcurrent != 0 && host.ReqConcurrent != int64(defaultConcurrent)) ||
!host.credRefresh.IsZero() {
return false
}
return true
}
// Merge adds fields from a new config host entry.
func (host *Host) Merge(newHost Host, log *slog.Logger) error {
name := newHost.Name
if name == "" {
name = host.Name
}
if log == nil {
log = slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{}))
}
// merge the existing and new config host
if host.Name == "" {
// only set the name if it's not initialized, this shouldn't normally change
host.Name = newHost.Name
}
if newHost.CredHelper == "" && (newHost.Pass != "" || host.Token != "") {
// unset existing cred helper for user/pass or token
host.CredHelper = ""
host.CredExpire = 0
}
if newHost.CredHelper != "" && newHost.User == "" && newHost.Pass == "" && newHost.Token == "" {
// unset existing user/pass/token for cred helper
host.User = ""
host.Pass = ""
host.Token = ""
}
if newHost.User != "" {
if host.User != "" && host.User != newHost.User {
log.Warn("Changing login user for registry",
slog.String("orig", host.User),
slog.String("new", newHost.User),
slog.String("host", name))
}
host.User = newHost.User
}
if newHost.Pass != "" {
if host.Pass != "" && host.Pass != newHost.Pass {
log.Warn("Changing login password for registry",
slog.String("host", name))
}
host.Pass = newHost.Pass
}
if newHost.Token != "" {
if host.Token != "" && host.Token != newHost.Token {
log.Warn("Changing login token for registry",
slog.String("host", name))
}
host.Token = newHost.Token
}
if newHost.CredHelper != "" {
if host.CredHelper != "" && host.CredHelper != newHost.CredHelper {
log.Warn("Changing credential helper for registry",
slog.String("host", name),
slog.String("orig", host.CredHelper),
slog.String("new", newHost.CredHelper))
}
host.CredHelper = newHost.CredHelper
}
if newHost.CredExpire != 0 {
if host.CredExpire != 0 && host.CredExpire != newHost.CredExpire {
log.Warn("Changing credential expire for registry",
slog.String("host", name),
slog.Any("orig", host.CredExpire),
slog.Any("new", newHost.CredExpire))
}
host.CredExpire = newHost.CredExpire
}
if newHost.CredHost != "" {
if host.CredHost != "" && host.CredHost != newHost.CredHost {
log.Warn("Changing credential host for registry",
slog.String("host", name),
slog.String("orig", host.CredHost),
slog.String("new", newHost.CredHost))
}
host.CredHost = newHost.CredHost
}
if newHost.TLS != TLSUndefined {
if host.TLS != TLSUndefined && host.TLS != newHost.TLS {
tlsOrig, _ := host.TLS.MarshalText()
tlsNew, _ := newHost.TLS.MarshalText()
log.Warn("Changing TLS settings for registry",
slog.String("orig", string(tlsOrig)),
slog.String("new", string(tlsNew)),
slog.String("host", name))
}
host.TLS = newHost.TLS
}
if newHost.RegCert != "" {
if host.RegCert != "" && host.RegCert != newHost.RegCert {
log.Warn("Changing certificate settings for registry",
slog.String("orig", host.RegCert),
slog.String("new", newHost.RegCert),
slog.String("host", name))
}
host.RegCert = newHost.RegCert
}
if newHost.ClientCert != "" {
if host.ClientCert != "" && host.ClientCert != newHost.ClientCert {
log.Warn("Changing client certificate settings for registry",
slog.String("orig", host.ClientCert),
slog.String("new", newHost.ClientCert),
slog.String("host", name))
}
host.ClientCert = newHost.ClientCert
}
if newHost.ClientKey != "" {
if host.ClientKey != "" && host.ClientKey != newHost.ClientKey {
log.Warn("Changing client certificate key settings for registry",
slog.String("host", name))
}
host.ClientKey = newHost.ClientKey
}
if newHost.Hostname != "" {
if host.Hostname != "" && host.Hostname != newHost.Hostname {
log.Warn("Changing hostname settings for registry",
slog.String("orig", host.Hostname),
slog.String("new", newHost.Hostname),
slog.String("host", name))
}
host.Hostname = newHost.Hostname
}
if newHost.PathPrefix != "" {
newHost.PathPrefix = strings.Trim(newHost.PathPrefix, "/") // leading and trailing / are not needed
if host.PathPrefix != "" && host.PathPrefix != newHost.PathPrefix {
log.Warn("Changing path prefix settings for registry",
slog.String("orig", host.PathPrefix),
slog.String("new", newHost.PathPrefix),
slog.String("host", name))
}
host.PathPrefix = newHost.PathPrefix
}
if len(newHost.Mirrors) > 0 {
if len(host.Mirrors) > 0 && !slices.Equal(host.Mirrors, newHost.Mirrors) {
log.Warn("Changing mirror settings for registry",
slog.Any("orig", host.Mirrors),
slog.Any("new", newHost.Mirrors),
slog.String("host", name))
}
host.Mirrors = newHost.Mirrors
}
if newHost.Priority != 0 {
if host.Priority != 0 && host.Priority != newHost.Priority {
log.Warn("Changing priority settings for registry",
slog.Uint64("orig", uint64(host.Priority)),
slog.Uint64("new", uint64(newHost.Priority)),
slog.String("host", name))
}
host.Priority = newHost.Priority
}
if newHost.RepoAuth {
host.RepoAuth = newHost.RepoAuth
}
// TODO: eventually delete
if newHost.API != "" {
log.Warn("API field has been deprecated",
slog.String("api", newHost.API),
slog.String("host", name))
}
if len(newHost.APIOpts) > 0 {
if len(host.APIOpts) > 0 {
merged := maps.Clone(host.APIOpts)
for k, v := range newHost.APIOpts {
if host.APIOpts[k] != "" && host.APIOpts[k] != v {
log.Warn("Changing APIOpts setting for registry",
slog.String("orig", host.APIOpts[k]),
slog.String("new", newHost.APIOpts[k]),
slog.String("opt", k),
slog.String("host", name))
}
merged[k] = v
}
host.APIOpts = merged
} else {
host.APIOpts = newHost.APIOpts
}
}
if newHost.BlobChunk > 0 {
if host.BlobChunk != 0 && host.BlobChunk != newHost.BlobChunk {
log.Warn("Changing blobChunk settings for registry",
slog.Int64("orig", host.BlobChunk),
slog.Int64("new", newHost.BlobChunk),
slog.String("host", name))
}
host.BlobChunk = newHost.BlobChunk
}
if newHost.BlobMax != 0 {
if host.BlobMax != 0 && host.BlobMax != newHost.BlobMax {
log.Warn("Changing blobMax settings for registry",
slog.Int64("orig", host.BlobMax),
slog.Int64("new", newHost.BlobMax),
slog.String("host", name))
}
host.BlobMax = newHost.BlobMax
}
if newHost.ReqPerSec != 0 {
if host.ReqPerSec != 0 && host.ReqPerSec != newHost.ReqPerSec {
log.Warn("Changing reqPerSec settings for registry",
slog.Float64("orig", host.ReqPerSec),
slog.Float64("new", newHost.ReqPerSec),
slog.String("host", name))
}
host.ReqPerSec = newHost.ReqPerSec
}
if newHost.ReqConcurrent > 0 {
if host.ReqConcurrent != 0 && host.ReqConcurrent != newHost.ReqConcurrent {
log.Warn("Changing reqPerSec settings for registry",
slog.Int64("orig", host.ReqConcurrent),
slog.Int64("new", newHost.ReqConcurrent),
slog.String("host", name))
}
host.ReqConcurrent = newHost.ReqConcurrent
}
return nil
}
// parseName splits a registry into the scheme, hostname, and repository/path.
func parseName(name string) (string, string, string) {
scheme := "https"
path := ""
// Docker Hub is a special case
if name == DockerRegistryAuth || name == DockerRegistryDNS || name == DockerRegistry {
return scheme, DockerRegistry, ""
}
// handle http/https prefix
i := strings.Index(name, "://")
if i > 0 {
scheme = name[:i]
name = name[i+3:]
}
// trim any repository path
i = strings.Index(name, "/")
if i > 0 {
path = name[i+1:]
name = name[:i]
}
return scheme, name, path
}
+1905
View File
File diff suppressed because it is too large Load Diff
+923
View File
@@ -0,0 +1,923 @@
// Package auth is used for HTTP authentication
package auth
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"slices"
"strings"
"sync"
"time"
"github.com/regclient/regclient/internal/regnet"
"github.com/regclient/regclient/types/errs"
)
type charLU byte
var charLUs [256]charLU
var defaultClientID = "regclient"
// minTokenLife tokens are required to last at least 60 seconds to support older docker clients
var minTokenLife = 60
// tokenBuffer is used to renew a token before it expires to account for time to process requests on the server
var tokenBuffer = time.Second * 5
const (
isSpace charLU = 1 << iota
isToken
)
func init() {
for c := range 256 {
charLUs[c] = 0
if strings.ContainsRune(" \t\r\n", rune(c)) {
charLUs[c] |= isSpace
}
if (rune('a') <= rune(c) && rune(c) <= rune('z')) || (rune('A') <= rune(c) && rune(c) <= rune('Z') || (rune('0') <= rune(c) && rune(c) <= rune('9')) || strings.ContainsRune("-._~+/", rune(c))) {
charLUs[c] |= isToken
}
}
}
// CredsFn is passed to lookup credentials for a given hostname, response is a username and password or empty strings
type CredsFn func(host string) Cred
// Cred is returned by the CredsFn.
// If Token is provided and auth method is bearer, it will attempt to use it as a refresh token.
// Else if user and password are provided, they are attempted with all auth methods.
// Else if neither are provided and auth method is bearer, an anonymous login is attempted.
type Cred struct {
//#nosec G117 exported struct intentionally holds secrets
User, Password string // clear text username and password
Token string // refresh token only used for bearer auth
}
// challenge is the extracted contents of the WWW-Authenticate header.
type challenge struct {
authType string
params map[string]string
}
// handler handles a challenge for a host to return an auth header
type handler interface {
AddScope(scope string) error
ProcessChallenge(challenge) error
UpdateRequest(*http.Request) error
}
// handlerBuild is used to make a new handler for a specific authType and URL
type handlerBuild func(client *http.Client, clientID, host string, credFn CredsFn, slog *slog.Logger) handler
// Opts configures options for NewAuth
type Opts func(*Auth)
// Auth is used to handle authentication requests.
type Auth struct {
httpClient *http.Client
clientID string
credsFn CredsFn
hbs map[string]handlerBuild // handler builders based on authType
hs map[string]map[string]handler // handlers based on url and authType
authTypes []string
slog *slog.Logger
mu sync.Mutex
}
// NewAuth creates a new Auth
func NewAuth(opts ...Opts) *Auth {
a := &Auth{
httpClient: &http.Client{},
clientID: defaultClientID,
credsFn: DefaultCredsFn,
hbs: map[string]handlerBuild{},
hs: map[string]map[string]handler{},
authTypes: []string{},
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
}
for _, opt := range opts {
opt(a)
}
if len(a.authTypes) == 0 {
a.addDefaultHandlers()
}
return a
}
// WithCreds provides a user/pass lookup for a url
func WithCreds(f CredsFn) Opts {
return func(a *Auth) {
if f != nil {
a.credsFn = f
}
}
}
// WithHTTPClient uses a specific http client with requests
func WithHTTPClient(h *http.Client) Opts {
return func(a *Auth) {
if h != nil {
a.httpClient = h
}
}
}
// WithClientID uses a client ID with request headers
func WithClientID(clientID string) Opts {
return func(a *Auth) {
a.clientID = clientID
}
}
// WithHandler includes a handler for a specific auth type
func WithHandler(authType string, hb handlerBuild) Opts {
return func(a *Auth) {
lcat := strings.ToLower(authType)
a.hbs[lcat] = hb
a.authTypes = append(a.authTypes, lcat)
}
}
// WithDefaultHandlers includes a Basic and Bearer handler, this is automatically added with "WithHandler" is not called
func WithDefaultHandlers() Opts {
return func(a *Auth) {
a.addDefaultHandlers()
}
}
// WithLog injects a Logger
func WithLog(slog *slog.Logger) Opts {
return func(a *Auth) {
a.slog = slog
}
}
// AddScope extends an existing auth with additional scopes.
// This is used to pre-populate scopes with the Docker convention rather than
// depend on the registry to respond with the correct http status and headers.
func (a *Auth) AddScope(host, scope string) error {
a.mu.Lock()
defer a.mu.Unlock()
success := false
if a.hs[host] == nil {
return errs.ErrNoNewChallenge
}
for _, at := range a.authTypes {
if a.hs[host][at] != nil {
err := a.hs[host][at].AddScope(scope)
if err == nil {
success = true
} else if err != errs.ErrNoNewChallenge {
return err
}
}
}
if !success {
return errs.ErrNoNewChallenge
}
a.slog.Debug("Auth scope added",
slog.String("host", host),
slog.String("scope", scope))
return nil
}
// HandleResponse parses the 401 response, extracting the WWW-Authenticate
// header and verifying the requirement is different from what was included in
// the last request
func (a *Auth) HandleResponse(resp *http.Response) error {
a.mu.Lock()
defer a.mu.Unlock()
// verify response is an access denied
if resp.StatusCode != http.StatusUnauthorized {
return errs.ErrUnsupported
}
// extract host and auth header
host := resp.Request.URL.Host
cl, err := ParseAuthHeaders(resp.Header.Values("WWW-Authenticate"))
if err != nil {
return err
}
a.slog.Debug("Auth request parsed",
slog.Any("challenge", cl))
if len(cl) < 1 {
return errs.ErrEmptyChallenge
}
goodChallenge := false
// loop over the received challenge(s)
for _, c := range cl {
if _, ok := a.hbs[c.authType]; !ok {
a.slog.Warn("Unsupported auth type",
slog.String("authtype", c.authType))
continue
}
// setup a handler for the host and auth type
if _, ok := a.hs[host]; !ok {
a.hs[host] = map[string]handler{}
}
if _, ok := a.hs[host][c.authType]; !ok {
h := a.hbs[c.authType](a.httpClient, a.clientID, host, a.credsFn, a.slog)
if h == nil {
continue
}
a.hs[host][c.authType] = h
}
// process the challenge with that handler
err := a.hs[host][c.authType].ProcessChallenge(c)
if err == nil {
goodChallenge = true
} else if err == errs.ErrNoNewChallenge {
// handle race condition when another request updates the challenge
// detect that by seeing the current auth header is different
prevAH := resp.Request.Header.Get("Authorization")
err := a.hs[host][c.authType].UpdateRequest(resp.Request)
if err == nil && prevAH != resp.Request.Header.Get("Authorization") {
goodChallenge = true
}
} else {
return err
}
}
if !goodChallenge {
return errs.ErrHTTPUnauthorized
}
return nil
}
// UpdateRequest adds Authorization headers to a request
func (a *Auth) UpdateRequest(req *http.Request) error {
a.mu.Lock()
defer a.mu.Unlock()
host := req.URL.Host
if a.hs[host] == nil {
return nil
}
var err error
for _, at := range a.authTypes {
if a.hs[host][at] != nil {
err = a.hs[host][at].UpdateRequest(req)
if err != nil {
a.slog.Debug("Failed to generate auth",
slog.String("err", err.Error()),
slog.String("host", host),
slog.String("authtype", at))
continue
}
break
}
}
if err != nil {
return err
}
return nil
}
func (a *Auth) addDefaultHandlers() {
if _, ok := a.hbs["basic"]; !ok {
a.hbs["basic"] = NewBasicHandler
a.authTypes = append(a.authTypes, "basic")
}
if _, ok := a.hbs["bearer"]; !ok {
a.hbs["bearer"] = NewBearerHandler
a.authTypes = append(a.authTypes, "bearer")
}
}
// DefaultCredsFn is used to return no credentials when auth is not configured with a CredsFn
// This avoids the need to check for nil pointers
func DefaultCredsFn(h string) Cred {
return Cred{}
}
// ParseAuthHeaders extracts the scheme and realm from WWW-Authenticate headers
func ParseAuthHeaders(ahl []string) ([]challenge, error) {
var cl []challenge
for _, ah := range ahl {
c, err := parseAuthHeader(ah)
if err != nil {
return nil, fmt.Errorf("failed to parse challenge header: %s, %w", ah, err)
}
cl = append(cl, c...)
}
return cl, nil
}
// parseAuthHeader parses a single header line for WWW-Authenticate
// Example values:
// Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push"
// Basic realm="GitHub Package Registry"
func parseAuthHeader(ah string) ([]challenge, error) {
var cl []challenge
var c *challenge
curElement := []byte{}
curKey := ""
stateElement := "string"
stateSyntax := "start"
for _, b := range []byte(ah) {
switch stateElement {
case "string":
// string: ignore leading space, enter quote only if first character, handle escapes, handle valid tokens, else end
if charLUs[b]&isToken != 0 {
// add valid tokens to element
curElement = append(curElement, b)
} else if charLUs[b]&isSpace != 0 && len(curElement) == 0 {
// ignore leading spaces
} else if b == '"' && len(curElement) == 0 {
stateElement = "quote"
} else if b == '\\' {
stateElement = "escape_string"
} else {
stateElement = "end"
}
case "quote":
// quote: handle escapes, handle closing quote (quote_end to read next character), all other tokens are valid
if b == '\\' {
stateElement = "escape_quote"
} else if b == '"' {
stateElement = "quote_end"
} else {
curElement = append(curElement, b)
}
case "quote_end":
// any character after the close quote is the end of the element
stateElement = "end"
case "escape_string":
// escape_string: handle any character and return to string state
curElement = append(curElement, b)
stateElement = "string"
case "escape_quote":
// escape_quote: handle any character and return to quote state
curElement = append(curElement, b)
stateElement = "quote"
case "end":
// finished parsing element, continue to processing element according to the current state
default:
return nil, fmt.Errorf("unhandled element case: %w", errs.ErrParsingFailed)
}
if stateElement != "end" {
// continue parsing the element until it ends
continue
}
// syntax looks at each string within the overall challenge syntax
switch stateSyntax {
case "start":
// start: (start of auth_type) read auth_type and space (end_auth_type) or auth_type and comma (start)
if charLUs[b]&isSpace != 0 && len(curElement) > 0 {
stateSyntax = "end_auth_type"
} else if b == ',' && len(curElement) > 0 {
// state remains at start
} else {
return nil, fmt.Errorf("start element did not end with a space or comma: %w", errs.ErrParsingFailed)
}
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
cl = append(cl, *c)
case "start_or_param":
// start_or_param: (after param_value) read auth_type and space (end_auth_type) or param_key and equals (param_value)
if charLUs[b]&isSpace != 0 && len(curElement) > 0 {
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
cl = append(cl, *c)
stateSyntax = "end_auth_type"
} else if b == '=' && len(curElement) > 0 {
curKey = strings.ToLower(string(curElement))
stateSyntax = "param_value"
} else {
return nil, fmt.Errorf("expected auth type or param: %w", errs.ErrParsingFailed)
}
case "end_auth_type":
// end_auth_type: (after reading auth_type) read param_key and equals (param_value) or just a comma (start)
if b == '=' && len(curElement) > 0 {
curKey = strings.ToLower(string(curElement))
stateSyntax = "param_value"
} else if b == ',' && len(curElement) == 0 {
// ignore white space between end of auth_type and comma
stateSyntax = "start"
} else {
return nil, fmt.Errorf("expected param or comma: %w", errs.ErrParsingFailed)
}
case "param_value":
// param_value: (after param_key) read param_value and comma (start_or_param)
if b == ',' {
c.params[curKey] = string(curElement)
stateSyntax = "start_or_param"
curKey = ""
} else {
return nil, fmt.Errorf("expected param value: %w", errs.ErrParsingFailed)
}
default:
return nil, fmt.Errorf("unhandled syntax case: %w", errs.ErrParsingFailed)
}
// reset element state
stateElement = "string"
curElement = []byte{}
}
// at end of parsing, if the element is not empty, process according to syntax state:
if len(curElement) > 0 {
// ensure this is not within an unclosed quote or partial escape
if stateElement != "string" && stateElement != "quote_end" {
return nil, fmt.Errorf("eol element in state %s: %w", stateElement, errs.ErrParsingFailed)
}
switch stateSyntax {
case "start", "start_or_param":
// add a new auth type if a string is seen at the start, before any equals
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
cl = append(cl, *c)
case "param_value":
// add the last param key=val
c.params[curKey] = string(curElement)
case "end_auth_type":
// missing equals for param
return nil, fmt.Errorf("eol at param without value: %w", errs.ErrParsingFailed)
}
}
return cl, nil
}
// basicHandler supports Basic auth type requests
type basicHandler struct {
realm string
host string
credsFn CredsFn
}
// NewBasicHandler creates a new BasicHandler
func NewBasicHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
return &basicHandler{
realm: "",
host: host,
credsFn: credsFn,
}
}
// AddScope is not valid for BasicHandler
func (b *basicHandler) AddScope(scope string) error {
return errs.ErrNoNewChallenge
}
// ProcessChallenge for BasicHandler is a noop
func (b *basicHandler) ProcessChallenge(c challenge) error {
if _, ok := c.params["realm"]; !ok {
return errs.ErrInvalidChallenge
}
if b.realm != c.params["realm"] {
b.realm = c.params["realm"]
return nil
}
return errs.ErrNoNewChallenge
}
// UpdateRequest for BasicHandler generates base64 encoded user/pass for a host
func (b *basicHandler) UpdateRequest(req *http.Request) error {
cred := b.credsFn(b.host)
if cred.User == "" || cred.Password == "" {
return fmt.Errorf("no credentials available: %w", errs.ErrHTTPUnauthorized)
}
req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
base64.StdEncoding.EncodeToString([]byte(cred.User+":"+cred.Password))))
return nil
}
// bearerHandler supports Bearer auth type requests
type bearerHandler struct {
client *http.Client
clientID string
realm, service string
host string
credsFn CredsFn
scopes []string
tokenURL *url.URL
token bearerToken
slog *slog.Logger
}
// bearerToken is the json response to the Bearer request
type bearerToken struct {
Token string `json:"token"`
AccessToken string `json:"access_token"` //#nosec G117 exported struct intentionally holds secrets
ExpiresIn int `json:"expires_in"`
IssuedAt time.Time `json:"issued_at"`
RefreshToken string `json:"refresh_token"` //#nosec G117 exported struct intentionally holds secrets
Scope string `json:"scope"`
}
// NewBearerHandler creates a new BearerHandler
func NewBearerHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
return &bearerHandler{
client: client,
clientID: clientID,
host: host,
credsFn: credsFn,
realm: "",
service: "",
scopes: []string{},
slog: slog,
}
}
// AddScope appends a new scope if it doesn't already exist
func (b *bearerHandler) AddScope(scope string) error {
if b.scopeExists(scope) {
if b.token.Token == "" || !b.isExpired() {
return errs.ErrNoNewChallenge
}
return nil
}
b.addScope(scope)
return nil
}
func (b *bearerHandler) addScope(scope string) {
if !b.tryExtendExistingScope(scope) {
b.scopes = append(b.scopes, scope)
}
// delete old token
b.token.Token = ""
}
var knownActions = []string{"pull", "push", "delete"}
// tryExtendExistingScope extends an existing scope if both the new scope and the current scope contain only knownActions.
// It returns true if actions are added or are already present. Otherwise, it returns false,
// indicating that the new scope should be appended to b.scopes instead.
func (b *bearerHandler) tryExtendExistingScope(scope string) bool {
repo, actions, ok := parseScope(scope)
if !ok {
return false
}
scopePrefix := "repository:" + repo + ":"
for i, cur := range b.scopes {
if !strings.HasPrefix(cur, scopePrefix) {
continue
}
_, curActions, curOk := parseScope(cur)
if !curOk {
continue
}
for _, a := range actions {
if !slices.Contains(curActions, a) {
curActions = append(curActions, a)
}
}
b.scopes[i] = scopePrefix + strings.Join(curActions, ",")
return true
}
return false
}
// parseScope splits a scope into the repo and slice of actions.
// Unknown actions in the scope will set bool to false.
func parseScope(scope string) (string, []string, bool) {
scopeSplit := strings.SplitN(scope, ":", 3)
if scopeSplit[0] != "repository" || len(scopeSplit) < 3 {
return "", nil, false
}
actionSplit := strings.Split(scopeSplit[2], ",")
for _, a := range actionSplit {
if !slices.Contains(knownActions, a) {
return "", nil, false
}
}
return scopeSplit[1], actionSplit, true
}
// ProcessChallenge handles WWW-Authenticate header for bearer tokens
// Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push"
func (b *bearerHandler) ProcessChallenge(c challenge) error {
if _, ok := c.params["realm"]; !ok {
return errs.ErrInvalidChallenge
}
if _, ok := c.params["service"]; !ok {
c.params["service"] = ""
}
if _, ok := c.params["scope"]; !ok {
c.params["scope"] = ""
}
existingScope := b.scopeExists(c.params["scope"])
if b.realm == c.params["realm"] && b.service == c.params["service"] && existingScope && (b.token.Token == "" || !b.isExpired()) {
return errs.ErrNoNewChallenge
}
if b.realm == "" {
b.realm = c.params["realm"]
} else if b.realm != c.params["realm"] {
return errs.ErrInvalidChallenge
}
if b.service == "" {
b.service = c.params["service"]
} else if b.service != c.params["service"] {
return errs.ErrInvalidChallenge
}
if !existingScope {
b.addScope(c.params["scope"])
}
return nil
}
// UpdateRequest for BearerHandler adds a bearer token to the request.
func (b *bearerHandler) UpdateRequest(req *http.Request) error {
// handle relative realm values
if b.tokenURL == nil {
u, err := req.URL.Parse(b.realm)
if err != nil {
return err
}
b.tokenURL = u
}
// verify tokenURL is allowed for request URL
if err := regnet.AllowRedirect(*req.URL, *b.tokenURL); err != nil {
return err
}
// if unexpired token already exists, return it
if b.token.Token != "" && !b.isExpired() {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
return nil
}
// attempt to post if a refresh token is available or token auth is being used
cred := b.credsFn(b.host)
if b.token.RefreshToken != "" || cred.Token != "" {
if err := b.tryPost(cred); err == nil {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
return nil
} else if err != errs.ErrHTTPUnauthorized {
return fmt.Errorf("failed to request auth token (post): %w%.0w", err, errs.ErrHTTPUnauthorized)
}
}
// attempt a get (with basic auth if user/pass available)
if err := b.tryGet(cred); err == nil {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
return nil
} else if err != errs.ErrHTTPUnauthorized {
return fmt.Errorf("failed to request auth token (get): %w%.0w", err, errs.ErrHTTPUnauthorized)
}
return errs.ErrHTTPUnauthorized
}
// isExpired returns true when token issue date is either 0, token has expired,
// or will expire within buffer time
func (b *bearerHandler) isExpired() bool {
if b.token.IssuedAt.IsZero() {
return true
}
expireSec := b.token.IssuedAt.Add(time.Duration(b.token.ExpiresIn) * time.Second)
expireSec = expireSec.Add(tokenBuffer * -1)
return time.Now().After(expireSec)
}
// tryGet requests a new token with a GET request
func (b *bearerHandler) tryGet(cred Cred) error {
//#nosec G704 inputs follow specification
req, err := http.NewRequest("GET", b.tokenURL.String(), nil)
if err != nil {
return err
}
reqParams := req.URL.Query()
reqParams.Add("client_id", b.clientID)
// Note, an offline_token should not be requested by default due to broken OAuth2 implementations returning an invalid token
if b.service != "" {
reqParams.Add("service", b.service)
}
for _, s := range b.scopes {
reqParams.Add("scope", s)
}
if cred.User != "" && cred.Password != "" {
reqParams.Add("account", cred.User)
req.SetBasicAuth(cred.User, cred.Password)
}
req.Header.Add("User-Agent", b.clientID)
req.URL.RawQuery = reqParams.Encode()
//#nosec G704 inputs follow specification
resp, err := b.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return b.validateResponse(resp)
}
// tryPost requests a new token via a POST request
func (b *bearerHandler) tryPost(cred Cred) error {
form := url.Values{}
if len(b.scopes) > 0 {
form.Set("scope", strings.Join(b.scopes, " "))
}
if b.service != "" {
form.Set("service", b.service)
}
form.Set("client_id", b.clientID)
if b.token.RefreshToken != "" {
form.Set("grant_type", "refresh_token")
form.Set("refresh_token", b.token.RefreshToken)
} else if cred.Token != "" {
form.Set("grant_type", "refresh_token")
form.Set("refresh_token", cred.Token)
} else if cred.User != "" && cred.Password != "" {
form.Set("grant_type", "password")
form.Set("username", cred.User)
form.Set("password", cred.Password)
}
//#nosec G704 inputs are user controlled or follow specification
req, err := http.NewRequest("POST", b.tokenURL.String(), strings.NewReader(form.Encode()))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
req.Header.Add("User-Agent", b.clientID)
//#nosec G704 inputs are user controlled or follow specification
resp, err := b.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return b.validateResponse(resp)
}
// scopeExists check if the scope already exists within the list of scopes
func (b *bearerHandler) scopeExists(search string) bool {
if search == "" {
return true
}
searchRepo, searchActions, searchOk := parseScope(search)
if !searchOk {
return slices.Contains(b.scopes, search)
}
scopePrefix := "repository:" + searchRepo + ":"
for _, scope := range b.scopes {
if scope == search {
return true
}
if !strings.HasPrefix(scope, scopePrefix) {
continue
}
_, actions, ok := parseScope(scope)
if !ok {
continue
}
for _, sa := range searchActions {
if !slices.Contains(actions, sa) {
return false
}
}
return true
}
return false
}
// validateResponse extracts the returned token
func (b *bearerHandler) validateResponse(resp *http.Response) error {
if resp.StatusCode != 200 {
return errs.ErrHTTPUnauthorized
}
// decode response and if successful, update token
decoder := json.NewDecoder(resp.Body)
decoded := bearerToken{}
if err := decoder.Decode(&decoded); err != nil {
return err
}
b.token = decoded
if b.token.ExpiresIn < minTokenLife {
b.token.ExpiresIn = minTokenLife
}
// If token is already expired, it was sent with a zero value or
// there may be a clock skew between the client and auth server.
// Also handle cases of remote time in the future.
// But if remote time is slightly in the past, leave as is so token
// expires here before the server.
if b.isExpired() || b.token.IssuedAt.After(time.Now()) {
b.token.IssuedAt = time.Now().UTC()
}
// AccessToken and Token should be the same and we use Token elsewhere
if b.token.AccessToken != "" {
b.token.Token = b.token.AccessToken
}
return nil
}
// jwtHubHandler supports JWT auth type requests.
type jwtHubHandler struct {
client *http.Client
clientID string
realm string
host string
credsFn CredsFn
jwt string
}
type jwtHubPost struct {
User string `json:"username"`
Pass string `json:"password"` //#nosec G117 exported struct intentionally holds secrets
}
type jwtHubResp struct {
Detail string `json:"detail"`
Token string `json:"token"`
RefreshToken string `json:"refresh_token"` //#nosec G117 exported struct intentionally holds secrets
}
// NewJWTHubHandler creates a new JWTHandler for Docker Hub.
func NewJWTHubHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
// JWT handler is only tested against Hub, and the API is Hub specific
if host == "hub.docker.com" {
return &jwtHubHandler{
client: client,
clientID: clientID,
host: host,
credsFn: credsFn,
realm: "https://hub.docker.com/v2/users/login",
}
}
return nil
}
// AddScope is not valid for JWTHubHandler
func (j *jwtHubHandler) AddScope(scope string) error {
return errs.ErrNoNewChallenge
}
// ProcessChallenge handles WWW-Authenticate header for JWT auth on Docker Hub
func (j *jwtHubHandler) ProcessChallenge(c challenge) error {
cred := j.credsFn(j.host)
// use token if provided
if cred.Token != "" {
j.jwt = cred.Token
return nil
}
// send a login request to hub
bodyBytes, err := json.Marshal(jwtHubPost{
User: cred.User,
Pass: cred.Password, //#nosec G117 field name follows spec
})
if err != nil {
return err
}
req, err := http.NewRequest("POST", j.realm, bytes.NewReader(bodyBytes))
if err != nil {
return err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
req.Header.Add("User-Agent", j.clientID)
//#nosec G704 inputs are user controlled or follow specification requirements
resp, err := j.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != 200 || resp.StatusCode >= 300 {
return errs.ErrHTTPUnauthorized
}
var bodyParsed jwtHubResp
err = json.Unmarshal(body, &bodyParsed)
if err != nil {
return err
}
j.jwt = bodyParsed.Token
return nil
}
// UpdateRequest for JWTHubHandler adds JWT header
func (j *jwtHubHandler) UpdateRequest(req *http.Request) error {
if len(j.jwt) > 0 {
req.Header.Set("Authorization", fmt.Sprintf("JWT %s", j.jwt))
return nil
}
return errs.ErrHTTPUnauthorized
}
+48
View File
@@ -0,0 +1,48 @@
package auth
import (
"github.com/regclient/regclient/types/errs"
)
var (
// ErrEmptyChallenge indicates an issue with the received challenge in the WWW-Authenticate header
//
// Deprecated: replace with [errs.ErrEmptyChallenge].
//go:fix inline
ErrEmptyChallenge = errs.ErrEmptyChallenge
// ErrInvalidChallenge indicates an issue with the received challenge in the WWW-Authenticate header
//
// Deprecated: replace with [errs.ErrInvalidChallenge].
//go:fix inline
ErrInvalidChallenge = errs.ErrInvalidChallenge
// ErrNoNewChallenge indicates a challenge update did not result in any change
//
// Deprecated: replace with [errs.ErrNoNewChallenge].
//go:fix inline
ErrNoNewChallenge = errs.ErrNoNewChallenge
// ErrNotFound indicates no credentials found for basic auth
//
// Deprecated: replace with [errs.ErrNotFound].
//go:fix inline
ErrNotFound = errs.ErrNotFound
// ErrNotImplemented returned when method has not been implemented yet
//
// Deprecated: replace with [errs.ErrNotImplemented].
//go:fix inline
ErrNotImplemented = errs.ErrNotImplemented
// ErrParseFailure indicates the WWW-Authenticate header could not be parsed
//
// Deprecated: replace with [errs.ErrParseFailure].
//go:fix inline
ErrParseFailure = errs.ErrParsingFailed
// ErrUnauthorized request was not authorized
//
// Deprecated: replace with [errs.ErrUnauthorized].
//go:fix inline
ErrUnauthorized = errs.ErrHTTPUnauthorized
// ErrUnsupported indicates the request was unsupported
//
// Deprecated: replace with [errs.ErrUnsupported].
//go:fix inline
ErrUnsupported = errs.ErrUnsupported
)
+181
View File
@@ -0,0 +1,181 @@
//go:build go1.18
// Package cache is used to store values with limits.
// Items are automatically pruned when too many entries are stored, or values become stale.
package cache
import (
"sort"
"sync"
"time"
"github.com/regclient/regclient/types/errs"
)
type Cache[k comparable, v any] struct {
mu sync.Mutex
minAge time.Duration
maxAge time.Duration
minCount int
maxCount int
timer *time.Timer
entries map[k]*Entry[v]
}
type Entry[v any] struct {
used time.Time
value v
}
type sortKeys[k comparable] struct {
keys []k
lessFn func(a, b k) bool
}
type conf struct {
minAge time.Duration
maxCount int
}
type cacheOpts func(*conf)
func WithAge(age time.Duration) cacheOpts {
return func(c *conf) {
c.minAge = age
}
}
func WithCount(count int) cacheOpts {
return func(c *conf) {
c.maxCount = count
}
}
func New[k comparable, v any](opts ...cacheOpts) Cache[k, v] {
c := conf{}
for _, opt := range opts {
opt(&c)
}
maxAge := c.minAge + (c.minAge / 10)
minCount := 0
if c.maxCount > 0 {
minCount = int(float64(c.maxCount) * 0.9)
}
return Cache[k, v]{
minAge: c.minAge,
maxAge: maxAge,
minCount: minCount,
maxCount: c.maxCount,
entries: map[k]*Entry[v]{},
}
}
func (c *Cache[k, v]) Delete(key k) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
delete(c.entries, key)
if len(c.entries) == 0 && c.timer != nil {
c.timer.Stop()
c.timer = nil
}
}
func (c *Cache[k, v]) Set(key k, val v) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
c.entries[key] = &Entry[v]{
used: time.Now(),
value: val,
}
if len(c.entries) > c.maxCount {
c.pruneLocked()
} else if c.timer == nil {
// prune resets the timer, so this is only needed if the prune wasn't triggered
c.timer = time.AfterFunc(c.maxAge, c.prune)
}
}
func (c *Cache[k, v]) Get(key k) (v, error) {
if c == nil {
var val v
return val, errs.ErrNotFound
}
c.mu.Lock()
defer c.mu.Unlock()
if e, ok := c.entries[key]; ok {
if e.used.Add(c.minAge).Before(time.Now()) {
// entry expired
go c.prune()
} else {
c.entries[key].used = time.Now()
return e.value, nil
}
}
var val v
return val, errs.ErrNotFound
}
func (c *Cache[k, v]) prune() {
c.mu.Lock()
defer c.mu.Unlock()
c.pruneLocked()
}
func (c *Cache[k, v]) pruneLocked() {
// sort key list by last used date
keyList := make([]k, 0, len(c.entries))
for key := range c.entries {
keyList = append(keyList, key)
}
sk := sortKeys[k]{
keys: keyList,
lessFn: func(a, b k) bool {
return c.entries[a].used.Before(c.entries[b].used)
},
}
sort.Sort(&sk)
// prune entries
now := time.Now()
cutoff := now.Add(c.minAge * -1)
nextTime := now
delCount := len(keyList) - c.minCount
for i, key := range keyList {
if i < delCount || c.entries[key].used.Before(cutoff) {
delete(c.entries, key)
} else {
nextTime = c.entries[key].used
break
}
}
// set next timer
if len(c.entries) > 0 {
dur := nextTime.Sub(now) + c.maxAge
if c.timer == nil {
// this shouldn't be possible
c.timer = time.AfterFunc(dur, c.prune)
} else {
c.timer.Reset(dur)
}
} else if c.timer != nil {
c.timer.Stop()
c.timer = nil
}
}
func (sk *sortKeys[k]) Len() int {
return len(sk.keys)
}
func (sk *sortKeys[k]) Less(i, j int) bool {
return sk.lessFn(sk.keys[i], sk.keys[j])
}
func (sk *sortKeys[k]) Swap(i, j int) {
sk.keys[i], sk.keys[j] = sk.keys[j], sk.keys[i]
}
+188
View File
@@ -0,0 +1,188 @@
// Package conffile wraps the read and write of configuration files
package conffile
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"os/user"
"path/filepath"
)
type File struct {
perms int
fullname string
}
type Opt func(*File)
// New returns a new File.
// The last successful option determines the filename.
func New(opts ...Opt) *File {
f := File{perms: 0o600}
for _, fn := range opts {
fn(&f)
}
if f.fullname == "" {
return nil
}
return &f
}
// WithAppDir determines the filename from the XDG or Windows specification.
// By default, this is based in $HOME/.config on Linux and %APPDATA% on Windows.
// If the file does not exist, this will set the filename only if "force" is true.
func WithAppDir(unixDir, winDir, name string, force bool) Opt {
var dir string
if winDir == "" {
dir = unixDir
} else {
dir = osString(unixDir, winDir)
}
return func(f *File) {
fullname := filepath.Join(appDir(), dir, name)
if force || exists(fullname) {
f.fullname = fullname
}
}
}
// WithDirName determines the filename from a subdirectory in the user's HOME.
//
// Deprecated: Replace with [WithHomeDir]
//
//go:fix inline
func WithDirName(dir, name string) Opt {
return WithHomeDir(dir, name, true)
}
// WithEnvFile sets the fullname to the environment value if defined.
func WithEnvFile(envVar string) Opt {
return func(f *File) {
val := os.Getenv(envVar)
if val != "" {
f.fullname = val
}
}
}
// WithEnvDir sets the fullname to the environment value + filename if the environment variable is defined.
func WithEnvDir(envVar, name string) Opt {
return func(f *File) {
val := os.Getenv(envVar)
if val != "" {
f.fullname = filepath.Join(val, name)
}
}
}
// WithFullname specifies the filename.
// This will always set the filename even if the file does not exist.
func WithFullname(fullname string) Opt {
return func(f *File) {
f.fullname = fullname
}
}
// WithHomeDir determines the filename from a subdirectory in the user's HOME
// e.g. dir=".app", name="config.json", sets the fullname to "$HOME/.app/config.json".
// If the file does not exist, this will set the filename only if "force" is true.
func WithHomeDir(dir, name string, force bool) Opt {
return func(f *File) {
filename := filepath.Join(homeDir(), dir, name)
if force || exists(filename) {
f.fullname = filename
}
}
}
// WithPerms specifies the permissions to create a file with (default 0600).
func WithPerms(perms int) Opt {
return func(f *File) {
f.perms = perms
}
}
func (f *File) Name() string {
return f.fullname
}
func (f *File) Open() (io.ReadCloser, error) {
return os.Open(f.fullname)
}
func (f *File) Write(rdr io.Reader) error {
// create temp file/open
dir := filepath.Dir(f.fullname)
if err := os.MkdirAll(dir, 0o700); err != nil {
return err
}
tmp, err := os.CreateTemp(dir, filepath.Base(f.fullname))
if err != nil {
return err
}
tmpStat, err := tmp.Stat()
if err != nil {
return err
}
tmpName := tmpStat.Name()
tmpFullname := filepath.Join(dir, tmpName)
defer os.Remove(tmpFullname)
// copy from rdr to temp file
_, err = io.Copy(tmp, rdr)
errC := tmp.Close()
if err != nil {
return fmt.Errorf("failed to write config: %w", err)
}
if errC != nil {
return fmt.Errorf("failed to close config: %w", errC)
}
// adjust file ownership/permissions
mode := os.FileMode(0o600)
uid := os.Getuid()
gid := os.Getgid()
// adjust defaults based on existing file if available
stat, err := os.Stat(f.fullname)
if err == nil {
// adjust mode to existing file
if stat.Mode().IsRegular() {
mode = stat.Mode()
}
uid, gid, _ = getFileOwner(stat)
} else if !errors.Is(err, fs.ErrNotExist) {
return err
}
// update mode and owner of temp file
//#nosec G703 tempfile location is user controlled
if err := os.Chmod(tmpFullname, mode); err != nil {
return err
}
if uid > 0 && gid > 0 {
//#nosec G703 tempfile location is user controlled
_ = os.Chown(tmpFullname, uid, gid)
}
// move temp file to target filename
//#nosec G703 tempfile location is user controlled
return os.Rename(tmpFullname, f.fullname)
}
func exists(name string) bool {
_, err := os.Stat(name)
return err == nil
}
func homeDir() string {
home := os.Getenv(homeEnv)
if home == "" {
u, err := user.Current()
if err == nil {
home = u.HomeDir
}
}
return home
}
@@ -0,0 +1,37 @@
//go:build !windows
package conffile
import (
"io/fs"
"os"
"path/filepath"
"syscall"
)
const (
appDirEnv = "XDG_CONFIG_HOME"
homeEnv = "HOME"
)
func appDir() string {
appDir := os.Getenv(appDirEnv)
if appDir == "" {
home := homeDir()
appDir = filepath.Join(home, ".config")
}
return appDir
}
func getFileOwner(stat fs.FileInfo) (int, int, error) {
var uid, gid int
if sysstat, ok := stat.Sys().(*syscall.Stat_t); ok {
uid = int(sysstat.Uid)
gid = int(sysstat.Gid)
}
return uid, gid, nil
}
func osString(unix, _ string) string {
return unix
}
@@ -0,0 +1,31 @@
//go:build windows
package conffile
import (
"io/fs"
"os"
"path/filepath"
)
const (
appDirEnv = "APPDATA"
homeEnv = "USERPROFILE"
)
func appDir() string {
appDir := os.Getenv(appDirEnv)
if appDir == "" {
home := homeDir()
appDir = filepath.Join(home, "AppData")
}
return appDir
}
func getFileOwner(_ fs.FileInfo) (int, int, error) {
return 0, 0, nil
}
func osString(_, win string) string {
return win
}
+198
View File
@@ -0,0 +1,198 @@
// Package httplink parses the Link header from HTTP responses according to RFC5988
package httplink
import (
"fmt"
"strings"
"github.com/regclient/regclient/types/errs"
)
type (
Links []Link
Link struct {
URI string
Param map[string]string
}
)
type charLU byte
var charLUs [256]charLU
const (
isSpace charLU = 1 << iota
isToken
isAlphaNum
)
func init() {
for c := range 256 {
charLUs[c] = 0
if strings.ContainsRune(" \t\r\n", rune(c)) {
charLUs[c] |= isSpace
}
if (rune('a') <= rune(c) && rune(c) <= rune('z')) || (rune('A') <= rune(c) && rune(c) <= rune('Z') || (rune('0') <= rune(c) && rune(c) <= rune('9'))) {
charLUs[c] |= isAlphaNum | isToken
}
if strings.ContainsRune("!#$%&'()*+-./:<=>?@[]^_`{|}~", rune(c)) {
charLUs[c] |= isToken
}
}
}
// Parse reads "Link" http headers into an array of Link structs.
// Header array should be the output of resp.Header.Values("link").
func Parse(headers []string) (Links, error) {
links := []Link{}
for _, h := range headers {
state := "init"
var ub, pnb, pvb []byte
parms := map[string]string{}
endLink := func() {
links = append(links, Link{
URI: string(ub),
Param: parms,
})
// reset state
ub, pnb, pvb = []byte{}, []byte{}, []byte{}
parms = map[string]string{}
}
endParm := func() {
if _, ok := parms[string(pnb)]; !ok {
parms[string(pnb)] = string(pvb)
}
// reset parm
pnb, pvb = []byte{}, []byte{}
}
for i, b := range []byte(h) {
switch state {
case "init":
if b == '<' {
state = "uriQuoted"
} else if charLUs[b]&isToken != 0 {
state = "uri"
ub = append(ub, b)
} else if charLUs[b]&isSpace != 0 || b == ',' {
// noop
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "uri":
// parse tokens until space or comma
if charLUs[b]&isToken != 0 {
ub = append(ub, b)
} else if charLUs[b]&isSpace != 0 {
state = "fieldSep"
} else if b == ';' {
state = "parmName"
} else if b == ',' {
state = "init"
endLink()
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "uriQuoted":
// parse tokens until quote
if b == '>' {
state = "fieldSep"
} else {
ub = append(ub, b)
}
case "fieldSep":
if b == ';' {
state = "parmName"
} else if b == ',' {
state = "init"
endLink()
} else if charLUs[b]&isSpace != 0 {
// noop
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "parmName":
if len(pnb) > 0 && b == '=' {
state = "parmValue"
} else if len(pnb) > 0 && b == '*' {
state = "parmNameStar"
} else if charLUs[b]&isAlphaNum != 0 {
pnb = append(pnb, b)
} else if len(pnb) == 0 && charLUs[b]&isSpace != 0 {
// noop
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "parmNameStar":
if b == '=' {
state = "parmValue"
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "parmValue":
if len(pvb) == 0 {
if charLUs[b]&isToken != 0 {
pvb = append(pvb, b)
} else if b == '"' {
state = "parmValueQuoted"
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
} else {
if charLUs[b]&isToken != 0 {
pvb = append(pvb, b)
} else if charLUs[b]&isSpace != 0 {
state = "fieldSep"
endParm()
} else if b == ';' {
state = "parmName"
endParm()
} else if b == ',' {
state = "init"
endParm()
endLink()
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
}
case "parmValueQuoted":
if b == '"' {
state = "fieldSep"
endParm()
} else {
pvb = append(pvb, b)
}
}
}
// check for valid state at end of header
switch state {
case "parmValue":
endParm()
endLink()
case "uri", "fieldSep":
endLink()
case "init":
// noop
default:
return nil, fmt.Errorf("unexpected end state %s for header %s: %w", state, h, errs.ErrParsingFailed)
}
}
return links, nil
}
// Get returns a link with a specific parm value, e.g. rel="next"
func (links Links) Get(parm, val string) (Link, error) {
for _, link := range links {
if link.Param != nil && link.Param[parm] == val {
return link, nil
}
}
return Link{}, errs.ErrNotFound
}
+29
View File
@@ -0,0 +1,29 @@
// Package limitread provides a reader that will error if the limit is ever exceeded
package limitread
import (
"fmt"
"io"
"github.com/regclient/regclient/types/errs"
)
type LimitRead struct {
Reader io.Reader
Limit int64
}
func (lr *LimitRead) Read(p []byte) (int, error) {
if lr.Limit < 0 {
return 0, fmt.Errorf("read limit exceeded%.0w", errs.ErrSizeLimitExceeded)
}
if int64(len(p)) > lr.Limit+1 {
p = p[0 : lr.Limit+1]
}
n, err := lr.Reader.Read(p)
lr.Limit -= int64(n)
if lr.Limit < 0 {
return n, fmt.Errorf("read limit exceeded%.0w", errs.ErrSizeLimitExceeded)
}
return n, err
}
+257
View File
@@ -0,0 +1,257 @@
// Package pqueue implements a priority queue.
package pqueue
import (
"context"
"fmt"
"slices"
"sync"
)
type Queue[T any] struct {
mu sync.Mutex
max int
next func(queued, active []*T) int
active []*T
queued []*T
wait []*chan struct{}
}
// Opts is used to configure a new priority queue.
type Opts[T any] struct {
Max int // maximum concurrent entries, defaults to 1.
Next func(queued, active []*T) int // function to lookup index of next queued entry to release, defaults to oldest entry.
}
// New creates a new priority queue.
func New[T any](opts Opts[T]) *Queue[T] {
if opts.Max <= 0 {
opts.Max = 1
}
return &Queue[T]{
max: opts.Max,
next: opts.Next,
}
}
// Acquire adds a new entry to the queue and returns once it is ready.
// The returned function must be called when the queued job completes to release the next entry.
// If there is any error, the returned function will be nil.
func (q *Queue[T]) Acquire(ctx context.Context, e T) (func(), error) {
if q == nil {
return func() {}, nil
}
found, err := q.checkContext(ctx)
if err != nil {
return nil, err
}
if found {
return func() {}, nil
}
q.mu.Lock()
if len(q.active)+len(q.queued) < q.max {
q.active = append(q.active, &e)
q.mu.Unlock()
return q.releaseFn(&e), nil
}
// limit reached, add to queue and wait
w := make(chan struct{}, 1)
q.queued = append(q.queued, &e)
q.wait = append(q.wait, &w)
q.mu.Unlock()
// wait on both context and queue
select {
case <-ctx.Done():
// context abort, remove queued entry
q.mu.Lock()
if i := slices.Index(q.queued, &e); i >= 0 {
q.queued = slices.Delete(q.queued, i, i+1)
q.wait = slices.Delete(q.wait, i, i+1)
q.mu.Unlock()
return nil, ctx.Err()
}
q.mu.Unlock()
// queued entry found, assume race condition with context and entry being released, release next entry
q.release(&e)
return nil, ctx.Err()
case <-w:
return q.releaseFn(&e), nil
}
}
// TryAcquire attempts to add an entry on to the list of active entries.
// If the returned function is nil, the queue was not available.
// If the returned function is not nil, it must be called when the job is complete to release the next entry.
func (q *Queue[T]) TryAcquire(ctx context.Context, e T) (func(), error) {
if q == nil {
return func() {}, nil
}
found, err := q.checkContext(ctx)
if err != nil {
return nil, err
}
if found {
return func() {}, nil
}
q.mu.Lock()
defer q.mu.Unlock()
if len(q.active)+len(q.queued) < q.max {
q.active = append(q.active, &e)
return q.releaseFn(&e), nil
}
return nil, nil
}
// release next entry or noop.
func (q *Queue[T]) release(prev *T) {
q.mu.Lock()
defer q.mu.Unlock()
// remove prev entry from active list
if i := slices.Index(q.active, prev); i >= 0 {
q.active = slices.Delete(q.active, i, i+1)
}
// skip checks when at limit or nothing queued
if len(q.queued) == 0 {
if len(q.active) == 0 {
// free up slices if this was the last active entry
q.active = nil
q.queued = nil
q.wait = nil
}
return
}
if len(q.active) >= q.max {
return
}
i := 0
if q.next != nil && len(q.queued) > 1 {
i = q.next(q.queued, q.active)
// validate response
i = max(min(i, len(q.queued)-1), 0)
}
// release queued entry, move to active list, and remove from queued/wait lists
close(*q.wait[i])
q.active = append(q.active, q.queued[i])
q.queued = slices.Delete(q.queued, i, i+1)
q.wait = slices.Delete(q.wait, i, i+1)
}
// releaseFn is a convenience wrapper around [release].
func (q *Queue[T]) releaseFn(prev *T) func() {
return func() {
q.release(prev)
}
}
// TODO: is there a way to make a different context key for each generic type?
type ctxType int
var ctxKey ctxType
type valMulti[T any] struct {
qList []*Queue[T]
}
// AcquireMulti is used to simultaneously lock multiple queues without the risk of deadlock.
// The returned context needs to be used on calls to [Acquire] or [TryAcquire] which will immediately succeed since the resource is already acquired.
// Attempting to acquire other resources with [Acquire], [TryAcquire], or [AcquireMulti] using the returned context and will fail for being outside of the transaction.
// The returned function must be called to release the resources.
// The returned function is not thread safe, ensure no other simultaneous calls to [Acquire] or [TryAcquire] using the returned context have finished before it is called.
func AcquireMulti[T any](ctx context.Context, e T, qList ...*Queue[T]) (context.Context, func(), error) {
// verify context not already holding locks
qCtx := ctx.Value(ctxKey)
if qCtx != nil {
if qCtxVal, ok := qCtx.(*valMulti[T]); !ok || qCtxVal.qList != nil {
return ctx, nil, fmt.Errorf("context already used by another AcquireMulti request")
}
}
// delete nil entries
for i := len(qList) - 1; i >= 0; i-- {
if qList[i] == nil {
qList = slices.Delete(qList, i, i+1)
}
}
// empty/nil list is a noop
if len(qList) == 0 {
return ctx, func() {}, nil
}
// dedup entries from the list
for i := len(qList) - 2; i >= 0; i-- {
for j := len(qList) - 1; j > i; j-- {
if qList[i] == qList[j] {
qList[j] = qList[len(qList)-1]
qList = qList[:len(qList)-1]
}
}
}
// Loop through queues to acquire, waiting on the first, and attempting the remaining.
// If any of the remaining entries cannot be immediately acquired, reset and make it the new queue to wait on.
lockI := 0
doneList := make([]func(), len(qList))
for {
acquired := true
i := 0
done, err := qList[lockI].Acquire(ctx, e)
if err != nil {
return ctx, nil, err
}
doneList[lockI] = done
for i < len(qList) {
if i != lockI {
doneList[i], err = qList[i].TryAcquire(ctx, e)
if doneList[i] == nil || err != nil {
acquired = false
break
}
}
i++
}
if err == nil && acquired {
break
}
// cleanup on failed attempt
if lockI > i {
doneList[lockI]()
}
// track blocking index for a retry
lockI = i
for i > 0 {
i--
doneList[i]()
}
// abort on errors
if err != nil {
return ctx, nil, err
}
}
// success, update context
ctxVal := valMulti[T]{qList: qList}
newCtx := context.WithValue(ctx, ctxKey, &ctxVal)
cleanup := func() {
ctxVal.qList = nil
// dequeue in reverse order to minimize chance of another AcquireMulti being freed and immediately blocking on the next queue
for i := len(doneList) - 1; i >= 0; i-- {
doneList[i]()
}
}
return newCtx, cleanup, nil
}
func (q *Queue[T]) checkContext(ctx context.Context) (bool, error) {
qCtx := ctx.Value(ctxKey)
if qCtx == nil {
return false, nil
}
qCtxVal, ok := qCtx.(*valMulti[T])
if !ok {
return false, nil // another type is using the context, treat it as unset
}
if qCtxVal.qList == nil {
return false, nil
}
if slices.Contains(qCtxVal.qList, q) {
// instance already locked
return true, nil
}
return true, fmt.Errorf("cannot acquire new locks during a transaction")
}
+987
View File
@@ -0,0 +1,987 @@
// Package reghttp is used for HTTP requests to a registry
package reghttp
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/regclient/regclient/config"
"github.com/regclient/regclient/internal/auth"
"github.com/regclient/regclient/internal/pqueue"
"github.com/regclient/regclient/internal/regnet"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/types"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/warning"
)
var (
defaultDelayInit, _ = time.ParseDuration("0.1s")
defaultDelayMax, _ = time.ParseDuration("30s")
warnRegexp = regexp.MustCompile(`^299\s+-\s+"([^"]+)"`)
)
const (
DefaultRetryLimit = 5 // number of times a request will be retried
backoffResetCount = 5 // number of successful requests needed to reduce the backoff
)
// Client is an HTTP client wrapper.
// It handles features like authentication, retries, backoff delays, TLS settings.
type Client struct {
httpClient *http.Client // upstream [http.Client], this is wrapped per repository for an auth handler on redirects
getConfigHost func(string) *config.Host // call-back to get the [config.Host] for a specific registry
host map[string]*clientHost // host specific settings, wrap access with a mutex lock
rootCAPool [][]byte // list of root CAs for configuring the http.Client transport
rootCADirs []string // list of directories for additional root CAs
retryLimit int // number of retries before failing a request, this applies to each host, and each request
delayInit time.Duration // how long to initially delay requests on a failure
delayMax time.Duration // maximum time to delay a request
slog *slog.Logger // logging for tracing and failures
userAgent string // user agent to specify in http request headers
mu sync.Mutex // mutex to prevent data races
}
type clientHost struct {
config *config.Host // config entry
httpClient *http.Client // modified http client for registry specific settings
userAgent string // user agent to specify in http request headers
slog *slog.Logger // logging for tracing and failures
auth map[string]*auth.Auth // map of auth handlers by repository
backoffCur int // current count of backoffs for this host
backoffLast time.Time // time the last request was released, this may be in the future if there is a queue, or zero if no delay is needed
backoffReset int // count of successful requests when a backoff is experienced, once [backoffResetCount] is reached, [backoffCur] is reduced by one and this is reset to 0
reqFreq time.Duration // how long between submitting requests for this host
reqNext time.Time // time to release the next request
throttle *pqueue.Queue[reqmeta.Data] // limit concurrent requests to the host
mu sync.Mutex // mutex to prevent data races
}
// Req is a request to send to a registry.
type Req struct {
MetaKind reqmeta.Kind // kind of request for the priority queue
Host string // registry name, hostname and mirrors will be looked up from host configuration
Method string // http method to call
DirectURL *url.URL // url to query, overrides repository, path, and query
Repository string // repository to scope the request
Path string // path of the request within a repository
Query url.Values // url query parameters
BodyLen int64 // length of body to send
BodyBytes []byte // bytes of the body, overridden by BodyFunc
BodyFunc func() (io.ReadCloser, error) // function to return a new body
Headers http.Header // headers to send in the request
NoPrefix bool // do not include the repository prefix
NoMirrors bool // do not send request to a mirror
ExpectLen int64 // expected size of the returned body
TransactLen int64 // size of an overall transaction for the priority queue
IgnoreErr bool // ignore http errors and do not trigger backoffs
}
// Resp is used to handle the result of a request.
type Resp struct {
ctx context.Context
client *Client
req *Req
resp *http.Response
mirror string
done bool
reader io.Reader
readCur, readMax int64
retryCount int
throttleDone func()
}
// Opts is used to configure client options.
type Opts func(*Client)
// NewClient returns a client for handling requests.
func NewClient(opts ...Opts) *Client {
c := Client{
httpClient: &http.Client{},
host: map[string]*clientHost{},
retryLimit: DefaultRetryLimit,
delayInit: defaultDelayInit,
delayMax: defaultDelayMax,
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
rootCAPool: [][]byte{},
rootCADirs: []string{},
}
for _, opt := range opts {
opt(&c)
}
return &c
}
// WithCerts adds certificates.
func WithCerts(certs [][]byte) Opts {
return func(c *Client) {
c.rootCAPool = append(c.rootCAPool, certs...)
}
}
// WithCertDirs adds directories to check for host specific certs.
func WithCertDirs(dirs []string) Opts {
return func(c *Client) {
c.rootCADirs = append(c.rootCADirs, dirs...)
}
}
// WithCertFiles adds certificates by filename.
func WithCertFiles(files []string) Opts {
return func(c *Client) {
for _, f := range files {
//#nosec G304 command is run by a user accessing their own files
cert, err := os.ReadFile(f)
if err != nil {
c.slog.Warn("Failed to read certificate",
slog.String("err", err.Error()),
slog.String("file", f))
} else {
c.rootCAPool = append(c.rootCAPool, cert)
}
}
}
}
// WithConfigHostFn adds the callback to request a [config.Host] struct.
// The function must normalize the hostname for Docker Hub support.
func WithConfigHostFn(gch func(string) *config.Host) Opts {
return func(c *Client) {
c.getConfigHost = gch
}
}
// WithDelay initial time to wait between retries (increased with exponential backoff).
func WithDelay(delayInit time.Duration, delayMax time.Duration) Opts {
return func(c *Client) {
if delayInit > 0 {
c.delayInit = delayInit
}
// delayMax must be at least delayInit, if 0 initialize to 30x delayInit
if delayMax > c.delayInit {
c.delayMax = delayMax
} else if delayMax > 0 {
c.delayMax = c.delayInit
} else {
c.delayMax = c.delayInit * 30
}
}
}
// WithHTTPClient uses a specific http client with retryable requests.
func WithHTTPClient(hc *http.Client) Opts {
return func(c *Client) {
c.httpClient = hc
}
}
// WithRetryLimit restricts the number of retries (defaults to 5).
func WithRetryLimit(rl int) Opts {
return func(c *Client) {
if rl > 0 {
c.retryLimit = rl
}
}
}
// WithLog injects a slog Logger configuration.
func WithLog(slog *slog.Logger) Opts {
return func(c *Client) {
c.slog = slog
}
}
// WithTransport uses a specific http transport with retryable requests.
func WithTransport(t *http.Transport) Opts {
return func(c *Client) {
c.httpClient = &http.Client{Transport: t}
}
}
// WithUserAgent sets a user agent header.
func WithUserAgent(ua string) Opts {
return func(c *Client) {
c.userAgent = ua
}
}
// Do runs a request, returning the response result.
func (c *Client) Do(ctx context.Context, req *Req) (*Resp, error) {
resp := &Resp{
ctx: ctx,
client: c,
req: req,
readCur: 0,
readMax: req.ExpectLen,
}
err := resp.next()
return resp, err
}
// next sends requests until a mirror responds or all requests fail.
func (resp *Resp) next() error {
var err error
c := resp.client
req := resp.req
// lookup reqHost entry
reqHost := c.getHost(req.Host)
// create sorted list of mirrors, based on backoffs, upstream, and priority
hosts := make([]*clientHost, 0, 1+len(reqHost.config.Mirrors))
if !req.NoMirrors {
for _, m := range reqHost.config.Mirrors {
hosts = append(hosts, c.getHost(m))
}
}
hosts = append(hosts, reqHost)
sort.Slice(hosts, sortHostsCmp(hosts, reqHost.config.Name))
// loop over requests to mirrors and retries
curHost := 0
for {
backoff := false
dropHost := false
retryHost := false
if len(hosts) == 0 {
if err != nil {
return err
}
return errs.ErrAllRequestsFailed
}
if curHost >= len(hosts) {
curHost = 0
}
h := hosts[curHost]
resp.mirror = h.config.Name
// there is an intentional extra retry in this check to allow for auth requests
if resp.retryCount > c.retryLimit {
return errs.ErrRetryLimitExceeded
}
resp.retryCount++
// check that context isn't canceled/done
ctxErr := resp.ctx.Err()
if ctxErr != nil {
return ctxErr
}
// wait for other concurrent requests to this host
throttleDone, throttleErr := h.throttle.Acquire(resp.ctx, reqmeta.Data{
Kind: req.MetaKind,
Size: req.BodyLen + req.ExpectLen + req.TransactLen,
})
if throttleErr != nil {
return throttleErr
}
// try each host in a closure to handle all the backoff/dropHost from one place
loopErr := func() error {
var err error
if req.Method == "HEAD" && h.config.APIOpts != nil {
var disableHead bool
disableHead, err = strconv.ParseBool(h.config.APIOpts["disableHead"])
if err == nil && disableHead {
dropHost = true
return fmt.Errorf("head requests disabled for host \"%s\": %w", h.config.Name, errs.ErrUnsupportedAPI)
}
}
// build the url
var u url.URL
if req.DirectURL != nil {
u = *req.DirectURL
} else {
u = url.URL{
Host: h.config.Hostname,
Scheme: "https",
}
path := strings.Builder{}
path.WriteString("/v2")
if h.config.PathPrefix != "" && !req.NoPrefix {
path.WriteString("/" + h.config.PathPrefix)
}
if req.Repository != "" {
path.WriteString("/" + req.Repository)
}
path.WriteString("/" + req.Path)
u.Path = path.String()
if h.config.TLS == config.TLSDisabled {
u.Scheme = "http"
}
query := url.Values{}
if req.Query != nil {
query = req.Query
}
if h.config.Hostname != reqHost.config.Hostname {
query.Set("ns", reqHost.config.Hostname)
}
u.RawQuery = query.Encode()
}
// close previous response
if resp.resp != nil && resp.resp.Body != nil {
_ = resp.resp.Body.Close()
}
// delay for backoff if needed
bu := resp.backoffGet()
if !bu.IsZero() && bu.After(time.Now()) {
sleepTime := time.Until(bu)
c.slog.Debug("Sleeping for backoff",
slog.String("Host", h.config.Name),
slog.Duration("Duration", sleepTime))
select {
case <-resp.ctx.Done():
return errs.ErrCanceled
case <-time.After(sleepTime):
}
}
var httpReq *http.Request
httpReq, err = http.NewRequestWithContext(resp.ctx, req.Method, u.String(), nil)
if err != nil {
dropHost = true
return err
}
if req.BodyFunc != nil {
body, err := req.BodyFunc()
if err != nil {
dropHost = true
return err
}
httpReq.Body = body
httpReq.GetBody = req.BodyFunc
httpReq.ContentLength = req.BodyLen
} else if len(req.BodyBytes) > 0 {
body := io.NopCloser(bytes.NewReader(req.BodyBytes))
httpReq.Body = body
httpReq.GetBody = func() (io.ReadCloser, error) { return body, nil }
httpReq.ContentLength = req.BodyLen
}
if len(req.Headers) > 0 {
httpReq.Header = req.Headers.Clone()
}
if c.userAgent != "" && httpReq.Header.Get("User-Agent") == "" {
httpReq.Header.Add("User-Agent", c.userAgent)
}
if resp.readCur > 0 && resp.readMax > 0 {
if req.Headers.Get("Range") == "" {
httpReq.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", resp.readCur, resp.readMax))
} else {
// TODO: support Seek within a range request
dropHost = true
return fmt.Errorf("unable to resume a connection within a range request")
}
}
hAuth := h.getAuth(req.Repository)
if hAuth != nil {
// include docker generated scope to emulate docker clients
if req.Repository != "" {
scope := "repository:" + req.Repository + ":pull"
if req.Method != "HEAD" && req.Method != "GET" {
scope = scope + ",push"
}
_ = hAuth.AddScope(h.config.Hostname, scope)
}
// add auth headers
err = hAuth.UpdateRequest(httpReq)
// abort on auth errors, but only after the first request has been attempted
if err != nil && resp.resp != nil {
if errors.Is(err, errs.ErrHTTPUnauthorized) {
dropHost = true
} else {
backoff = true
}
return err
}
}
// delay for the rate limit
if h.reqFreq > 0 {
sleep := time.Duration(0)
h.mu.Lock()
if time.Now().Before(h.reqNext) {
sleep = time.Until(h.reqNext)
h.reqNext = h.reqNext.Add(h.reqFreq)
} else {
h.reqNext = time.Now().Add(h.reqFreq)
}
h.mu.Unlock()
if sleep > 0 {
time.Sleep(sleep)
}
}
// send request
hc := h.getHTTPClient(req.Repository)
//#nosec G704 inputs are user controlled and sanitized
resp.resp, err = hc.Do(httpReq)
if err != nil {
c.slog.Debug("Request failed",
slog.String("URL", u.String()),
slog.String("err", err.Error()))
backoff = true
return err
}
statusCode := resp.resp.StatusCode
if statusCode < 200 || statusCode >= 300 {
switch statusCode {
case http.StatusUnauthorized:
// if auth can be done, retry same host without delay, otherwise drop/backoff
if hAuth != nil {
err = hAuth.HandleResponse(resp.resp)
} else {
err = fmt.Errorf("authentication handler unavailable")
}
if err != nil {
if errors.Is(err, errs.ErrEmptyChallenge) || errors.Is(err, errs.ErrNoNewChallenge) || errors.Is(err, errs.ErrHTTPUnauthorized) {
c.slog.Debug("Failed to handle auth request",
slog.String("URL", u.String()),
slog.String("Err", err.Error()))
} else {
c.slog.Warn("Failed to handle auth request",
slog.String("URL", u.String()),
slog.String("Err", err.Error()))
}
dropHost = true
} else {
err = fmt.Errorf("authentication required")
retryHost = true
}
return err
case http.StatusNotFound:
// if not found, drop mirror for this req, but other requests don't need backoff
dropHost = true
case http.StatusRequestedRangeNotSatisfiable:
// if range request error (blob push), drop mirror for this req, but other requests don't need backoff
dropHost = true
case http.StatusTooManyRequests, http.StatusRequestTimeout, http.StatusGatewayTimeout, http.StatusBadGateway, http.StatusInternalServerError:
// server is likely overloaded, backoff but still retry
backoff = true
default:
// all other errors indicate a bigger issue, don't retry and set backoff
backoff = true
dropHost = true
}
errHTTP := HTTPError(resp.resp.StatusCode)
errBody, _ := io.ReadAll(resp.resp.Body)
_ = resp.resp.Body.Close()
return fmt.Errorf("request failed: %w: %s", errHTTP, errBody)
}
resp.reader = resp.resp.Body
resp.done = false
// set variables from headers if found
clHeader := resp.resp.Header.Get("Content-Length")
if resp.readCur == 0 && clHeader != "" {
cl, parseErr := strconv.ParseInt(clHeader, 10, 64)
if parseErr != nil {
c.slog.Debug("failed to parse content-length header",
slog.String("err", parseErr.Error()),
slog.String("header", clHeader))
} else if resp.readMax > 0 {
if resp.readMax != cl {
return fmt.Errorf("unexpected content-length, expected %d, received %d", resp.readMax, cl)
}
} else {
resp.readMax = cl
}
}
// verify Content-Range header when range request used, fail if missing
if httpReq.Header.Get("Range") != "" && resp.resp.Header.Get("Content-Range") == "" {
dropHost = true
_ = resp.resp.Body.Close()
return fmt.Errorf("range request not supported by server")
}
return nil
}()
// return on success
if loopErr == nil {
resp.throttleDone = throttleDone
return nil
}
// backoff, dropHost, and/or go to next host in the list
if backoff {
if req.IgnoreErr {
// don't set a backoff, immediately drop the host when errors ignored
dropHost = true
} else {
boErr := resp.backoffSet()
if boErr != nil {
// reached backoff limit
dropHost = true
}
}
}
throttleDone()
// when error does not allow retries, abort with the last known err value
if err != nil && errors.Is(loopErr, errs.ErrNotRetryable) {
return err
}
err = loopErr
if dropHost {
hosts = slices.Delete(hosts, curHost, curHost+1)
} else if !retryHost {
curHost++
}
}
}
// GetThrottle returns the current [pqueue.Queue] for a host used to throttle connections.
// This can be used to acquire multiple throttles before performing a request across multiple hosts.
func (c *Client) GetThrottle(host string) *pqueue.Queue[reqmeta.Data] {
ch := c.getHost(host)
return ch.throttle
}
// HTTPResponse returns the [http.Response] from the last request.
func (resp *Resp) HTTPResponse() *http.Response {
return resp.resp
}
// Read provides a retryable read from the body of the response.
func (resp *Resp) Read(b []byte) (int, error) {
if resp.done || resp.reader == nil {
return 0, io.EOF
}
if resp.resp == nil {
return 0, errs.ErrNotFound
}
// perform the read
i, err := resp.reader.Read(b)
resp.readCur += int64(i)
if err == io.EOF || err == io.ErrUnexpectedEOF {
if resp.resp.Request.Method == "HEAD" || resp.readCur >= resp.readMax {
resp.backoffReset()
resp.done = true
} else {
// short read, retry?
resp.client.slog.Debug("EOF before reading all content, retrying",
slog.Int64("curRead", resp.readCur),
slog.Int64("contentLen", resp.readMax))
// retry
respErr := resp.backoffSet()
if respErr == nil {
respErr = resp.next()
}
// unrecoverable EOF
if respErr != nil {
resp.client.slog.Warn("Failed to recover from short read",
slog.String("err", respErr.Error()))
resp.done = true
return i, err
}
// retry successful, no EOF
return i, nil
}
}
if err == nil {
return i, nil
}
return i, err
}
// Close frees up resources from the request.
func (resp *Resp) Close() error {
if resp.throttleDone != nil {
resp.throttleDone()
resp.throttleDone = nil
}
if resp.resp == nil {
return errs.ErrNotFound
}
if !resp.done {
resp.backoffReset()
}
resp.done = true
return resp.resp.Body.Close()
}
// Seek provides a limited ability seek within the request response.
func (resp *Resp) Seek(offset int64, whence int) (int64, error) {
newOffset := resp.readCur
switch whence {
case io.SeekStart:
newOffset = offset
case io.SeekCurrent:
newOffset += offset
case io.SeekEnd:
if resp.readMax <= 0 {
return resp.readCur, fmt.Errorf("seek from end is not supported")
} else if resp.readMax+offset < 0 {
return resp.readCur, fmt.Errorf("seek past beginning of the file is not supported")
}
newOffset = resp.readMax + offset
default:
return resp.readCur, fmt.Errorf("unknown value of whence: %d", whence)
}
if newOffset != resp.readCur {
resp.readCur = newOffset
// rerun the request to restart
resp.retryCount-- // do not count a seek as a retry
err := resp.next()
if err != nil {
return resp.readCur, err
}
}
return resp.readCur, nil
}
func (resp *Resp) backoffGet() time.Time {
c := resp.client
ch := c.getHost(resp.mirror)
ch.mu.Lock()
defer ch.mu.Unlock()
if ch.backoffCur > 0 {
delay := c.delayInit << ch.backoffCur
delay = min(delay, c.delayMax)
next := ch.backoffLast.Add(delay)
now := time.Now()
if now.After(next) {
next = now
}
ch.backoffLast = next
return next
}
// reset a stale "retry-after" time
if !ch.backoffLast.IsZero() && ch.backoffLast.Before(time.Now()) {
ch.backoffLast = time.Time{}
}
return ch.backoffLast
}
func (resp *Resp) backoffSet() error {
c := resp.client
ch := c.getHost(resp.mirror)
ch.mu.Lock()
defer ch.mu.Unlock()
// check rate limit header and use that directly if possible
if resp.resp != nil && resp.resp.Header.Get("Retry-After") != "" {
ras := resp.resp.Header.Get("Retry-After")
ra, _ := time.ParseDuration(ras + "s")
if ra > 0 {
next := time.Now().Add(ra)
if ch.backoffLast.Before(next) {
ch.backoffLast = next
}
return nil
}
}
// Else track the number of backoffs and fail when the limit is exceeded.
// New requests always get at least one try, but fail fast if the server has been throwing errors.
ch.backoffCur++
if ch.backoffLast.IsZero() {
ch.backoffLast = time.Now()
}
if ch.backoffCur >= c.retryLimit {
return fmt.Errorf("%w: backoffs %d", errs.ErrBackoffLimit, ch.backoffCur)
}
return nil
}
func (resp *Resp) backoffReset() {
c := resp.client
ch := c.getHost(resp.mirror)
ch.mu.Lock()
defer ch.mu.Unlock()
if ch.backoffCur > 0 {
ch.backoffReset++
// If enough successful requests are seen, lower the backoffCur count.
// This requires multiple successful requests of a flaky server, but quickly drops when above the retry limit.
if ch.backoffReset > backoffResetCount || ch.backoffCur > c.retryLimit {
ch.backoffReset = 0
ch.backoffCur--
if ch.backoffCur == 0 {
// reset the last time to the zero value
ch.backoffLast = time.Time{}
}
}
}
}
// getHost looks up or creates a clientHost for a given registry.
func (c *Client) getHost(host string) *clientHost {
c.mu.Lock()
defer c.mu.Unlock()
if h, ok := c.host[host]; ok {
return h
}
var conf *config.Host
if c.getConfigHost != nil {
conf = c.getConfigHost(host)
} else {
conf = config.HostNewName(host)
}
if conf.Name != host {
if h, ok := c.host[conf.Name]; ok {
return h
}
}
h := &clientHost{
config: conf,
userAgent: c.userAgent,
slog: c.slog,
auth: map[string]*auth.Auth{},
}
if h.config.ReqPerSec > 0 {
h.reqFreq = time.Duration(float64(time.Second) / h.config.ReqPerSec)
}
if h.config.ReqConcurrent > 0 {
h.throttle = pqueue.New(pqueue.Opts[reqmeta.Data]{Max: int(h.config.ReqConcurrent), Next: reqmeta.DataNext})
}
// copy the http client and configure registry specific settings
hc := *c.httpClient
h.httpClient = &hc
if h.httpClient.Transport == nil {
h.httpClient.Transport = http.DefaultTransport.(*http.Transport).Clone()
}
// configure transport for insecure requests and root certs
if h.config.TLS == config.TLSInsecure || len(c.rootCAPool) > 0 || len(c.rootCADirs) > 0 || h.config.RegCert != "" || (h.config.ClientCert != "" && h.config.ClientKey != "") {
t, ok := h.httpClient.Transport.(*http.Transport)
if ok {
var tlsc *tls.Config
if t.TLSClientConfig != nil {
tlsc = t.TLSClientConfig.Clone()
} else {
//#nosec G402 the default TLS 1.2 minimum version is allowed to support older registries
tlsc = &tls.Config{}
}
if h.config.TLS == config.TLSInsecure {
tlsc.InsecureSkipVerify = true
} else {
rootPool, err := makeRootPool(c.rootCAPool, c.rootCADirs, h.config.Hostname, h.config.RegCert)
if err != nil {
c.slog.Warn("failed to setup CA pool",
slog.String("err", err.Error()))
} else {
tlsc.RootCAs = rootPool
}
}
if h.config.ClientCert != "" && h.config.ClientKey != "" {
cert, err := tls.X509KeyPair([]byte(h.config.ClientCert), []byte(h.config.ClientKey))
if err != nil {
c.slog.Warn("failed to configure client certs",
slog.String("err", err.Error()))
} else {
tlsc.Certificates = []tls.Certificate{cert}
}
}
t.TLSClientConfig = tlsc
h.httpClient.Transport = t
}
}
// wrap the transport for logging and to handle warning headers
h.httpClient.Transport = &wrapTransport{c: c, orig: h.httpClient.Transport}
c.host[conf.Name] = h
if conf.Name != host {
// save another reference for faster lookups
c.host[host] = h
}
return h
}
// getHTTPClient returns a client specific to the repo being queried.
// Repository specific authentication needs a dedicated CheckRedirect handler.
func (ch *clientHost) getHTTPClient(repo string) *http.Client {
hc := *ch.httpClient
hc.CheckRedirect = ch.checkRedirect(repo, hc.CheckRedirect)
return &hc
}
// checkRedirect wraps http.CheckRedirect to inject auth headers to specific hosts in the redirect chain
func (ch *clientHost) checkRedirect(repo string, orig func(req *http.Request, via []*http.Request) error) func(req *http.Request, via []*http.Request) error {
return func(req *http.Request, via []*http.Request) error {
// fail on too many redirects
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
// verify redirect is allowed
last := via[len(via)-1]
if err := regnet.AllowRedirect(*last.URL, *req.URL); err != nil {
return err
}
// add auth headers if appropriate for the target host
hAuth := ch.getAuth(repo)
err := hAuth.UpdateRequest(req)
if err != nil {
return err
}
// wrap original redirect check
if orig != nil {
return orig(req, via)
}
return nil
}
}
// getAuth returns an auth, which may be repository specific.
func (ch *clientHost) getAuth(repo string) *auth.Auth {
ch.mu.Lock()
defer ch.mu.Unlock()
if !ch.config.RepoAuth {
repo = "" // without RepoAuth, unset the provided repo
}
if _, ok := ch.auth[repo]; !ok {
ch.auth[repo] = auth.NewAuth(
auth.WithLog(ch.slog),
auth.WithHTTPClient(ch.httpClient),
auth.WithCreds(ch.AuthCreds()),
auth.WithClientID(ch.userAgent),
)
}
return ch.auth[repo]
}
func (ch *clientHost) AuthCreds() func(h string) auth.Cred {
if ch == nil || ch.config == nil {
return auth.DefaultCredsFn
}
return func(h string) auth.Cred {
// only return credentials to challenges from the registry server, not to any redirects
if h == ch.config.Hostname {
hCred := ch.config.GetCred()
return auth.Cred{User: hCred.User, Password: hCred.Password, Token: hCred.Token}
} else {
return auth.Cred{}
}
}
}
type wrapTransport struct {
c *Client
orig http.RoundTripper
}
func (wt *wrapTransport) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := wt.orig.RoundTrip(req)
// copy headers to censor auth field
reqHead := req.Header.Clone()
if reqHead.Get("Authorization") != "" {
reqHead.Set("Authorization", "[censored]")
}
if err != nil {
wt.c.slog.Debug("reg http request",
slog.String("req-method", req.Method),
slog.String("req-url", req.URL.String()),
slog.Any("req-headers", reqHead),
slog.String("err", err.Error()))
} else {
// extract any warnings
for _, wh := range resp.Header.Values("Warning") {
if match := warnRegexp.FindStringSubmatch(wh); len(match) == 2 {
// TODO(bmitch): pass other fields (registry hostname) with structured logging
warning.Handle(req.Context(), wt.c.slog, match[1])
}
}
wt.c.slog.Log(req.Context(), types.LevelTrace, "reg http request",
slog.String("req-method", req.Method),
slog.String("req-url", req.URL.String()),
slog.Any("req-headers", reqHead),
slog.String("resp-status", resp.Status),
slog.Any("resp-headers", resp.Header))
}
return resp, err
}
// HTTPError returns an error based on the status code.
func HTTPError(statusCode int) error {
switch statusCode {
case 401:
return fmt.Errorf("%w [http %d]", errs.ErrHTTPUnauthorized, statusCode)
case 403:
return fmt.Errorf("%w [http %d]", errs.ErrHTTPUnauthorized, statusCode)
case 404:
return fmt.Errorf("%w [http %d]", errs.ErrNotFound, statusCode)
case 429:
return fmt.Errorf("%w [http %d]", errs.ErrHTTPRateLimit, statusCode)
default:
return fmt.Errorf("%w: %s [http %d]", errs.ErrHTTPStatus, http.StatusText(statusCode), statusCode)
}
}
func makeRootPool(rootCAPool [][]byte, rootCADirs []string, hostname string, hostcert string) (*x509.CertPool, error) {
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
for _, ca := range rootCAPool {
if ok := pool.AppendCertsFromPEM(ca); !ok {
return nil, fmt.Errorf("failed to load ca: %s", ca)
}
}
for _, dir := range rootCADirs {
hostDir := filepath.Join(dir, hostname)
files, err := os.ReadDir(hostDir)
if err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to read directory %s: %w", hostDir, err)
}
continue
}
for _, f := range files {
if f.IsDir() {
continue
}
if strings.HasSuffix(f.Name(), ".crt") {
f := filepath.Join(hostDir, f.Name())
//#nosec G304 file from a known directory and extension read by the user running the command on their own host
cert, err := os.ReadFile(f)
if err != nil {
return nil, fmt.Errorf("failed to read %s: %w", f, err)
}
if ok := pool.AppendCertsFromPEM(cert); !ok {
return nil, fmt.Errorf("failed to import cert from %s", f)
}
}
}
}
if hostcert != "" {
if ok := pool.AppendCertsFromPEM([]byte(hostcert)); !ok {
// try to parse the certificate and generate a useful error
block, _ := pem.Decode([]byte(hostcert))
if block == nil {
err = fmt.Errorf("pem.Decode is nil")
} else {
_, err = x509.ParseCertificate(block.Bytes)
}
return nil, fmt.Errorf("failed to load host specific ca (registry: %s): %w: %s", hostname, err, hostcert)
}
}
return pool, nil
}
// sortHostCmp to sort host list of mirrors.
func sortHostsCmp(hosts []*clientHost, upstream string) func(i, j int) bool {
now := time.Now()
// sort by backoff first, then priority decending, then upstream name last
return func(i, j int) bool {
if now.Before(hosts[i].backoffLast) || now.Before(hosts[j].backoffLast) {
return hosts[i].backoffLast.Before(hosts[j].backoffLast)
}
if hosts[i].config.Priority != hosts[j].config.Priority {
return hosts[i].config.Priority < hosts[j].config.Priority
}
return hosts[i].config.Name != upstream
}
}
+48
View File
@@ -0,0 +1,48 @@
// Package regnet contains networking helper functions for interacting with registries.
package regnet
import (
"fmt"
"net"
"net/url"
"github.com/regclient/regclient/types/errs"
)
func AllowRedirect(src, dest url.URL) error {
if src.Scheme == "https" && dest.Scheme != "https" {
return fmt.Errorf("redirect from an https to non-https server is not allowed (%s)%.0w", dest.String(), errs.ErrHTTPRedirectRefused)
}
if !IsLocal(src.Host) && IsLocal(dest.Host) {
return fmt.Errorf("redirect to a local domain is not allowed (%s)%.0w", dest.String(), errs.ErrHTTPRedirectRefused)
}
return nil
}
func IsLocal(hostPort string) bool {
// strip trailing port
host, _, err := net.SplitHostPort(hostPort)
if err != nil {
host = hostPort
}
// parse IP
ip := net.ParseIP(host)
if ip != nil {
return isIPLocal(ip)
}
// else resolve the hostname and then check each IP
ips, err := net.LookupIP(host)
if err != nil {
return false
}
for _, ip := range ips {
if ip != nil && isIPLocal(ip) {
return true
}
}
return false
}
func isIPLocal(ip net.IP) bool {
return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
}
@@ -0,0 +1,3 @@
// Package reproducible is used for reproducibility.
// See <https://reproducible-builds.org/docs/> for more information of the issues being solved.
package reproducible
+35
View File
@@ -0,0 +1,35 @@
package reproducible
import (
"errors"
"os"
"strconv"
"time"
)
const EpocEnv = "SOURCE_DATE_EPOC"
var errInvalidEpoc = errors.New("invalid epoc var")
// TimeNow returns the current time or SOURCE_DATE_EPOC if that is set.
func TimeNow() time.Time {
now, err := TimeEpocEnv()
if err == nil {
return now
}
return time.Now().UTC()
}
// TimeEpocEnv returns the time parsed by SOURCE_DATE_EPOC.
// This should be used to override any timestamps that should be reproducible.
func TimeEpocEnv() (time.Time, error) {
sec := os.Getenv(EpocEnv)
if sec == "" {
return time.Time{}, errInvalidEpoc
}
secI, err := strconv.ParseInt(sec, 10, 64)
if err != nil {
return time.Time{}, err
}
return time.Unix(secI, 0).UTC(), nil
}
+88
View File
@@ -0,0 +1,88 @@
// Package reqmeta provides metadata on requests for prioritizing with a pqueue.
package reqmeta
type Data struct {
Kind Kind
Size int64
}
type Kind int
const (
Unknown Kind = iota
Head
Manifest
Query
Blob
)
const (
smallLimit = 4194304 // 4MiB
largePct = 0.9 // anything above 90% of largest queued entry size is large
)
func DataNext(queued, active []*Data) int {
if len(queued) == 0 {
return -1
}
// After removing one small entry, split remaining requests 50/50 between large and old (truncated int division always rounds down).
// If len active = 2, this function returns the 3rd entry (+1), minus 1 for the small, divide by 2 to split with old = goal of 1.
largeGoal := len(active) / 2
largeI := 0
var largeSize int64
if largeGoal > 0 {
// find the largest queued blob requests
for i, cur := range queued {
if cur.Kind == Blob && cur.Size > largeSize {
largeI = i
largeSize = cur.Size
}
}
}
largeCutoff := int64(float64(largeSize) * 0.9)
// count active requests by type
small := 0
large := 0
old := 0
for _, cur := range active {
if cur.Kind != Blob && cur.Size <= smallLimit {
small++
} else if cur.Kind == Blob && largeSize > 0 && cur.Size >= largeCutoff {
large++
} else {
old++
}
}
// if there is at least one active, and none are small, return the best small entry if available.
if len(active) > 0 && small == 0 {
var sizeI int64
bestI := -1
kindI := Unknown
for i, cur := range queued {
// the small search skips blobs and large requests
if cur.Kind == Blob || cur.Size > smallLimit {
continue
}
// the best small entry is the:
// - first one found if no other matches
// - one with a better Kind (Head > Manifest > Query)
// - one with the same kind but smaller request
if bestI < 0 ||
(cur.Kind != Unknown && (kindI == Unknown || cur.Kind < kindI)) ||
(cur.Kind == kindI && cur.Size > 0 && (cur.Size < sizeI || sizeI <= 0)) {
bestI = i
kindI = cur.Kind
sizeI = cur.Size
}
}
if bestI >= 0 {
return bestI
}
}
// Prefer the biggest of these blobs to minimize the size of the last running blob.
if largeGoal > 0 && large < largeGoal && largeSize > 0 {
return largeI
}
// enough small and large, or none available, so return the oldest queued entry to avoid starvation.
return 0
}
+125
View File
@@ -0,0 +1,125 @@
//go:build !wasm
// Package sloghandle provides a transition handler for migrating from logrus to slog.
package sloghandle
import (
"context"
"log/slog"
"strings"
"github.com/sirupsen/logrus"
"github.com/regclient/regclient/types"
)
func Logrus(logger *logrus.Logger) *logrusHandler {
return &logrusHandler{
logger: logger,
}
}
type logrusHandler struct {
logger *logrus.Logger
attrs []slog.Attr
groups []string
}
func (h *logrusHandler) Enabled(_ context.Context, level slog.Level) bool {
ll := h.logger.GetLevel()
if curLevel, ok := logrusToSlog[ll]; ok {
return level >= curLevel
}
return true
}
func (h *logrusHandler) Handle(ctx context.Context, r slog.Record) error {
log := logrus.NewEntry(h.logger).WithContext(ctx)
if !r.Time.IsZero() {
log = log.WithTime(r.Time)
}
fields := logrus.Fields{}
for _, a := range h.attrs {
if a.Key != "" {
fields[a.Key] = a.Value
}
}
r.Attrs(func(a slog.Attr) bool {
if a.Key != "" {
fields[a.Key] = a.Value
}
return true
})
if len(fields) > 0 {
log = log.WithFields(fields)
}
log.Log(slogToLogrus(r.Level), r.Message)
return nil
}
func (h *logrusHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
ret := h.clone()
prefix := ""
if len(h.groups) > 0 {
prefix = strings.Join(h.groups, ":") + ":"
}
for _, a := range attrs {
if a.Key == "" {
continue
}
ret.attrs = append(ret.attrs, slog.Attr{
Key: prefix + a.Key,
Value: a.Value,
})
}
return ret
}
func (h *logrusHandler) WithGroup(name string) slog.Handler {
if name == "" {
return h
}
ret := h.clone()
ret.groups = append(ret.groups, name)
return ret
}
func (h *logrusHandler) clone() *logrusHandler {
attrs := make([]slog.Attr, len(h.attrs))
copy(attrs, h.attrs)
groups := make([]string, len(h.groups))
copy(groups, h.groups)
return &logrusHandler{
logger: h.logger,
attrs: attrs,
groups: groups,
}
}
var logrusToSlog = map[logrus.Level]slog.Level{
logrus.TraceLevel: types.LevelTrace,
logrus.DebugLevel: slog.LevelDebug,
logrus.InfoLevel: slog.LevelInfo,
logrus.WarnLevel: slog.LevelWarn,
logrus.ErrorLevel: slog.LevelError,
logrus.FatalLevel: slog.LevelError + 4,
logrus.PanicLevel: slog.LevelError + 8,
}
func slogToLogrus(level slog.Level) logrus.Level {
if level <= types.LevelTrace {
return logrus.TraceLevel
} else if level <= slog.LevelDebug {
return logrus.DebugLevel
} else if level <= slog.LevelInfo {
return logrus.InfoLevel
} else if level <= slog.LevelWarn {
return logrus.WarnLevel
} else if level <= slog.LevelError {
return logrus.ErrorLevel
} else if level <= slog.LevelError+4 {
return logrus.FatalLevel
} else {
return logrus.PanicLevel
}
}
+91
View File
@@ -0,0 +1,91 @@
// Package strparse is used to parse strings
package strparse
import (
"fmt"
"github.com/regclient/regclient/types/errs"
)
// SplitCSKV splits a comma separated key=value list into a map
func SplitCSKV(s string) (map[string]string, error) {
state := "key"
key := ""
val := ""
result := map[string]string{}
procKV := func() {
if key != "" {
result[key] = val
}
state = "key"
key = ""
val = ""
}
for _, c := range s {
switch state {
case "key":
switch c {
case '"':
state = "keyQuote"
case '\\':
state = "keyEscape"
case '=':
state = "val"
case ',':
procKV()
default:
key = key + string(c)
}
case "keyQuote":
switch c {
case '"':
state = "key"
case '\\':
state = "keyEscapeQuote"
default:
key = key + string(c)
}
case "keyEscape":
key = key + string(c)
state = "key"
case "keyEscapeQuote":
key = key + string(c)
state = "keyQuote"
case "val":
switch c {
case '"':
state = "valQuote"
case ',':
procKV()
case '\\':
state = "valEscape"
default:
val = val + string(c)
}
case "valQuote":
switch c {
case '"':
state = "val"
case '\\':
state = "valEscapeQuote"
default:
val = val + string(c)
}
case "valEscape":
val = val + string(c)
state = "val"
case "valEscapeQuote":
val = val + string(c)
state = "valQuote"
default:
return nil, fmt.Errorf("unhandled state: %s", state)
}
}
switch state {
case "val", "key":
procKV()
default:
return nil, fmt.Errorf("string parsing failed, end state: %s%.0w", state, errs.ErrParsingFailed)
}
return result, nil
}
+41
View File
@@ -0,0 +1,41 @@
// Package timejson extends time methods with marshal/unmarshal for json
package timejson
import (
"encoding/json"
"errors"
"time"
)
var errInvalid = errors.New("invalid duration")
// Duration is an alias to time.Duration
// Implementation taken from https://stackoverflow.com/questions/48050945/how-to-unmarshal-json-into-durations
type Duration time.Duration
// MarshalJSON converts a duration to json
func (d Duration) MarshalJSON() ([]byte, error) {
return json.Marshal(time.Duration(d).String())
}
// UnmarshalJSON converts json to a duration
func (d *Duration) UnmarshalJSON(b []byte) error {
var v any
if err := json.Unmarshal(b, &v); err != nil {
return err
}
switch value := v.(type) {
case float64:
*d = Duration(time.Duration(value))
return nil
case string:
timeDur, err := time.ParseDuration(value)
if err != nil {
return err
}
*d = Duration(timeDur)
return nil
default:
return errInvalid
}
}
+59
View File
@@ -0,0 +1,59 @@
// Package units is taken from https://github.com/docker/go-units
package units
// Copyright 2015 Docker, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// https://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import (
"fmt"
)
var (
decimapAbbrs = []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
binaryAbbrs = []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"}
)
func getSizeAndUnit(size float64, base float64, unitList []string) (float64, string) {
i := 0
unitsLimit := len(unitList) - 1
for size >= base && i < unitsLimit {
size = size / base
i++
}
return size, unitList[i]
}
// CustomSize returns a human-readable approximation of a size using custom format.
func CustomSize(format string, size float64, base float64, unitList []string) string {
size, unit := getSizeAndUnit(size, base, unitList)
return fmt.Sprintf(format, size, unit)
}
// HumanSizeWithPrecision allows the size to be in any precision.
func HumanSizeWithPrecision(size float64, width, precision int) string {
size, unit := getSizeAndUnit(size, 1000.0, decimapAbbrs)
return fmt.Sprintf("%*.*f%s", width, precision, size, unit)
}
// HumanSize returns a human-readable approximation of a size
// with a width of 5 (eg. "2.746MB", "796.0KB").
func HumanSize(size float64) string {
return HumanSizeWithPrecision(size, 5, 3)
}
// BytesSize returns a human-readable size in bytes, kibibytes,
// mebibytes, gibibytes, or tebibytes (eg. "44.2kiB", "17.6MiB").
func BytesSize(size float64) string {
return CustomSize("%5.3f%s", size, 1024.0, binaryAbbrs)
}
+32
View File
@@ -0,0 +1,32 @@
// Package version returns details on the Go and Git repo used in the build
package version
import (
"bytes"
"fmt"
"text/tabwriter"
)
const (
stateClean = "clean"
stateDirty = "dirty"
unknown = "unknown"
biVCSDate = "vcs.time"
biVCSCommit = "vcs.revision"
biVCSModified = "vcs.modified"
)
func (i Info) MarshalPretty() ([]byte, error) {
buf := &bytes.Buffer{}
tw := tabwriter.NewWriter(buf, 0, 0, 1, ' ', 0)
fmt.Fprintf(tw, "VCSTag:\t%s\n", i.VCSTag)
fmt.Fprintf(tw, "VCSRef:\t%s\n", i.VCSRef)
fmt.Fprintf(tw, "VCSCommit:\t%s\n", i.VCSCommit)
fmt.Fprintf(tw, "VCSState:\t%s\n", i.VCSState)
fmt.Fprintf(tw, "VCSDate:\t%s\n", i.VCSDate)
fmt.Fprintf(tw, "Platform:\t%s\n", i.Platform)
fmt.Fprintf(tw, "GoVer:\t%s\n", i.GoVer)
fmt.Fprintf(tw, "GoCompiler:\t%s\n", i.GoCompiler)
err := tw.Flush()
return buf.Bytes(), err
}
@@ -0,0 +1,74 @@
//go:build go1.18
package version
import (
"fmt"
"runtime"
"runtime/debug"
"time"
)
var vcsTag = ""
type Info struct {
GoVer string `json:"goVersion"` // go version
GoCompiler string `json:"goCompiler"` // go compiler
Platform string `json:"platform"` // os/arch
VCSCommit string `json:"vcsCommit"` // commit sha
VCSDate string `json:"vcsDate"` // commit date in RFC3339 format
VCSRef string `json:"vcsRef"` // commit sha + dirty if state is not clean
VCSState string `json:"vcsState"` // clean or dirty
VCSTag string `json:"vcsTag"` // tag is not available from Go
Debug *debug.BuildInfo `json:"debug,omitempty"` // build info debugging data
}
func GetInfo() Info {
i := Info{
GoVer: unknown,
Platform: unknown,
VCSCommit: unknown,
VCSDate: unknown,
VCSRef: unknown,
VCSState: unknown,
VCSTag: vcsTag,
}
i.GoVer = runtime.Version()
i.GoCompiler = runtime.Compiler
i.Platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH)
if bi, ok := debug.ReadBuildInfo(); ok && bi != nil {
i.Debug = bi
if i.VCSTag == "" {
i.VCSTag = bi.Main.Version
}
date := biSetting(bi, biVCSDate)
if t, err := time.Parse(time.RFC3339, date); err == nil {
i.VCSDate = t.UTC().Format(time.RFC3339)
}
i.VCSCommit = biSetting(bi, biVCSCommit)
i.VCSRef = i.VCSCommit
modified := biSetting(bi, biVCSModified)
if modified == "true" {
i.VCSState = stateDirty
i.VCSRef += "-" + stateDirty
} else if modified == "false" {
i.VCSState = stateClean
}
}
return i
}
func biSetting(bi *debug.BuildInfo, key string) string {
if bi == nil {
return unknown
}
for _, setting := range bi.Settings {
if setting.Key == key {
return setting.Value
}
}
return unknown
}
+37
View File
@@ -0,0 +1,37 @@
//go:build !go1.18
package version
import (
"fmt"
"runtime"
)
type Info struct {
GoVer string `json:"goVersion"` // go version
GoCompiler string `json:"goCompiler"` // go compiler
Platform string `json:"platform"` // os/arch
VCSCommit string `json:"vcsCommit"` // commit sha
VCSDate string `json:"vcsDate"` // commit date in RFC3339 format
VCSRef string `json:"vcsRef"` // commit sha + dirty if state is not clean
VCSState string `json:"vcsState"` // clean or dirty
VCSTag string `json:"vcsTag"` // tag
}
func GetInfo() Info {
i := Info{
GoVer: unknown,
Platform: unknown,
VCSCommit: unknown,
VCSDate: unknown,
VCSRef: unknown,
VCSState: unknown,
VCSTag: "",
}
i.GoVer = runtime.Version()
i.GoCompiler = runtime.Compiler
i.Platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH)
return i
}
+206
View File
@@ -0,0 +1,206 @@
package regclient
import (
"context"
"fmt"
"log/slog"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/platform"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/warning"
)
type manifestOpt struct {
d descriptor.Descriptor
platform *platform.Platform
schemeOpts []scheme.ManifestOpts
requireDigest bool
}
// ManifestOpts define options for the Manifest* commands.
type ManifestOpts func(*manifestOpt)
// WithManifest passes a manifest to ManifestDelete.
func WithManifest(m manifest.Manifest) ManifestOpts {
return func(opts *manifestOpt) {
opts.schemeOpts = append(opts.schemeOpts, scheme.WithManifest(m))
}
}
// WithManifestCheckReferrers checks for referrers field on ManifestDelete.
// This will update the client managed referrer listing.
func WithManifestCheckReferrers() ManifestOpts {
return func(opts *manifestOpt) {
opts.schemeOpts = append(opts.schemeOpts, scheme.WithManifestCheckReferrers())
}
}
// WithManifestChild for ManifestPut indicates the manifest is not the top level manifest being copied.
// This is used by the ocidir scheme to determine what entries to include in the index.json.
func WithManifestChild() ManifestOpts {
return func(opts *manifestOpt) {
opts.schemeOpts = append(opts.schemeOpts, scheme.WithManifestChild())
}
}
// WithManifestDesc includes the descriptor for ManifestGet.
// This is used to automatically extract a Data field if available.
func WithManifestDesc(d descriptor.Descriptor) ManifestOpts {
return func(opts *manifestOpt) {
opts.d = d
}
}
// WithManifestPlatform resolves the platform specific manifest on Get and Head requests.
// This causes an additional GET query to a registry when an Index or Manifest List is encountered.
// This option is ignored if the retrieved manifest is not an Index or Manifest List.
func WithManifestPlatform(p platform.Platform) ManifestOpts {
return func(opts *manifestOpt) {
opts.platform = &p
}
}
// WithManifestRequireDigest falls back from a HEAD to a GET request when digest headers aren't received.
func WithManifestRequireDigest() ManifestOpts {
return func(opts *manifestOpt) {
opts.requireDigest = true
}
}
// ManifestDelete removes a manifest, including all tags pointing to that registry.
// The reference must include the digest to delete (see TagDelete for deleting a tag).
// All tags pointing to the manifest will be deleted.
func (rc *RegClient) ManifestDelete(ctx context.Context, r ref.Ref, opts ...ManifestOpts) error {
if !r.IsSet() {
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
for _, fn := range opts {
fn(&opt)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return err
}
return schemeAPI.ManifestDelete(ctx, r, opt.schemeOpts...)
}
// ManifestGet retrieves a manifest.
func (rc *RegClient) ManifestGet(ctx context.Context, r ref.Ref, opts ...ManifestOpts) (manifest.Manifest, error) {
if !r.IsSet() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
for _, fn := range opts {
fn(&opt)
}
if opt.d.Digest != "" {
r = r.AddDigest(opt.d.Digest.String())
data, err := opt.d.GetData()
if err == nil {
return manifest.New(
manifest.WithDesc(opt.d),
manifest.WithRaw(data),
manifest.WithRef(r),
)
}
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
m, err := schemeAPI.ManifestGet(ctx, r)
if err != nil {
return m, err
}
if opt.platform != nil && !m.IsList() {
rc.slog.Debug("ignoring platform option, image is not an index",
slog.String("platform", opt.platform.String()),
slog.String("ref", r.CommonName()))
}
// this will loop to handle a nested index
for opt.platform != nil && m.IsList() {
d, err := manifest.GetPlatformDesc(m, opt.platform)
if err != nil {
return m, err
}
r = r.SetDigest(d.Digest.String())
m, err = schemeAPI.ManifestGet(ctx, r)
if err != nil {
return m, err
}
}
return m, err
}
// ManifestHead queries for the existence of a manifest and returns metadata (digest, media-type, size).
func (rc *RegClient) ManifestHead(ctx context.Context, r ref.Ref, opts ...ManifestOpts) (manifest.Manifest, error) {
if !r.IsSet() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
for _, fn := range opts {
fn(&opt)
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
m, err := schemeAPI.ManifestHead(ctx, r)
if err != nil {
return m, err
}
if opt.platform != nil && !m.IsList() {
rc.slog.Debug("ignoring platform option, image is not an index",
slog.String("platform", opt.platform.String()),
slog.String("ref", r.CommonName()))
}
// this will loop to handle a nested index
for opt.platform != nil && m.IsList() {
if !m.IsSet() {
m, err = schemeAPI.ManifestGet(ctx, r)
}
d, err := manifest.GetPlatformDesc(m, opt.platform)
if err != nil {
return m, err
}
r = r.SetDigest(d.Digest.String())
m, err = schemeAPI.ManifestHead(ctx, r)
if err != nil {
return m, err
}
}
if opt.requireDigest && m.GetDescriptor().Digest.String() == "" {
m, err = schemeAPI.ManifestGet(ctx, r)
}
return m, err
}
// ManifestPut pushes a manifest.
// Any descriptors referenced by the manifest typically need to be pushed first.
func (rc *RegClient) ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...ManifestOpts) error {
if !r.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
for _, fn := range opts {
fn(&opt)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return err
}
return schemeAPI.ManifestPut(ctx, r, m, opt.schemeOpts...)
}
+18
View File
@@ -0,0 +1,18 @@
package regclient
import (
"context"
"github.com/regclient/regclient/types/ping"
"github.com/regclient/regclient/types/ref"
)
// Ping verifies access to a registry or equivalent.
func (rc *RegClient) Ping(ctx context.Context, r ref.Ref) (ping.Result, error) {
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return ping.Result{}, err
}
return schemeAPI.Ping(ctx, r)
}
+2
View File
@@ -0,0 +1,2 @@
// Package archive is used to read and write tar files
package archive
+160
View File
@@ -0,0 +1,160 @@
package archive
import (
"bufio"
"bytes"
"compress/bzip2"
"compress/gzip"
"errors"
"fmt"
"io"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
)
// CompressType identifies the detected compression type
type CompressType int
const (
CompressNone CompressType = iota // uncompressed or unable to detect compression
CompressBzip2 // bzip2
CompressGzip // gzip
CompressXz // xz
CompressZstd // zstd
)
// compressHeaders are used to detect the compression type
var compressHeaders = map[CompressType][]byte{
CompressBzip2: []byte("\x42\x5A\x68"),
CompressGzip: []byte("\x1F\x8B\x08"),
CompressXz: []byte("\xFD\x37\x7A\x58\x5A\x00"),
CompressZstd: []byte("\x28\xB5\x2F\xFD"),
}
func Compress(r io.Reader, oComp CompressType) (io.ReadCloser, error) {
switch oComp {
// note, bzip2 compression is not supported
case CompressGzip:
return writeToRead(r, newGzipWriter)
case CompressXz:
return writeToRead(r, xz.NewWriter)
case CompressZstd:
return writeToRead(r, newZstdWriter)
case CompressNone:
return io.NopCloser(r), nil
default:
return nil, ErrUnknownType
}
}
// newGzipWriter generates a writer and an always nil error.
func newGzipWriter(w io.Writer) (io.WriteCloser, error) {
return gzip.NewWriter(w), nil
}
// newZstdWriter generates a writer with the default options.
func newZstdWriter(w io.Writer) (io.WriteCloser, error) {
return zstd.NewWriter(w)
}
// writeToRead uses a pipe + goroutine + copy to switch from a writer to a reader.
func writeToRead[wc io.WriteCloser](src io.Reader, newWriterFn func(io.Writer) (wc, error)) (io.ReadCloser, error) {
pr, pw := io.Pipe()
go func() {
// buffer output to avoid lots of small reads
bw := bufio.NewWriterSize(pw, 2<<16)
dest, err := newWriterFn(bw)
if err != nil {
_ = pw.CloseWithError(err)
return
}
if _, err := io.Copy(dest, src); err != nil {
_ = pw.CloseWithError(err)
}
if err := dest.Close(); err != nil {
_ = pw.CloseWithError(err)
}
if err := bw.Flush(); err != nil {
_ = pw.CloseWithError(err)
}
_ = pw.Close()
}()
return pr, nil
}
// Decompress extracts gzip and bzip streams
func Decompress(r io.Reader) (io.Reader, error) {
// create bufio to peak on first few bytes
br := bufio.NewReader(r)
head, err := br.Peek(10)
if err != nil && !errors.Is(err, io.EOF) {
return br, fmt.Errorf("failed to detect compression: %w", err)
}
// compare peaked data against known compression types
switch DetectCompression(head) {
case CompressBzip2:
return bzip2.NewReader(br), nil
case CompressGzip:
return gzip.NewReader(br)
case CompressXz:
return xz.NewReader(br)
case CompressZstd:
return zstd.NewReader(br)
default:
return br, nil
}
}
// DetectCompression identifies the compression type based on the first few bytes
func DetectCompression(head []byte) CompressType {
for c, b := range compressHeaders {
if bytes.HasPrefix(head, b) {
return c
}
}
return CompressNone
}
func (ct CompressType) String() string {
mt, err := ct.MarshalText()
if err != nil {
return "unknown"
}
return string(mt)
}
func (ct CompressType) MarshalText() ([]byte, error) {
switch ct {
case CompressNone:
return []byte("none"), nil
case CompressBzip2:
return []byte("bzip2"), nil
case CompressGzip:
return []byte("gzip"), nil
case CompressXz:
return []byte("xz"), nil
case CompressZstd:
return []byte("zstd"), nil
}
return nil, fmt.Errorf("unknown compression type")
}
func (ct *CompressType) UnmarshalText(text []byte) error {
switch string(text) {
case "none":
*ct = CompressNone
case "bzip2":
*ct = CompressBzip2
case "gzip":
*ct = CompressGzip
case "xz":
*ct = CompressXz
case "zstd":
*ct = CompressZstd
default:
return fmt.Errorf("unknown compression type %s", string(text))
}
return nil
}
+13
View File
@@ -0,0 +1,13 @@
package archive
import "errors"
var (
// ErrNotImplemented used for routines that need to be developed still
ErrNotImplemented = errors.New("this archive routine is not implemented yet")
// ErrUnknownType used for unknown compression types
ErrUnknownType = errors.New("unknown compression type")
// ErrXzUnsupported because there isn't a Go package for this and I'm
// avoiding dependencies on external binaries
ErrXzUnsupported = errors.New("xz compression is currently unsupported")
)
+170
View File
@@ -0,0 +1,170 @@
package archive
import (
"archive/tar"
"compress/gzip"
"context"
"fmt"
"io"
"io/fs"
"math"
"os"
"path/filepath"
"time"
)
// TarOpts configures options for Create/Extract tar
type TarOpts func(*tarOpts)
// TODO: add support for compressed files with bzip
type tarOpts struct {
// allowRelative bool // allow relative paths outside of target folder
compress string
}
// TarCompressGzip option to use gzip compression on tar files
func TarCompressGzip(to *tarOpts) {
to.compress = "gzip"
}
// TarUncompressed option to tar (noop)
func TarUncompressed(to *tarOpts) {
}
// TODO: add option for full path or to adjust the relative path
// Tar creation
func Tar(ctx context.Context, path string, w io.Writer, opts ...TarOpts) error {
to := tarOpts{}
for _, opt := range opts {
opt(&to)
}
twOut := w
if to.compress == "gzip" {
gw := gzip.NewWriter(w)
defer gw.Close()
twOut = gw
}
tw := tar.NewWriter(twOut)
defer tw.Close()
// walk the path performing a recursive tar
err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error {
// return any errors filepath encounters accessing the file
if err != nil {
return err
}
// TODO: handle symlinks, security attributes, hard links
// TODO: add options for file owner and timestamps
// TODO: add options to override time, or disable access/change stamps
// adjust for relative path
relPath, err := filepath.Rel(path, file)
if err != nil || relPath == "." {
return nil
}
header, err := tar.FileInfoHeader(fi, relPath)
if err != nil {
return err
}
header.Format = tar.FormatPAX
header.Name = filepath.ToSlash(relPath)
header.AccessTime = time.Time{}
header.ChangeTime = time.Time{}
header.ModTime = header.ModTime.Truncate(time.Second)
if err = tw.WriteHeader(header); err != nil {
return err
}
// open file and copy contents into tar writer
if header.Typeflag == tar.TypeReg && header.Size > 0 {
//#nosec G122 G304 filename is limited to provided path directory
f, err := os.Open(file)
if err != nil {
return err
}
if _, err = io.Copy(tw, f); err != nil {
return err
}
err = f.Close()
if err != nil {
return fmt.Errorf("failed to close file: %w", err)
}
}
return nil
})
return err
}
// Extract Tar
func Extract(ctx context.Context, path string, r io.Reader, opts ...TarOpts) error {
to := tarOpts{}
for _, opt := range opts {
opt(&to)
}
// verify path exists
fi, err := os.Stat(path)
if err != nil {
return err
}
if !fi.IsDir() {
return fmt.Errorf("extract path must be a directory: \"%s\"", path)
}
// decompress
rd, err := Decompress(r)
if err != nil {
return err
}
rt := tar.NewReader(rd)
for {
hdr, err := rt.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
// join a cleaned version of the filename with the path
fn := filepath.Join(path, filepath.Clean("/"+hdr.Name))
switch hdr.Typeflag {
case tar.TypeDir:
if hdr.Mode < 0 || hdr.Mode > math.MaxUint32 {
return fmt.Errorf("integer conversion overflow/underflow (file mode = %d)", hdr.Mode)
}
err = os.MkdirAll(fn, fs.FileMode(hdr.Mode))
if err != nil {
return err
}
case tar.TypeReg:
// TODO: configure file mode, creation timestamp, etc
//#nosec G304 filename is limited to provided path directory
fh, err := os.Create(fn)
if err != nil {
return err
}
n, err := io.CopyN(fh, rt, hdr.Size)
errC := fh.Close()
if err != nil {
return err
}
if errC != nil {
return fmt.Errorf("failed to close file: %w", errC)
}
if n != hdr.Size {
return fmt.Errorf("size mismatch extracting \"%s\", expected %d, extracted %d", hdr.Name, hdr.Size, n)
}
// TODO: handle other tar types (symlinks, etc)
}
}
return nil
}
+57
View File
@@ -0,0 +1,57 @@
package regclient
import (
"context"
"fmt"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/platform"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/referrer"
"github.com/regclient/regclient/types/warning"
)
// ReferrerList retrieves a list of referrers to a manifest.
// The descriptor list should contain manifests that each have a subject field matching the requested ref.
func (rc *RegClient) ReferrerList(ctx context.Context, rSubject ref.Ref, opts ...scheme.ReferrerOpts) (referrer.ReferrerList, error) {
if !rSubject.IsSet() {
return referrer.ReferrerList{}, fmt.Errorf("ref is not set: %s%.0w", rSubject.CommonName(), errs.ErrInvalidReference)
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
// set the digest on the subject reference
config := scheme.ReferrerConfig{}
for _, opt := range opts {
opt(&config)
}
if rSubject.Digest == "" || config.Platform != "" {
mo := []ManifestOpts{WithManifestRequireDigest()}
if config.Platform != "" {
p, err := platform.Parse(config.Platform)
if err != nil {
return referrer.ReferrerList{}, fmt.Errorf("failed to lookup referrer platform: %w", err)
}
mo = append(mo, WithManifestPlatform(p))
}
m, err := rc.ManifestHead(ctx, rSubject, mo...)
if err != nil {
return referrer.ReferrerList{}, fmt.Errorf("failed to get digest for subject: %w", err)
}
rSubject = rSubject.SetDigest(m.GetDescriptor().Digest.String())
}
// lookup the scheme for the appropriate ref
var r ref.Ref
if config.SrcRepo.IsSet() {
r = config.SrcRepo
} else {
r = rSubject
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return referrer.ReferrerList{}, err
}
return schemeAPI.ReferrerList(ctx, rSubject, opts...)
}
+275
View File
@@ -0,0 +1,275 @@
// Package regclient is used to access OCI registries.
package regclient
import (
"fmt"
"io"
"log/slog"
"time"
"github.com/regclient/regclient/config"
"github.com/regclient/regclient/internal/version"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/scheme/ocidir"
"github.com/regclient/regclient/scheme/reg"
)
const (
// DefaultUserAgent sets the header on http requests.
DefaultUserAgent = "regclient/regclient"
// DockerCertDir default location for docker certs.
DockerCertDir = "/etc/docker/certs.d"
// DockerRegistry is the well known name of Docker Hub, "docker.io".
DockerRegistry = config.DockerRegistry
// DockerRegistryAuth is the name of Docker Hub seen in docker's config.json.
DockerRegistryAuth = config.DockerRegistryAuth
// DockerRegistryDNS is the actual registry DNS name for Docker Hub.
DockerRegistryDNS = config.DockerRegistryDNS
)
// RegClient is used to access OCI distribution-spec registries.
type RegClient struct {
hosts map[string]*config.Host
hostDefault *config.Host
regOpts []reg.Opts
schemes map[string]scheme.API
slog *slog.Logger
userAgent string
}
// Opt functions are used by [New] to create a [*RegClient].
type Opt func(*RegClient)
// New returns a registry client.
func New(opts ...Opt) *RegClient {
rc := RegClient{
hosts: map[string]*config.Host{},
userAgent: DefaultUserAgent,
regOpts: []reg.Opts{},
schemes: map[string]scheme.API{},
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
}
info := version.GetInfo()
if info.VCSTag != "" {
rc.userAgent = fmt.Sprintf("%s (%s)", rc.userAgent, info.VCSTag)
} else {
rc.userAgent = fmt.Sprintf("%s (%s)", rc.userAgent, info.VCSRef)
}
// inject Docker Hub settings
_ = rc.hostSet(*config.HostNewName(config.DockerRegistryAuth))
for _, opt := range opts {
opt(&rc)
}
// configure regOpts
hostList := []*config.Host{}
for _, h := range rc.hosts {
hostList = append(hostList, h)
}
rc.regOpts = append(
rc.regOpts,
reg.WithConfigHosts(hostList),
reg.WithConfigHostDefault(rc.hostDefault),
reg.WithSlog(rc.slog),
reg.WithUserAgent(rc.userAgent),
)
// setup scheme's
rc.schemes["reg"] = reg.New(rc.regOpts...)
rc.schemes["ocidir"] = ocidir.New(
ocidir.WithSlog(rc.slog),
)
rc.slog.Debug("regclient initialized",
slog.String("VCSRef", info.VCSRef),
slog.String("VCSTag", info.VCSTag))
return &rc
}
// WithBlobLimit sets the max size for chunked blob uploads which get stored in memory.
//
// Deprecated: replace with WithRegOpts(reg.WithBlobLimit(limit)), see [WithRegOpts] and [reg.WithBlobLimit].
//
//go:fix inline
func WithBlobLimit(limit int64) Opt {
return WithRegOpts(reg.WithBlobLimit(limit))
}
// WithBlobSize overrides default blob sizes.
//
// Deprecated: replace with WithRegOpts(reg.WithBlobSize(chunk, max)), see [WithRegOpts] and [reg.WithBlobSize].
//
//go:fix inline
func WithBlobSize(chunk, max int64) Opt {
return WithRegOpts(reg.WithBlobSize(chunk, max))
}
// WithCertDir adds a path of certificates to trust similar to Docker's /etc/docker/certs.d.
//
// Deprecated: replace with WithRegOpts(reg.WithCertDirs(path)), see [WithRegOpts] and [reg.WithCertDirs].
//
//go:fix inline
func WithCertDir(path ...string) Opt {
return WithRegOpts(reg.WithCertDirs(path))
}
// WithConfigHost adds a list of config host settings.
func WithConfigHost(configHost ...config.Host) Opt {
return func(rc *RegClient) {
rc.hostLoad("host", configHost)
}
}
// WithConfigHostDefault adds default settings for new hosts.
func WithConfigHostDefault(configHost config.Host) Opt {
return func(rc *RegClient) {
rc.hostDefault = &configHost
}
}
// WithConfigHosts adds a list of config host settings.
//
// Deprecated: replace with [WithConfigHost].
//
//go:fix inline
func WithConfigHosts(configHosts []config.Host) Opt {
return WithConfigHost(configHosts...)
}
// WithDockerCerts adds certificates trusted by docker in /etc/docker/certs.d.
func WithDockerCerts() Opt {
return WithRegOpts(reg.WithCertDirs([]string{DockerCertDir}))
}
// WithDockerCreds adds configuration from users docker config with registry logins.
// This changes the default value from the config file, and should be added after the config file is loaded.
func WithDockerCreds() Opt {
return func(rc *RegClient) {
configHosts, err := config.DockerLoad()
if err != nil {
rc.slog.Warn("Failed to load docker creds",
slog.String("err", err.Error()))
return
}
rc.hostLoad("docker", configHosts)
}
}
// WithDockerCredsFile adds configuration from a named docker config file with registry logins.
// This changes the default value from the config file, and should be added after the config file is loaded.
func WithDockerCredsFile(fname string) Opt {
return func(rc *RegClient) {
configHosts, err := config.DockerLoadFile(fname)
if err != nil {
rc.slog.Warn("Failed to load docker creds",
slog.String("err", err.Error()))
return
}
rc.hostLoad("docker-file", configHosts)
}
}
// WithRegOpts passes through opts to the reg scheme.
func WithRegOpts(opts ...reg.Opts) Opt {
return func(rc *RegClient) {
if len(opts) == 0 {
return
}
rc.regOpts = append(rc.regOpts, opts...)
}
}
// WithRetryDelay specifies the time permitted for retry delays.
//
// Deprecated: replace with WithRegOpts(reg.WithDelay(delayInit, delayMax)), see [WithRegOpts] and [reg.WithDelay].
//
//go:fix inline
func WithRetryDelay(delayInit, delayMax time.Duration) Opt {
return WithRegOpts(reg.WithDelay(delayInit, delayMax))
}
// WithRetryLimit specifies the number of retries for non-fatal errors.
//
// Deprecated: replace with WithRegOpts(reg.WithRetryLimit(retryLimit)), see [WithRegOpts] and [reg.WithRetryLimit].
//
//go:fix inline
func WithRetryLimit(retryLimit int) Opt {
return WithRegOpts(reg.WithRetryLimit(retryLimit))
}
// WithSlog configures the slog Logger.
func WithSlog(slog *slog.Logger) Opt {
return func(rc *RegClient) {
rc.slog = slog
}
}
// WithUserAgent specifies the User-Agent http header.
func WithUserAgent(ua string) Opt {
return func(rc *RegClient) {
rc.userAgent = ua
}
}
func (rc *RegClient) hostLoad(src string, hosts []config.Host) {
for _, configHost := range hosts {
if configHost.Name == "" {
if configHost.Pass != "" {
configHost.Pass = "***"
}
if configHost.Token != "" {
configHost.Token = "***"
}
rc.slog.Warn("Ignoring registry config without a name",
slog.Any("entry", configHost))
continue
}
if configHost.Name == DockerRegistry || configHost.Name == DockerRegistryDNS || configHost.Name == DockerRegistryAuth {
configHost.Name = DockerRegistry
if configHost.Hostname == "" || configHost.Hostname == DockerRegistry || configHost.Hostname == DockerRegistryAuth {
configHost.Hostname = DockerRegistryDNS
}
}
tls, _ := configHost.TLS.MarshalText()
rc.slog.Debug("Loading config",
slog.Int64("blobChunk", configHost.BlobChunk),
slog.Int64("blobMax", configHost.BlobMax),
slog.String("helper", configHost.CredHelper),
slog.String("hostname", configHost.Hostname),
slog.Any("mirrors", configHost.Mirrors),
slog.String("name", configHost.Name),
slog.String("pathPrefix", configHost.PathPrefix),
slog.Bool("repoAuth", configHost.RepoAuth),
slog.String("source", src),
slog.String("tls", string(tls)),
slog.String("user", configHost.User))
err := rc.hostSet(configHost)
if err != nil {
rc.slog.Warn("Failed to update host config",
slog.String("host", configHost.Name),
slog.String("user", configHost.User),
slog.String("error", err.Error()))
}
}
}
func (rc *RegClient) hostSet(newHost config.Host) error {
name := newHost.Name
var err error
if _, ok := rc.hosts[name]; !ok {
// merge newHost with default host settings
rc.hosts[name] = config.HostNewDefName(rc.hostDefault, name)
err = rc.hosts[name].Merge(newHost, nil)
} else {
// merge newHost with existing settings
err = rc.hosts[name].Merge(newHost, rc.slog)
}
if err != nil {
return err
}
return nil
}
+19
View File
@@ -0,0 +1,19 @@
//go:build !wasm
package regclient
import (
"log/slog"
"github.com/sirupsen/logrus"
"github.com/regclient/regclient/internal/sloghandle"
)
// WithLog configuring logging with a logrus Logger.
// Note that regclient has switched to log/slog for logging and my eventually deprecate logrus support.
func WithLog(log *logrus.Logger) Opt {
return func(rc *RegClient) {
rc.slog = slog.New(sloghandle.Logrus(log))
}
}
+19
View File
@@ -0,0 +1,19 @@
# Release v0.11.5
Security:
- Prevent https to non-https downgrades and localhost redirects. ([PR 1093][pr-1093])
- Forbid sending auth on redirects. ([PR 1095][pr-1095])
Features:
- Add regbot `manifest.descriptor` to the sandbox. ([PR 1091][pr-1091])
Contributors:
- @GimmyDatBeeR
- @sudo-bmitch
[pr-1091]: https://github.com/regclient/regclient/pull/1091
[pr-1093]: https://github.com/regclient/regclient/pull/1093
[pr-1095]: https://github.com/regclient/regclient/pull/1095
+33
View File
@@ -0,0 +1,33 @@
package regclient
import (
"context"
"fmt"
"strings"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/repo"
)
type repoLister interface {
RepoList(ctx context.Context, hostname string, opts ...scheme.RepoOpts) (*repo.RepoList, error)
}
// RepoList returns a list of repositories on a registry.
// Note the underlying "_catalog" API is not supported on many cloud registries.
func (rc *RegClient) RepoList(ctx context.Context, hostname string, opts ...scheme.RepoOpts) (*repo.RepoList, error) {
i := strings.Index(hostname, "/")
if i > 0 {
return nil, fmt.Errorf("invalid hostname: %s%.0w", hostname, errs.ErrParsingFailed)
}
schemeAPI, err := rc.schemeGet("reg")
if err != nil {
return nil, err
}
rl, ok := schemeAPI.(repoLister)
if !ok {
return nil, errs.ErrNotImplemented
}
return rl.RepoList(ctx, hostname, opts...)
}
+33
View File
@@ -0,0 +1,33 @@
package regclient
import (
"context"
"fmt"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/ref"
)
func (rc *RegClient) schemeGet(scheme string) (scheme.API, error) {
s, ok := rc.schemes[scheme]
if !ok {
return nil, fmt.Errorf("%w: unknown scheme \"%s\"", errs.ErrNotImplemented, scheme)
}
return s, nil
}
// Close is used to free resources associated with a reference.
// With ocidir, this may trigger a garbage collection process.
func (rc *RegClient) Close(ctx context.Context, r ref.Ref) error {
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return err
}
// verify Closer api is defined, noop if missing
sc, ok := schemeAPI.(scheme.Closer)
if !ok {
return nil
}
return sc.Close(ctx, r)
}
+160
View File
@@ -0,0 +1,160 @@
package ocidir
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"os"
"path"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/types/blob"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/ref"
)
// BlobDelete removes a blob from the repository.
// This method does not verify that blobs are unused.
// Calling the [OCIDir.Close] method to trigger the garbage collection is preferred.
func (o *OCIDir) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error {
err := d.Digest.Validate()
if err != nil {
return fmt.Errorf("failed to validate digest %s: %w", d.Digest.String(), err)
}
file := path.Join(r.Path, "blobs", d.Digest.Algorithm().String(), d.Digest.Encoded())
return os.Remove(file)
}
// BlobGet retrieves a blob, returning a reader
func (o *OCIDir) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
err := d.Digest.Validate()
if err != nil {
return nil, fmt.Errorf("failed to validate digest %s: %w", d.Digest.String(), err)
}
file := path.Join(r.Path, "blobs", d.Digest.Algorithm().String(), d.Digest.Encoded())
//#nosec G304 users should validate references they attempt to open
fd, err := os.Open(file)
if err != nil {
return nil, err
}
if d.Size <= 0 {
fi, err := fd.Stat()
if err != nil {
_ = fd.Close()
return nil, err
}
d.Size = fi.Size()
}
br := blob.NewReader(
blob.WithRef(r),
blob.WithReader(fd),
blob.WithDesc(d),
)
o.slog.Debug("retrieved blob",
slog.String("ref", r.CommonName()),
slog.String("file", file))
return br, nil
}
// BlobHead verifies the existence of a blob, the reader contains the headers but no body to read
func (o *OCIDir) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
err := d.Digest.Validate()
if err != nil {
return nil, fmt.Errorf("failed to validate digest %s: %w", d.Digest.String(), err)
}
file := path.Join(r.Path, "blobs", d.Digest.Algorithm().String(), d.Digest.Encoded())
//#nosec G304 users should validate references they attempt to open
fd, err := os.Open(file)
if err != nil {
return nil, err
}
defer fd.Close()
if d.Size <= 0 {
fi, err := fd.Stat()
if err != nil {
return nil, err
}
d.Size = fi.Size()
}
br := blob.NewReader(
blob.WithRef(r),
blob.WithDesc(d),
)
return br, nil
}
// BlobMount attempts to perform a server side copy of the blob
func (o *OCIDir) BlobMount(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor) error {
return errs.ErrUnsupported
}
// BlobPut sends a blob to the repository, returns the digest and size when successful
func (o *OCIDir) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error) {
t := o.throttleGet(r, false)
done, err := t.Acquire(ctx, reqmeta.Data{Kind: reqmeta.Blob, Size: d.Size})
if err != nil {
return d, err
}
defer done()
err = o.initIndex(r, false)
if err != nil {
return d, err
}
digester := d.DigestAlgo().Digester()
rdr = io.TeeReader(rdr, digester.Hash())
// write the blob to a tmp file
dir := path.Join(r.Path, "blobs", d.DigestAlgo().String())
tmpPattern := "*.tmp"
//#nosec G301 defer to user umask settings
err = os.MkdirAll(dir, 0o777)
if err != nil && !errors.Is(err, fs.ErrExist) {
return d, fmt.Errorf("failed creating %s: %w", dir, err)
}
tmpFile, err := os.CreateTemp(dir, tmpPattern)
if err != nil {
return d, fmt.Errorf("failed creating blob tmp file: %w", err)
}
fi, err := tmpFile.Stat()
if err != nil {
return d, fmt.Errorf("failed to stat blob tmpfile: %w", err)
}
tmpName := fi.Name()
i, err := io.Copy(tmpFile, rdr)
errC := tmpFile.Close()
if err != nil {
return d, err
}
if errC != nil {
return d, errC
}
// validate result matches descriptor, or update descriptor if it wasn't defined
if d.Digest.Validate() != nil {
d.Digest = digester.Digest()
} else if d.Digest != digester.Digest() {
return d, fmt.Errorf("unexpected digest, expected %s, computed %s", d.Digest, digester.Digest())
}
if d.Size <= 0 {
d.Size = i
} else if i != d.Size {
return d, fmt.Errorf("unexpected blob length, expected %d, received %d", d.Size, i)
}
file := path.Join(r.Path, "blobs", d.Digest.Algorithm().String(), d.Digest.Encoded())
//#nosec G703 inputs are user controlled
err = os.Rename(path.Join(dir, tmpName), file)
if err != nil {
return d, fmt.Errorf("failed to write blob (rename tmp file %s to %s): %w", path.Join(dir, tmpName), file, err)
}
o.slog.Debug("pushed blob",
slog.String("ref", r.CommonName()),
slog.String("file", file))
o.mu.Lock()
o.refMod(r)
o.mu.Unlock()
return d, nil
}
+117
View File
@@ -0,0 +1,117 @@
package ocidir
import (
"context"
"fmt"
"log/slog"
"os"
"path"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/ref"
)
// Close triggers a garbage collection if the underlying path has been modified
func (o *OCIDir) Close(ctx context.Context, r ref.Ref) error {
if !o.gc {
return nil
}
o.mu.Lock()
defer o.mu.Unlock()
if gc, ok := o.modRefs[r.Path]; !ok || !gc.mod || gc.locks > 0 {
// unmodified or locked, skip gc
return nil
}
// perform GC
o.slog.Debug("running GC",
slog.String("ref", r.CommonName()))
dl := map[string]bool{}
// recurse through index, manifests, and blob lists, generating a digest list
index, err := o.readIndex(r, true)
if err != nil {
return err
}
im, err := manifest.New(manifest.WithOrig(index))
if err != nil {
return err
}
err = o.closeProcManifest(ctx, r, im, &dl)
if err != nil {
return err
}
// go through filesystem digest list, removing entries not seen in recursive pass
blobsPath := path.Join(r.Path, "blobs")
blobDirs, err := os.ReadDir(blobsPath)
if err != nil {
return err
}
for _, blobDir := range blobDirs {
if !blobDir.IsDir() {
// should this warn or delete unexpected files in the blobs folder?
continue
}
digestFiles, err := os.ReadDir(path.Join(blobsPath, blobDir.Name()))
if err != nil {
return err
}
for _, digestFile := range digestFiles {
digest := fmt.Sprintf("%s:%s", blobDir.Name(), digestFile.Name())
if !dl[digest] {
o.slog.Debug("ocidir garbage collect",
slog.String("digest", digest))
// delete
err = os.Remove(path.Join(blobsPath, blobDir.Name(), digestFile.Name()))
if err != nil {
return fmt.Errorf("failed to delete %s: %w", path.Join(blobsPath, blobDir.Name(), digestFile.Name()), err)
}
}
}
}
delete(o.modRefs, r.Path)
return nil
}
func (o *OCIDir) closeProcManifest(ctx context.Context, r ref.Ref, m manifest.Manifest, dl *map[string]bool) error {
if mi, ok := m.(manifest.Indexer); ok {
// go through manifest list, updating dl, and recursively processing nested manifests
ml, err := mi.GetManifestList()
if err != nil {
return err
}
for _, cur := range ml {
cr := r.SetDigest(cur.Digest.String())
(*dl)[cr.Digest] = true
cm, err := o.manifestGet(ctx, cr)
if err != nil {
// ignore errors in case a manifest has been deleted or sparse copy
o.slog.Debug("could not retrieve manifest",
slog.String("ref", cr.CommonName()),
slog.String("err", err.Error()))
continue
}
err = o.closeProcManifest(ctx, cr, cm, dl)
if err != nil {
return err
}
}
}
if mi, ok := m.(manifest.Imager); ok {
// get config from manifest if it exists
cd, err := mi.GetConfig()
if err == nil {
(*dl)[cd.Digest.String()] = true
}
// finally add all layers to digest list
layers, err := mi.GetLayers()
if err != nil {
return err
}
for _, layer := range layers {
(*dl)[layer.Digest.String()] = true
}
}
return nil
}
+306
View File
@@ -0,0 +1,306 @@
package ocidir
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"os"
"path"
"slices"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/mediatype"
"github.com/regclient/regclient/types/ref"
)
// ManifestDelete removes a manifest, including all tags that point to that manifest
func (o *OCIDir) ManifestDelete(ctx context.Context, r ref.Ref, opts ...scheme.ManifestOpts) error {
o.mu.Lock()
defer o.mu.Unlock()
if r.Digest == "" {
return fmt.Errorf("digest required to delete manifest, reference %s%.0w", r.CommonName(), errs.ErrMissingDigest)
}
mc := scheme.ManifestConfig{}
for _, opt := range opts {
opt(&mc)
}
// always check for refers with ocidir
if mc.Manifest == nil {
m, err := o.manifestGet(ctx, r)
if err != nil {
return fmt.Errorf("failed to pull manifest for refers: %w", err)
}
mc.Manifest = m
}
if mc.Manifest != nil {
if ms, ok := mc.Manifest.(manifest.Subjecter); ok {
sDesc, err := ms.GetSubject()
if err == nil && sDesc != nil && sDesc.Digest != "" {
// attempt to delete the referrer, but ignore if the referrer entry wasn't found
err = o.referrerDelete(ctx, r, mc.Manifest)
if err != nil && !errors.Is(err, errs.ErrNotFound) && !errors.Is(err, fs.ErrNotExist) {
return err
}
}
}
}
// get index
changed := false
index, err := o.readIndex(r, true)
if err != nil {
return fmt.Errorf("failed to read index: %w", err)
}
for i := len(index.Manifests) - 1; i >= 0; i-- {
// remove matching entry from index
if r.Digest != "" && index.Manifests[i].Digest.String() == r.Digest {
changed = true
index.Manifests = slices.Delete(index.Manifests, i, i+1)
}
}
// push manifest back out
if changed {
err = o.writeIndex(r, index, true)
if err != nil {
return fmt.Errorf("failed to write index: %w", err)
}
}
// delete from filesystem like a registry would do
d := digest.Digest(r.Digest)
file := path.Join(r.Path, "blobs", d.Algorithm().String(), d.Encoded())
err = os.Remove(file)
if err != nil {
return fmt.Errorf("failed to delete manifest: %w", err)
}
o.refMod(r)
return nil
}
// ManifestGet retrieves a manifest from a repository
func (o *OCIDir) ManifestGet(ctx context.Context, r ref.Ref) (manifest.Manifest, error) {
o.mu.Lock()
defer o.mu.Unlock()
return o.manifestGet(ctx, r)
}
func (o *OCIDir) manifestGet(_ context.Context, r ref.Ref) (manifest.Manifest, error) {
index, err := o.readIndex(r, true)
if err != nil {
return nil, fmt.Errorf("unable to read oci index: %w", err)
}
if r.Digest == "" && r.Tag == "" {
r = r.SetTag("latest")
}
desc, err := indexGet(index, r)
if err != nil {
if r.Digest != "" {
desc.Digest = digest.Digest(r.Digest)
} else {
return nil, err
}
}
if desc.Digest == "" {
return nil, errs.ErrNotFound
}
if err = desc.Digest.Validate(); err != nil {
return nil, fmt.Errorf("invalid digest in index: %s: %w", string(desc.Digest), err)
}
file := path.Join(r.Path, "blobs", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
//#nosec G304 users should validate references they attempt to open
fd, err := os.Open(file)
if err != nil {
return nil, fmt.Errorf("failed to open manifest: %w", err)
}
defer fd.Close()
mb, err := io.ReadAll(fd)
if err != nil {
return nil, fmt.Errorf("failed to read manifest: %w", err)
}
if desc.Size == 0 {
desc.Size = int64(len(mb))
}
o.slog.Debug("retrieved manifest",
slog.String("ref", r.CommonName()),
slog.String("file", file))
return manifest.New(
manifest.WithRef(r),
manifest.WithDesc(desc),
manifest.WithRaw(mb),
)
}
// ManifestHead gets metadata about the manifest (existence, digest, mediatype, size)
func (o *OCIDir) ManifestHead(ctx context.Context, r ref.Ref) (manifest.Manifest, error) {
index, err := o.readIndex(r, false)
if err != nil {
return nil, fmt.Errorf("unable to read oci index: %w", err)
}
if r.Digest == "" && r.Tag == "" {
r = r.SetTag("latest")
}
desc, err := indexGet(index, r)
if err != nil {
if r.Digest != "" {
desc.Digest = digest.Digest(r.Digest)
} else {
return nil, err
}
}
if desc.Digest == "" {
return nil, errs.ErrNotFound
}
if err = desc.Digest.Validate(); err != nil {
return nil, fmt.Errorf("invalid digest in index: %s: %w", string(desc.Digest), err)
}
// verify underlying file exists
file := path.Join(r.Path, "blobs", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
fi, err := os.Stat(file)
if err != nil || fi.IsDir() {
return nil, errs.ErrNotFound
}
// if missing, set media type on desc
if desc.MediaType == "" {
//#nosec G304 users should validate references they attempt to open
raw, err := os.ReadFile(file)
if err != nil {
return nil, err
}
mt := struct {
MediaType string `json:"mediaType,omitempty"`
SchemaVersion int `json:"schemaVersion,omitempty"`
Signatures []any `json:"signatures,omitempty"`
}{}
err = json.Unmarshal(raw, &mt)
if err != nil {
return nil, err
}
if mt.MediaType != "" {
desc.MediaType = mt.MediaType
desc.Size = int64(len(raw))
} else if mt.SchemaVersion == 1 && len(mt.Signatures) > 0 {
desc.MediaType = mediatype.Docker1ManifestSigned
} else if mt.SchemaVersion == 1 {
desc.MediaType = mediatype.Docker1Manifest
desc.Size = int64(len(raw))
}
}
return manifest.New(
manifest.WithRef(r),
manifest.WithDesc(desc),
)
}
// ManifestPut sends a manifest to the repository
func (o *OCIDir) ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...scheme.ManifestOpts) error {
o.mu.Lock()
defer o.mu.Unlock()
return o.manifestPut(ctx, r, m, opts...)
}
func (o *OCIDir) manifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...scheme.ManifestOpts) error {
config := scheme.ManifestConfig{}
for _, opt := range opts {
opt(&config)
}
if !config.Child && r.Digest == "" && r.Tag == "" {
r = r.SetTag("latest")
}
err := o.initIndex(r, true)
if err != nil {
return err
}
desc := m.GetDescriptor()
if err = desc.Digest.Validate(); err != nil {
return fmt.Errorf("invalid digest for manifest: %s: %w", string(desc.Digest), err)
}
b, err := m.RawBody()
if err != nil {
return fmt.Errorf("could not serialize manifest: %w", err)
}
if r.Digest != "" && desc.Digest.String() != r.Digest {
// Digest algorithm may have changed, try recreating the manifest with the provided ref.
// This will fail if the ref digest does not match the manifest.
m, err = manifest.New(manifest.WithRef(r), manifest.WithRaw(b))
if err != nil {
return fmt.Errorf("failed to rebuilding manifest with ref \"%s\": %w", r.CommonName(), err)
}
}
desc.Annotations = map[string]string{}
if r.Tag != "" {
desc.Annotations[types.AnnotationRefName] = r.Tag
}
// TODO: if the manifest contains a subject, option to add descriptor details and include entry in the index.json, config.child=false
// create manifest CAS file
dir := path.Join(r.Path, "blobs", desc.Digest.Algorithm().String())
//#nosec G301 defer to user umask settings
err = os.MkdirAll(dir, 0o777)
if err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("failed creating %s: %w", dir, err)
}
// write to a tmp file, rename after validating
tmpFile, err := os.CreateTemp(dir, desc.Digest.Encoded()+".*.tmp")
if err != nil {
return fmt.Errorf("failed to create manifest tmpfile: %w", err)
}
fi, err := tmpFile.Stat()
if err != nil {
return fmt.Errorf("failed to stat manifest tmpfile: %w", err)
}
tmpName := fi.Name()
_, err = tmpFile.Write(b)
errC := tmpFile.Close()
if err != nil {
return fmt.Errorf("failed to write manifest tmpfile: %w", err)
}
if errC != nil {
return fmt.Errorf("failed to close manifest tmpfile: %w", errC)
}
file := path.Join(dir, desc.Digest.Encoded())
//#nosec G703 inputs are user controlled
err = os.Rename(path.Join(dir, tmpName), file)
if err != nil {
return fmt.Errorf("failed to write manifest (rename tmpfile): %w", err)
}
// verify/update index
err = o.updateIndex(r, desc, config.Child, true)
if err != nil {
return err
}
o.refMod(r)
o.slog.Debug("pushed manifest",
slog.String("ref", r.CommonName()),
slog.String("file", file))
// update referrers if defined on this manifest
if ms, ok := m.(manifest.Subjecter); ok {
mDesc, err := ms.GetSubject()
if err != nil {
return err
}
if mDesc != nil && mDesc.Digest != "" {
err = o.referrerPut(ctx, r, m)
if err != nil {
return err
}
}
}
return nil
}
+419
View File
@@ -0,0 +1,419 @@
// Package ocidir implements the OCI Image Layout scheme with a directory (not packed in a tar)
package ocidir
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"os"
"path"
"slices"
"strings"
"sync"
"time"
"github.com/regclient/regclient/internal/pqueue"
"github.com/regclient/regclient/internal/reproducible"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/types"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/mediatype"
v1 "github.com/regclient/regclient/types/oci/v1"
"github.com/regclient/regclient/types/ref"
)
const (
imageLayoutFile = "oci-layout"
defThrottle = 3
)
// OCIDir is used for accessing OCI Image Layouts defined as a directory
type OCIDir struct {
slog *slog.Logger
gc bool
modRefs map[string]*ociGC
throttle map[string]*pqueue.Queue[reqmeta.Data]
throttleDef int
mu sync.Mutex
}
type ociGC struct {
mod bool
locks int
}
type ociConf struct {
gc bool
slog *slog.Logger
throttle int
}
// Opts are used for passing options to ocidir
type Opts func(*ociConf)
// New creates a new OCIDir with options
func New(opts ...Opts) *OCIDir {
conf := ociConf{
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
gc: true,
throttle: defThrottle,
}
for _, opt := range opts {
opt(&conf)
}
return &OCIDir{
slog: conf.slog,
gc: conf.gc,
modRefs: map[string]*ociGC{},
throttle: map[string]*pqueue.Queue[reqmeta.Data]{},
throttleDef: conf.throttle,
}
}
// WithGC configures the garbage collection setting
// This defaults to enabled
func WithGC(gc bool) Opts {
return func(c *ociConf) {
c.gc = gc
}
}
// WithSlog provides a slog logger.
// By default logging is disabled.
func WithSlog(slog *slog.Logger) Opts {
return func(c *ociConf) {
c.slog = slog
}
}
// WithThrottle provides a number of concurrent write actions (blob/manifest put)
func WithThrottle(count int) Opts {
return func(c *ociConf) {
c.throttle = count
}
}
// GCLock is used to prevent GC on a ref
func (o *OCIDir) GCLock(r ref.Ref) {
o.mu.Lock()
defer o.mu.Unlock()
if gc, ok := o.modRefs[r.Path]; ok && gc != nil {
gc.locks++
} else {
o.modRefs[r.Path] = &ociGC{locks: 1}
}
}
// GCUnlock removes a hold on GC of a ref, this must be done before the ref is closed
func (o *OCIDir) GCUnlock(r ref.Ref) {
o.mu.Lock()
defer o.mu.Unlock()
if gc, ok := o.modRefs[r.Path]; ok && gc != nil && gc.locks > 0 {
gc.locks--
}
}
// Throttle is used to limit concurrency
func (o *OCIDir) Throttle(r ref.Ref, put bool) []*pqueue.Queue[reqmeta.Data] {
tList := []*pqueue.Queue[reqmeta.Data]{}
// throttle only applies to put requests
if !put || o.throttleDef <= 0 {
return tList
}
return []*pqueue.Queue[reqmeta.Data]{o.throttleGet(r, false)}
}
func (o *OCIDir) throttleGet(r ref.Ref, locked bool) *pqueue.Queue[reqmeta.Data] {
if !locked {
o.mu.Lock()
defer o.mu.Unlock()
}
if t, ok := o.throttle[r.Path]; ok {
return t
}
// init a new throttle
o.throttle[r.Path] = pqueue.New(pqueue.Opts[reqmeta.Data]{Max: o.throttleDef})
return o.throttle[r.Path]
}
func (o *OCIDir) initIndex(r ref.Ref, locked bool) error {
if !locked {
o.mu.Lock()
defer o.mu.Unlock()
}
layoutFile := path.Join(r.Path, imageLayoutFile)
_, err := os.Stat(layoutFile)
if err == nil {
return nil
}
//#nosec G301 defer to user umask settings
err = os.MkdirAll(r.Path, 0o777)
if err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("failed creating %s: %w", r.Path, err)
}
// create/replace oci-layout file
layout := v1.ImageLayout{
Version: "1.0.0",
}
lb, err := json.Marshal(layout)
if err != nil {
return fmt.Errorf("cannot marshal layout: %w", err)
}
//#nosec G304 users should validate references they attempt to open
lfh, err := os.Create(layoutFile)
if err != nil {
return fmt.Errorf("cannot create %s: %w", imageLayoutFile, err)
}
defer lfh.Close()
_, err = lfh.Write(lb)
if err != nil {
return fmt.Errorf("cannot write %s: %w", imageLayoutFile, err)
}
return nil
}
func (o *OCIDir) readIndex(r ref.Ref, locked bool) (v1.Index, error) {
if !locked {
o.mu.Lock()
defer o.mu.Unlock()
}
// validate dir
index := v1.Index{}
err := o.valid(r.Path, true)
if err != nil {
return index, err
}
indexFile := path.Join(r.Path, "index.json")
//#nosec G304 users should validate references they attempt to open
fh, err := os.Open(indexFile)
if err != nil {
return index, fmt.Errorf("%s cannot be open: %w", indexFile, err)
}
defer fh.Close()
ib, err := io.ReadAll(fh)
if err != nil {
return index, fmt.Errorf("%s cannot be read: %w", indexFile, err)
}
err = json.Unmarshal(ib, &index)
if err != nil {
return index, fmt.Errorf("%s cannot be parsed: %w", indexFile, err)
}
return index, nil
}
func (o *OCIDir) updateIndex(r ref.Ref, d descriptor.Descriptor, child bool, locked bool) error {
if !locked {
o.mu.Lock()
defer o.mu.Unlock()
}
indexChanged := false
index, err := o.readIndex(r, true)
if err != nil {
index = indexCreate()
indexChanged = true
}
if !child {
err := indexSet(&index, r, d)
if err != nil {
return fmt.Errorf("failed to update index: %w", err)
}
indexChanged = true
}
if indexChanged {
err = o.writeIndex(r, index, true)
if err != nil {
return fmt.Errorf("failed to write index: %w", err)
}
}
return nil
}
func (o *OCIDir) writeIndex(r ref.Ref, i v1.Index, locked bool) error {
if !locked {
o.mu.Lock()
defer o.mu.Unlock()
}
//#nosec G301 defer to user umask settings
err := os.MkdirAll(r.Path, 0o777)
if err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("failed creating %s: %w", r.Path, err)
}
// create/replace oci-layout file
layout := v1.ImageLayout{
Version: "1.0.0",
}
lb, err := json.Marshal(layout)
if err != nil {
return fmt.Errorf("cannot marshal layout: %w", err)
}
lfh, err := os.Create(path.Join(r.Path, imageLayoutFile))
if err != nil {
return fmt.Errorf("cannot create %s: %w", imageLayoutFile, err)
}
defer lfh.Close()
_, err = lfh.Write(lb)
if err != nil {
return fmt.Errorf("cannot write %s: %w", imageLayoutFile, err)
}
// create/replace index.json file
tmpFile, err := os.CreateTemp(r.Path, "index.json.*.tmp")
if err != nil {
return fmt.Errorf("cannot create index tmpfile: %w", err)
}
fi, err := tmpFile.Stat()
if err != nil {
return fmt.Errorf("failed to stat index tmpfile: %w", err)
}
tmpName := fi.Name()
b, err := json.Marshal(i)
if err != nil {
return fmt.Errorf("cannot marshal index: %w", err)
}
_, err = tmpFile.Write(b)
errC := tmpFile.Close()
if err != nil {
return fmt.Errorf("cannot write index: %w", err)
}
if errC != nil {
return fmt.Errorf("cannot close index: %w", errC)
}
indexFile := path.Join(r.Path, "index.json")
//#nosec G703 inputs are user controlled
err = os.Rename(path.Join(r.Path, tmpName), indexFile)
if err != nil {
return fmt.Errorf("cannot rename tmpfile to index: %w", err)
}
return nil
}
// func valid (dir) (error) // check for `oci-layout` file and `index.json` for read
func (o *OCIDir) valid(dir string, locked bool) error {
if !locked {
o.mu.Lock()
defer o.mu.Unlock()
}
layout := v1.ImageLayout{}
reqVer := "1.0.0"
//#nosec G304 users should validate references they attempt to open
fh, err := os.Open(path.Join(dir, imageLayoutFile))
if err != nil {
return fmt.Errorf("%s cannot be open: %w", imageLayoutFile, err)
}
defer fh.Close()
lb, err := io.ReadAll(fh)
if err != nil {
return fmt.Errorf("%s cannot be read: %w", imageLayoutFile, err)
}
err = json.Unmarshal(lb, &layout)
if err != nil {
return fmt.Errorf("%s cannot be parsed: %w", imageLayoutFile, err)
}
if layout.Version != reqVer {
return fmt.Errorf("unsupported oci layout version, expected %s, received %s", reqVer, layout.Version)
}
return nil
}
func (o *OCIDir) refMod(r ref.Ref) {
if gc, ok := o.modRefs[r.Path]; ok && gc != nil {
gc.mod = true
} else {
o.modRefs[r.Path] = &ociGC{mod: true}
}
}
func indexCreate() v1.Index {
i := v1.Index{
Versioned: v1.IndexSchemaVersion,
MediaType: mediatype.OCI1ManifestList,
Manifests: []descriptor.Descriptor{},
Annotations: map[string]string{},
}
return i
}
func indexGet(index v1.Index, r ref.Ref) (descriptor.Descriptor, error) {
if r.Digest == "" && r.Tag == "" {
r = r.SetTag("latest")
}
if r.Digest != "" {
for _, im := range index.Manifests {
if im.Digest.String() == r.Digest {
return im, nil
}
}
} else if r.Tag != "" {
for _, im := range index.Manifests {
if name, ok := im.Annotations[types.AnnotationRefName]; ok && name == r.Tag {
return im, nil
}
}
// fall back to support full image name in annotation
for _, im := range index.Manifests {
if name, ok := im.Annotations[types.AnnotationRefName]; ok && strings.HasSuffix(name, ":"+r.Tag) {
return im, nil
}
}
}
return descriptor.Descriptor{}, errs.ErrNotFound
}
func indexSet(index *v1.Index, r ref.Ref, d descriptor.Descriptor) error {
if index == nil {
return fmt.Errorf("index is nil")
}
if d.Annotations == nil {
d.Annotations = map[string]string{}
}
if r.Tag != "" {
d.Annotations[types.AnnotationRefName] = r.Tag
}
if _, ok := d.Annotations[types.AnnotationCreated]; !ok {
d.Annotations[types.AnnotationCreated] = reproducible.TimeNow().Format(time.RFC3339)
}
if index.Manifests == nil {
index.Manifests = []descriptor.Descriptor{}
}
pos := -1
// search for existing
for i := range index.Manifests {
var name string
if index.Manifests[i].Annotations != nil {
name = index.Manifests[i].Annotations[types.AnnotationRefName]
}
if r.Tag == "" && name == "" && index.Manifests[i].Digest == d.Digest {
// untagged entry already exists, no need to replace it
pos = i
break
} else if r.Tag != "" && name == r.Tag {
if index.Manifests[i].Digest != d.Digest {
// replace the tag with a new digest/descriptor
index.Manifests[i] = d
}
pos = i
break
}
}
if pos >= 0 {
// existing entry was replaced, remove any dup entries
for i := len(index.Manifests) - 1; i > pos; i-- {
var name string
if index.Manifests[i].Annotations != nil {
name = index.Manifests[i].Annotations[types.AnnotationRefName]
}
// prune untagged entries with a matching digest if we didn't push a tag
// or entries with a matching tag
if (r.Tag == "" && name == "" && index.Manifests[i].Digest == d.Digest) || (r.Tag != "" && name == r.Tag) {
index.Manifests = slices.Delete(index.Manifests, i, i+1)
}
}
} else {
// existing entry to replace was not found, add the descriptor
index.Manifests = append(index.Manifests, d)
}
return nil
}
+19
View File
@@ -0,0 +1,19 @@
//go:build !wasm
package ocidir
import (
"log/slog"
"github.com/sirupsen/logrus"
"github.com/regclient/regclient/internal/sloghandle"
)
// WithLog provides a logrus logger.
// By default logging is disabled.
func WithLog(log *logrus.Logger) Opts {
return func(c *ociConf) {
c.slog = slog.New(sloghandle.Logrus(log))
}
}
+29
View File
@@ -0,0 +1,29 @@
package ocidir
import (
"context"
"fmt"
"os"
"github.com/regclient/regclient/types/ping"
"github.com/regclient/regclient/types/ref"
)
// Ping for an ocidir verifies access to read the path.
func (o *OCIDir) Ping(ctx context.Context, r ref.Ref) (ping.Result, error) {
ret := ping.Result{}
fd, err := os.Open(r.Path)
if err != nil {
return ret, err
}
defer fd.Close()
fi, err := fd.Stat()
if err != nil {
return ret, err
}
ret.Stat = fi
if !fi.IsDir() {
return ret, fmt.Errorf("failed to access %s: not a directory", r.Path)
}
return ret, nil
}
+218
View File
@@ -0,0 +1,218 @@
package ocidir
import (
"context"
"errors"
"fmt"
"strings"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/mediatype"
v1 "github.com/regclient/regclient/types/oci/v1"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/referrer"
)
// ReferrerList returns a list of referrers to a given reference.
// The reference must include the digest. Use [regclient.ReferrerList] to resolve the platform or tag.
func (o *OCIDir) ReferrerList(ctx context.Context, r ref.Ref, opts ...scheme.ReferrerOpts) (referrer.ReferrerList, error) {
o.mu.Lock()
defer o.mu.Unlock()
return o.referrerList(ctx, r, opts...)
}
func (o *OCIDir) referrerList(ctx context.Context, rSubject ref.Ref, opts ...scheme.ReferrerOpts) (referrer.ReferrerList, error) {
config := scheme.ReferrerConfig{}
for _, opt := range opts {
opt(&config)
}
var r ref.Ref
if config.SrcRepo.IsSet() {
r = config.SrcRepo.SetDigest(rSubject.Digest)
} else {
r = rSubject.SetDigest(rSubject.Digest)
}
rl := referrer.New(rSubject)
if rSubject.Digest == "" {
return rl, fmt.Errorf("digest required to query referrers %s", rSubject.CommonName())
}
// TODO: Add fast path with private field in index.json indicating referrers are tracked using an annotation.
// This can be implemented with backward compatible support by also pushing individual manifests or the fallback tag.
// TODO: Upstream OCI specification of how to track subject/referrers in an OCI Layout is still pending.
// optional slow query of all manifests for ORAS support
if config.SlowSearch {
idx, err := o.readIndex(r, true)
if err != nil {
return rl, err
}
for _, desc := range idx.Manifests {
// pull the manifest
rCur := r.SetDigest(desc.Digest.String())
m, err := o.manifestGet(ctx, rCur)
if err != nil {
// skip all errors
continue
}
// check for the subject
mSubj, ok := m.(manifest.Subjecter)
if !ok {
continue
}
subj, err := mSubj.GetSubject()
if err != nil || subj == nil || subj.Digest.String() != rSubject.Digest {
continue
}
// extract/track tag
if name, ok := desc.Annotations[types.AnnotationRefName]; ok && !strings.ContainsAny(name, ":/") {
rl.Tags = append(rl.Tags, name)
}
// if matching, add to rl
err = rl.Add(m)
if err != nil {
return rl, err
}
}
}
// try to request the fallback tag
rlFallback, err := o.referrerListFallback(ctx, r)
if err == nil {
err = rl.Merge(rlFallback)
if err != nil {
return rl, fmt.Errorf("failed to merge referrers from fallback tag: %w", err)
}
}
// update referrer list
rl.Subject = rSubject
if config.SrcRepo.IsSet() {
rl.Source = config.SrcRepo
}
rl = scheme.ReferrerFilter(config, rl)
return rl, nil
}
func (o *OCIDir) referrerListFallback(ctx context.Context, r ref.Ref) (referrer.ReferrerList, error) {
rl := referrer.New(r)
rlTag, err := referrer.FallbackTag(r)
if err != nil {
return rl, err
}
m, err := o.manifestGet(ctx, rlTag)
if err != nil {
if errors.Is(err, errs.ErrNotFound) {
// empty list, initialize a new manifest
rl.Manifest, err = manifest.New(manifest.WithOrig(v1.Index{
Versioned: v1.IndexSchemaVersion,
MediaType: mediatype.OCI1ManifestList,
}))
if err != nil {
return rl, err
}
return rl, nil
}
return rl, err
}
ociML, ok := m.GetOrig().(v1.Index)
if !ok {
return rl, fmt.Errorf("manifest is not an OCI index: %s", rlTag.CommonName())
}
// update referrer list
rl.Manifest = m
rl.Descriptors = ociML.Manifests
rl.Annotations = ociML.Annotations
rl.Tags = append(rl.Tags, rlTag.Tag)
return rl, nil
}
// referrerDelete deletes a referrer associated with a manifest
func (o *OCIDir) referrerDelete(ctx context.Context, r ref.Ref, m manifest.Manifest) error {
// get refers field
mSubject, ok := m.(manifest.Subjecter)
if !ok {
return fmt.Errorf("manifest does not support subject: %w", errs.ErrUnsupportedMediaType)
}
subject, err := mSubject.GetSubject()
if err != nil {
return err
}
// validate/set subject descriptor
if subject == nil || subject.Digest == "" {
return fmt.Errorf("subject is not set%.0w", errs.ErrNotFound)
}
// get descriptor for subject
rSubject := r.SetDigest(subject.Digest.String())
// pull existing referrer list
rl, err := o.referrerList(ctx, rSubject)
if err != nil {
return err
}
err = rl.Delete(m)
if err != nil {
return err
}
// push updated referrer list by tag
rlTag, err := referrer.FallbackTag(rSubject)
if err != nil {
return err
}
if rl.IsEmpty() {
err = o.tagDelete(ctx, rlTag)
if err == nil {
return nil
}
// if delete is not supported, fall back to pushing empty list
}
return o.manifestPut(ctx, rlTag, rl.Manifest)
}
// referrerPut pushes a new referrer associated with a given reference
func (o *OCIDir) referrerPut(ctx context.Context, r ref.Ref, m manifest.Manifest) error {
// get subject field
mSubject, ok := m.(manifest.Subjecter)
if !ok {
return fmt.Errorf("manifest does not support subject: %w", errs.ErrUnsupportedMediaType)
}
subject, err := mSubject.GetSubject()
if err != nil {
return err
}
// validate/set subject descriptor
if subject == nil || subject.Digest == "" {
return fmt.Errorf("subject is not set%.0w", errs.ErrNotFound)
}
// get descriptor for subject
rSubject := r.SetDigest(subject.Digest.String())
// pull existing referrer list
rl, err := o.referrerList(ctx, rSubject)
if err != nil {
return err
}
err = rl.Add(m)
if err != nil {
return err
}
// TODO: add support for pushing a fast path generated response
// TODO: allow fallback tag creation to be skipped, potentially via a config in the manifestPut
// push updated referrer list by tag
rlTag, err := referrer.FallbackTag(rSubject)
if err != nil {
return err
}
return o.manifestPut(ctx, rlTag, rl.Manifest)
}
+90
View File
@@ -0,0 +1,90 @@
package ocidir
import (
"context"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/mediatype"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/tag"
)
// TagDelete removes a tag from the repository
func (o *OCIDir) TagDelete(ctx context.Context, r ref.Ref) error {
o.mu.Lock()
defer o.mu.Unlock()
return o.tagDelete(ctx, r)
}
func (o *OCIDir) tagDelete(_ context.Context, r ref.Ref) error {
if r.Tag == "" {
return errs.ErrMissingTag
}
// get index
index, err := o.readIndex(r, true)
if err != nil {
return fmt.Errorf("failed to read index: %w", err)
}
changed := false
for i, desc := range index.Manifests {
if t, ok := desc.Annotations[types.AnnotationRefName]; ok && t == r.Tag {
// remove matching entry from index
index.Manifests = slices.Delete(index.Manifests, i, i+1)
changed = true
}
}
if !changed {
return fmt.Errorf("failed deleting %s: %w", r.CommonName(), errs.ErrNotFound)
}
// push manifest back out
err = o.writeIndex(r, index, true)
if err != nil {
return fmt.Errorf("failed to write index: %w", err)
}
o.refMod(r)
return nil
}
// TagList returns a list of tags from the repository
func (o *OCIDir) TagList(ctx context.Context, r ref.Ref, opts ...scheme.TagOpts) (*tag.List, error) {
// get index
index, err := o.readIndex(r, false)
if err != nil {
return nil, err
}
tl := []string{}
for _, desc := range index.Manifests {
if t, ok := desc.Annotations[types.AnnotationRefName]; ok {
if i := strings.LastIndex(t, ":"); i >= 0 {
t = t[i+1:]
}
if !slices.Contains(tl, t) {
tl = append(tl, t)
}
}
}
sort.Strings(tl)
ib, err := json.Marshal(index)
if err != nil {
return nil, err
}
// return listing from index
t, err := tag.New(
tag.WithRaw(ib),
tag.WithRef(r),
tag.WithMT(mediatype.OCI1ManifestList),
tag.WithLayoutIndex(index),
tag.WithTags(tl),
)
if err != nil {
return nil, err
}
return t, nil
}
+680
View File
@@ -0,0 +1,680 @@
package reg
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"strconv"
"strings"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/internal/reghttp"
"github.com/regclient/regclient/internal/regnet"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/types/blob"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/warning"
)
var zeroDig = digest.SHA256.FromBytes([]byte{})
// BlobDelete removes a blob from the repository
func (reg *Reg) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error {
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
Method: "DELETE",
Repository: r.Repository,
Path: "blobs/" + d.Digest.String(),
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return fmt.Errorf("failed to delete blob, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), err)
}
if resp.HTTPResponse().StatusCode != 202 {
return fmt.Errorf("failed to delete blob, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
return nil
}
// BlobGet retrieves a blob from the repository, returning a blob reader
func (reg *Reg) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
// build/send request
req := &reghttp.Req{
MetaKind: reqmeta.Blob,
Host: r.Registry,
Method: "GET",
Repository: r.Repository,
Path: "blobs/" + d.Digest.String(),
ExpectLen: d.Size,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil && len(d.URLs) > 0 {
for _, curURL := range d.URLs {
// fallback for external blobs
var u *url.URL
u, err = url.Parse(curURL)
if err != nil {
return nil, fmt.Errorf("failed to parse external url \"%s\": %w", curURL, err)
}
// refuse requests to local URLs unless registry is also local
// Note: the AllowRedirect check is not used because blobs could be hosted on an http CDN and data is content addressable
if regnet.IsLocal(u.Host) && !regnet.IsLocal(req.Host) {
return nil, fmt.Errorf("refusing to redirect blob request to a local URL: %s%.0w", curURL, errs.ErrHTTPRedirectRefused)
}
req = &reghttp.Req{
MetaKind: reqmeta.Blob,
Host: r.Registry,
Method: "GET",
Repository: r.Repository,
DirectURL: u,
NoMirrors: true,
ExpectLen: d.Size,
}
resp, err = reg.reghttp.Do(ctx, req)
if err == nil {
break
}
}
}
if err != nil {
return nil, fmt.Errorf("failed to get blob, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), err)
}
if resp.HTTPResponse().StatusCode != 200 {
return nil, fmt.Errorf("failed to get blob, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
b := blob.NewReader(
blob.WithRef(r),
blob.WithReader(resp),
blob.WithDesc(d),
blob.WithResp(resp.HTTPResponse()),
)
return b, nil
}
// BlobHead is used to verify if a blob exists and is accessible
func (reg *Reg) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
// build/send request
req := &reghttp.Req{
MetaKind: reqmeta.Head,
Host: r.Registry,
Method: "HEAD",
Repository: r.Repository,
Path: "blobs/" + d.Digest.String(),
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil && len(d.URLs) > 0 {
for _, curURL := range d.URLs {
// fallback for external blobs
var u *url.URL
u, err = url.Parse(curURL)
if err != nil {
return nil, fmt.Errorf("failed to parse external url \"%s\": %w", curURL, err)
}
req = &reghttp.Req{
MetaKind: reqmeta.Head,
Host: r.Registry,
Method: "HEAD",
Repository: r.Repository,
DirectURL: u,
NoMirrors: true,
}
resp, err = reg.reghttp.Do(ctx, req)
if err == nil {
break
}
}
}
if err != nil {
return nil, fmt.Errorf("failed to request blob head, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 200 {
return nil, fmt.Errorf("failed to request blob head, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
b := blob.NewReader(
blob.WithRef(r),
blob.WithDesc(d),
blob.WithResp(resp.HTTPResponse()),
)
return b, nil
}
// BlobMount attempts to perform a server side copy/mount of the blob between repositories
func (reg *Reg) BlobMount(ctx context.Context, rSrc ref.Ref, rTgt ref.Ref, d descriptor.Descriptor) error {
putURL, _, err := reg.blobMount(ctx, rTgt, d, rSrc)
// if mount fails and returns an upload location, cancel that upload
if err != nil {
_ = reg.blobUploadCancel(ctx, rTgt, putURL)
}
return err
}
// BlobPut uploads a blob to a repository.
// Descriptor is optional, leave size and digest to zero value if unknown.
// Reader must also be an [io.Seeker] to support chunked upload fallback.
//
// This will attempt an anonymous blob mount first which some registries may support.
// It will then try doing a full put of the blob without chunking (most widely supported).
// If the full put fails, it will fall back to a chunked upload (useful for flaky networks).
func (reg *Reg) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error) {
var putURL *url.URL
var err error
validDesc := (d.Size > 0 && d.Digest.Validate() == nil) || (d.Size == 0 && d.Digest == zeroDig)
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
// attempt an anonymous blob mount
if validDesc {
putURL, _, err = reg.blobMount(ctx, r, d, ref.Ref{})
if err == nil {
return d, nil
}
if err != errs.ErrMountReturnedLocation {
putURL = nil
}
}
// fallback to requesting upload URL
if putURL == nil {
putURL, err = reg.blobGetUploadURL(ctx, r, d)
if err != nil {
return d, err
}
}
// send upload as one-chunk
tryPut := validDesc
if tryPut {
host := reg.hostGet(r.Registry)
maxPut := host.BlobMax
if maxPut == 0 {
maxPut = reg.blobMaxPut
}
if maxPut > 0 && d.Size > maxPut {
tryPut = false
}
}
if tryPut {
err = reg.blobPutUploadFull(ctx, r, d, putURL, rdr)
if err == nil {
return d, nil
}
// on failure, attempt to seek back to start to perform a chunked upload
rdrSeek, ok := rdr.(io.ReadSeeker)
if !ok {
_ = reg.blobUploadCancel(ctx, r, putURL)
return d, err
}
offset, errR := rdrSeek.Seek(0, io.SeekStart)
if errR != nil || offset != 0 {
_ = reg.blobUploadCancel(ctx, r, putURL)
return d, err
}
}
// send a chunked upload if full upload not possible or too large
d, err = reg.blobPutUploadChunked(ctx, r, d, putURL, rdr)
if err != nil {
_ = reg.blobUploadCancel(ctx, r, putURL)
}
return d, err
}
func (reg *Reg) blobGetUploadURL(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (*url.URL, error) {
q := url.Values{}
if d.DigestAlgo() != digest.Canonical {
// TODO(bmitch): EXPERIMENTAL parameter, registry support and OCI spec change needed
q.Add(paramBlobDigestAlgo, d.DigestAlgo().String())
}
// request an upload location
req := &reghttp.Req{
MetaKind: reqmeta.Blob,
Host: r.Registry,
NoMirrors: true,
Method: "POST",
Repository: r.Repository,
Path: "blobs/uploads/",
Query: q,
TransactLen: d.Size,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to send blob post, ref %s: %w", r.CommonName(), err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 202 {
return nil, fmt.Errorf("failed to send blob post, ref %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
// if min size header received, check/adjust host settings
minSizeStr := resp.HTTPResponse().Header.Get(blobChunkMinHeader)
if minSizeStr != "" {
minSize, err := strconv.ParseInt(minSizeStr, 10, 64)
if err != nil {
reg.slog.Warn("Failed to parse chunk size header",
slog.String("size", minSizeStr),
slog.String("err", err.Error()))
} else {
host := reg.hostGet(r.Registry)
reg.muHost.Lock()
if (host.BlobChunk > 0 && minSize > host.BlobChunk) || (host.BlobChunk <= 0 && minSize > reg.blobChunkSize) {
host.BlobChunk = min(minSize, reg.blobChunkLimit)
reg.slog.Debug("Registry requested min chunk size",
slog.Int64("size", host.BlobChunk),
slog.String("host", host.Name))
}
reg.muHost.Unlock()
}
}
// Extract the location into a new putURL based on whether it's relative, fqdn with a scheme, or without a scheme.
location := resp.HTTPResponse().Header.Get("Location")
if location == "" {
return nil, fmt.Errorf("failed to send blob post, ref %s: %w", r.CommonName(), errs.ErrMissingLocation)
}
reg.slog.Debug("Upload location received",
slog.String("location", location))
// put url may be relative to the above post URL, so parse in that context
postURL := resp.HTTPResponse().Request.URL
putURL, err := postURL.Parse(location)
if err != nil {
reg.slog.Warn("Location url failed to parse",
slog.String("location", location),
slog.String("err", err.Error()))
return nil, fmt.Errorf("blob upload url invalid, ref %s: %w", r.CommonName(), err)
}
return putURL, nil
}
func (reg *Reg) blobMount(ctx context.Context, rTgt ref.Ref, d descriptor.Descriptor, rSrc ref.Ref) (*url.URL, string, error) {
// build/send request
query := url.Values{}
query.Set("mount", d.Digest.String())
ignoreErr := true // ignore errors from anonymous blob mount attempts
if rSrc.Registry == rTgt.Registry && rSrc.Repository != "" {
query.Set("from", rSrc.Repository)
ignoreErr = false
}
req := &reghttp.Req{
MetaKind: reqmeta.Blob,
Host: rTgt.Registry,
NoMirrors: true,
Method: "POST",
Repository: rTgt.Repository,
Path: "blobs/uploads/",
Query: query,
IgnoreErr: ignoreErr,
TransactLen: d.Size,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return nil, "", fmt.Errorf("failed to mount blob, digest %s, ref %s: %w", d.Digest.String(), rTgt.CommonName(), err)
}
defer resp.Close()
// if min size header received, check/adjust host settings
minSizeStr := resp.HTTPResponse().Header.Get(blobChunkMinHeader)
if minSizeStr != "" {
minSize, err := strconv.ParseInt(minSizeStr, 10, 64)
if err != nil {
reg.slog.Warn("Failed to parse chunk size header",
slog.String("size", minSizeStr),
slog.String("err", err.Error()))
} else {
host := reg.hostGet(rTgt.Registry)
reg.muHost.Lock()
if (host.BlobChunk > 0 && minSize > host.BlobChunk) || (host.BlobChunk <= 0 && minSize > reg.blobChunkSize) {
host.BlobChunk = min(minSize, reg.blobChunkLimit)
reg.slog.Debug("Registry requested min chunk size",
slog.Int64("size", host.BlobChunk),
slog.String("host", host.Name))
}
reg.muHost.Unlock()
}
}
// 201 indicates the blob mount succeeded
if resp.HTTPResponse().StatusCode == 201 {
return nil, "", nil
}
// 202 indicates blob mount failed but server ready to receive an upload at location
location := resp.HTTPResponse().Header.Get("Location")
uuid := resp.HTTPResponse().Header.Get("Docker-Upload-UUID")
if resp.HTTPResponse().StatusCode == 202 && location != "" {
postURL := resp.HTTPResponse().Request.URL
putURL, err := postURL.Parse(location)
if err != nil {
reg.slog.Warn("Mount location header failed to parse",
slog.String("digest", d.Digest.String()),
slog.String("target", rTgt.CommonName()),
slog.String("location", location),
slog.String("err", err.Error()))
} else {
return putURL, uuid, errs.ErrMountReturnedLocation
}
}
// all other responses unhandled
return nil, "", fmt.Errorf("failed to mount blob, digest %s, ref %s: %w", d.Digest.String(), rTgt.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
func (reg *Reg) blobPutUploadFull(ctx context.Context, r ref.Ref, d descriptor.Descriptor, putURL *url.URL, rdr io.Reader) error {
// append digest to request to use the monolithic upload option
if putURL.RawQuery != "" {
putURL.RawQuery = putURL.RawQuery + "&digest=" + url.QueryEscape(d.Digest.String())
} else {
putURL.RawQuery = "digest=" + url.QueryEscape(d.Digest.String())
}
// make a reader function for the blob
readOnce := false
bodyFunc := func() (io.ReadCloser, error) {
// handle attempt to reuse blob reader (e.g. on a connection retry or fallback)
if readOnce {
rdrSeek, ok := rdr.(io.ReadSeeker)
if !ok {
return nil, fmt.Errorf("blob source is not a seeker%.0w", errs.ErrNotRetryable)
}
_, err := rdrSeek.Seek(0, io.SeekStart)
if err != nil {
return nil, fmt.Errorf("seek on blob source failed: %w%.0w", err, errs.ErrNotRetryable)
}
}
readOnce = true
return io.NopCloser(rdr), nil
}
// special case for the empty blob
if d.Size == 0 && d.Digest == zeroDig {
bodyFunc = nil
}
// build/send request
header := http.Header{
"Content-Type": {"application/octet-stream"},
}
req := &reghttp.Req{
MetaKind: reqmeta.Blob,
Host: r.Registry,
Method: "PUT",
Repository: r.Repository,
DirectURL: putURL,
BodyFunc: bodyFunc,
BodyLen: d.Size,
Headers: header,
NoMirrors: true,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return fmt.Errorf("failed to send blob (put), digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), err)
}
defer resp.Close()
// 201 follows distribution-spec, 204 is listed as possible in the Docker registry spec
if resp.HTTPResponse().StatusCode != 201 && resp.HTTPResponse().StatusCode != 204 {
return fmt.Errorf("failed to send blob (put), digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
return nil
}
func (reg *Reg) blobPutUploadChunked(ctx context.Context, r ref.Ref, d descriptor.Descriptor, putURL *url.URL, rdr io.Reader) (descriptor.Descriptor, error) {
host := reg.hostGet(r.Registry)
bufSize := host.BlobChunk
if bufSize <= 0 {
bufSize = reg.blobChunkSize
}
bufBytes := make([]byte, 0, bufSize)
bufRdr := bytes.NewReader(bufBytes)
bufStart := int64(0)
bufChange := false
// setup buffer and digest pipe
digester := d.DigestAlgo().Digester()
digestRdr := io.TeeReader(rdr, digester.Hash())
finalChunk := false
chunkStart := int64(0)
chunkSize := 0
bodyFunc := func() (io.ReadCloser, error) {
// reset to the start on every new read
_, err := bufRdr.Seek(0, io.SeekStart)
if err != nil {
return nil, err
}
return io.NopCloser(bufRdr), nil
}
chunkURL := *putURL
retryLimit := 10 // TODO: pull limit from reghttp
retryCur := 0
var err error
for !finalChunk || chunkStart < bufStart+int64(len(bufBytes)) {
bufChange = false
for chunkStart >= bufStart+int64(len(bufBytes)) && !finalChunk {
bufStart += int64(len(bufBytes))
// reset length if previous read was short
if cap(bufBytes) != len(bufBytes) {
bufBytes = bufBytes[:cap(bufBytes)]
bufChange = true
}
// read a chunk into an input buffer, computing the digest
chunkSize, err = io.ReadFull(digestRdr, bufBytes)
if err == io.EOF || err == io.ErrUnexpectedEOF {
finalChunk = true
} else if err != nil {
return d, fmt.Errorf("failed to send blob chunk, ref %s: %w", r.CommonName(), err)
}
// update length on partial read
if chunkSize != len(bufBytes) {
bufBytes = bufBytes[:chunkSize]
bufChange = true
}
}
if chunkStart > bufStart && chunkStart < bufStart+int64(len(bufBytes)) {
// next chunk is inside the existing buf
bufBytes = bufBytes[chunkStart-bufStart:]
bufStart = chunkStart
chunkSize = len(bufBytes)
bufChange = true
}
if chunkSize > 0 && chunkStart != bufStart {
return d, fmt.Errorf("chunkStart (%d) != bufStart (%d)", chunkStart, bufStart)
}
if bufChange {
// need to recreate the reader on a change to the slice length,
// old reader is looking at the old slice metadata
bufRdr = bytes.NewReader(bufBytes)
}
if chunkSize > 0 {
// write chunk
header := http.Header{
"Content-Type": {"application/octet-stream"},
"Content-Range": {fmt.Sprintf("%d-%d", chunkStart, chunkStart+int64(chunkSize)-1)},
}
req := &reghttp.Req{
MetaKind: reqmeta.Blob,
Host: r.Registry,
Method: "PATCH",
Repository: r.Repository,
DirectURL: &chunkURL,
BodyFunc: bodyFunc,
BodyLen: int64(chunkSize),
Headers: header,
NoMirrors: true,
TransactLen: d.Size - int64(chunkSize),
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil && !errors.Is(err, errs.ErrHTTPStatus) && !errors.Is(err, errs.ErrNotFound) {
return d, fmt.Errorf("failed to send blob (chunk), ref %s: http do: %w", r.CommonName(), err)
}
err = resp.Close()
if err != nil {
return d, fmt.Errorf("failed to close request: %w", err)
}
httpResp := resp.HTTPResponse()
// distribution-spec is 202, AWS ECR returns a 201 and rejects the put
if resp.HTTPResponse().StatusCode == 201 {
reg.slog.Debug("Early accept of chunk in PATCH before PUT request",
slog.String("ref", r.CommonName()),
slog.Int64("chunkStart", chunkStart),
slog.Int("chunkSize", chunkSize))
} else if resp.HTTPResponse().StatusCode >= 400 && resp.HTTPResponse().StatusCode < 500 &&
resp.HTTPResponse().Header.Get("Location") != "" &&
resp.HTTPResponse().Header.Get("Range") != "" {
retryCur++
if retryCur > retryLimit {
return d, fmt.Errorf("failed to send blob (chunk), ref %s: http status: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
reg.slog.Debug("Recoverable chunk upload error",
slog.String("ref", r.CommonName()),
slog.Int64("chunkStart", chunkStart),
slog.Int("chunkSize", chunkSize),
slog.String("range", resp.HTTPResponse().Header.Get("Range")))
} else if resp.HTTPResponse().StatusCode != 202 {
retryCur++
statusResp, statusErr := reg.blobUploadStatus(ctx, r, &chunkURL)
if retryCur > retryLimit || statusErr != nil {
return d, fmt.Errorf("failed to send blob (chunk), ref %s: http status: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
httpResp = statusResp
} else {
// successful request
if retryCur > 0 {
retryCur--
}
}
rangeEnd, err := blobUploadCurBytes(httpResp)
if err == nil {
chunkStart = rangeEnd + 1
} else {
chunkStart += int64(chunkSize)
}
location := httpResp.Header.Get("Location")
if location != "" {
reg.slog.Debug("Next chunk upload location received",
slog.String("location", location))
prevURL := httpResp.Request.URL
parseURL, err := prevURL.Parse(location)
if err != nil {
return d, fmt.Errorf("failed to send blob (parse next chunk location), ref %s: %w", r.CommonName(), err)
}
chunkURL = *parseURL
}
}
}
// compute digest
dOut := digester.Digest()
if d.Digest.Validate() == nil && dOut != d.Digest {
return d, fmt.Errorf("%w, expected %s, computed %s", errs.ErrDigestMismatch, d.Digest.String(), dOut.String())
}
if d.Size != 0 && chunkStart != d.Size {
return d, fmt.Errorf("blob content size does not match descriptor, expected %d, received %d%.0w", d.Size, chunkStart, errs.ErrMismatch)
}
d.Digest = dOut
d.Size = chunkStart
// send the final put
// append digest to request to use the monolithic upload option
if chunkURL.RawQuery != "" {
chunkURL.RawQuery = chunkURL.RawQuery + "&digest=" + url.QueryEscape(dOut.String())
} else {
chunkURL.RawQuery = "digest=" + url.QueryEscape(dOut.String())
}
header := http.Header{
"Content-Type": {"application/octet-stream"},
}
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
Method: "PUT",
Repository: r.Repository,
DirectURL: &chunkURL,
BodyLen: int64(0),
Headers: header,
NoMirrors: true,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return d, fmt.Errorf("failed to send blob (chunk digest), digest %s, ref %s: %w", dOut, r.CommonName(), err)
}
defer resp.Close()
// 201 follows distribution-spec, 204 is listed as possible in the Docker registry spec
if resp.HTTPResponse().StatusCode != 201 && resp.HTTPResponse().StatusCode != 204 {
return d, fmt.Errorf("failed to send blob (chunk digest), digest %s, ref %s: %w", dOut, r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
return d, nil
}
// blobUploadCancel stops an upload, releasing resources on the server.
func (reg *Reg) blobUploadCancel(ctx context.Context, r ref.Ref, putURL *url.URL) error {
if putURL == nil {
return fmt.Errorf("failed to cancel upload %s: url undefined", r.CommonName())
}
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
NoMirrors: true,
Method: "DELETE",
Repository: r.Repository,
DirectURL: putURL,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return fmt.Errorf("failed to cancel upload %s: %w", r.CommonName(), err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 202 {
return fmt.Errorf("failed to cancel upload %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
return nil
}
// blobUploadStatus provides a response with headers indicating the progress of an upload
func (reg *Reg) blobUploadStatus(ctx context.Context, r ref.Ref, putURL *url.URL) (*http.Response, error) {
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
Method: "GET",
Repository: r.Repository,
DirectURL: putURL,
NoMirrors: true,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to get upload status: %w", err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 204 {
return resp.HTTPResponse(), fmt.Errorf("failed to get upload status: %w", reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
return resp.HTTPResponse(), nil
}
func blobUploadCurBytes(resp *http.Response) (int64, error) {
if resp == nil {
return 0, fmt.Errorf("missing response")
}
r := resp.Header.Get("Range")
if r == "" {
return 0, fmt.Errorf("missing range header")
}
rSplit := strings.SplitN(r, "-", 2)
if len(rSplit) < 2 {
return 0, fmt.Errorf("missing offset in range header")
}
return strconv.ParseInt(rSplit[1], 10, 64)
}
+360
View File
@@ -0,0 +1,360 @@
package reg
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/internal/limitread"
"github.com/regclient/regclient/internal/reghttp"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/mediatype"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/warning"
)
// ManifestDelete removes a manifest by reference (digest) from a registry.
// This will implicitly delete all tags pointing to that manifest.
func (reg *Reg) ManifestDelete(ctx context.Context, r ref.Ref, opts ...scheme.ManifestOpts) error {
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
if r.Digest == "" {
return fmt.Errorf("digest required to delete manifest, reference %s%.0w", r.CommonName(), errs.ErrMissingDigest)
}
mc := scheme.ManifestConfig{}
for _, opt := range opts {
opt(&mc)
}
if mc.CheckReferrers && mc.Manifest == nil {
m, err := reg.ManifestGet(ctx, r)
if err != nil {
return fmt.Errorf("failed to pull manifest for refers: %w", err)
}
mc.Manifest = m
}
if mc.Manifest != nil {
if mr, ok := mc.Manifest.(manifest.Subjecter); ok {
sDesc, err := mr.GetSubject()
if err == nil && sDesc != nil && sDesc.Digest != "" {
// attempt to delete the referrer, but ignore if the referrer entry wasn't found
err = reg.referrerDelete(ctx, r, mc.Manifest)
if err != nil && !errors.Is(err, errs.ErrNotFound) {
return err
}
}
}
}
rCache := r.SetDigest(r.Digest)
reg.cacheMan.Delete(rCache)
// build/send request
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
NoMirrors: true,
Method: "DELETE",
Repository: r.Repository,
Path: "manifests/" + r.Digest,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return fmt.Errorf("failed to delete manifest %s: %w", r.CommonName(), err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 202 {
return fmt.Errorf("failed to delete manifest %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
return nil
}
// ManifestGet retrieves a manifest from the registry
func (reg *Reg) ManifestGet(ctx context.Context, r ref.Ref) (manifest.Manifest, error) {
var tagOrDigest string
if r.Digest != "" {
rCache := r.SetDigest(r.Digest)
if m, err := reg.cacheMan.Get(rCache); err == nil {
return m, nil
}
tagOrDigest = r.Digest
} else if r.Tag != "" {
tagOrDigest = r.Tag
} else {
return nil, fmt.Errorf("reference missing tag and digest: %s%.0w", r.CommonName(), errs.ErrMissingTagOrDigest)
}
// build/send request
headers := http.Header{
"Accept": []string{
mediatype.OCI1ManifestList,
mediatype.OCI1Manifest,
mediatype.Docker2ManifestList,
mediatype.Docker2Manifest,
mediatype.Docker1ManifestSigned,
mediatype.Docker1Manifest,
mediatype.OCI1Artifact,
},
}
req := &reghttp.Req{
MetaKind: reqmeta.Manifest,
Host: r.Registry,
Method: "GET",
Repository: r.Repository,
Path: "manifests/" + tagOrDigest,
Headers: headers,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to get manifest %s: %w", r.CommonName(), err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 200 {
return nil, fmt.Errorf("failed to get manifest %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
// limit length
size, _ := strconv.Atoi(resp.HTTPResponse().Header.Get("Content-Length"))
if size > 0 && reg.manifestMaxPull > 0 && int64(size) > reg.manifestMaxPull {
return nil, fmt.Errorf("manifest too large, received %d, limit %d: %s%.0w", size, reg.manifestMaxPull, r.CommonName(), errs.ErrSizeLimitExceeded)
}
rdr := &limitread.LimitRead{
Reader: resp,
Limit: reg.manifestMaxPull,
}
// read manifest
rawBody, err := io.ReadAll(rdr)
if err != nil {
return nil, fmt.Errorf("error reading manifest for %s: %w", r.CommonName(), err)
}
m, err := manifest.New(
manifest.WithRef(r),
manifest.WithHeader(resp.HTTPResponse().Header),
manifest.WithRaw(rawBody),
)
if err != nil {
return nil, err
}
rCache := r.SetDigest(m.GetDescriptor().Digest.String())
reg.cacheMan.Set(rCache, m)
return m, nil
}
// ManifestHead returns metadata on the manifest from the registry
func (reg *Reg) ManifestHead(ctx context.Context, r ref.Ref) (manifest.Manifest, error) {
// build the request
var tagOrDigest string
if r.Digest != "" {
rCache := r.SetDigest(r.Digest)
if m, err := reg.cacheMan.Get(rCache); err == nil {
return m, nil
}
tagOrDigest = r.Digest
} else if r.Tag != "" {
tagOrDigest = r.Tag
} else {
return nil, fmt.Errorf("reference missing tag and digest: %s%.0w", r.CommonName(), errs.ErrMissingTagOrDigest)
}
// build/send request
headers := http.Header{
"Accept": []string{
mediatype.OCI1ManifestList,
mediatype.OCI1Manifest,
mediatype.Docker2ManifestList,
mediatype.Docker2Manifest,
mediatype.Docker1ManifestSigned,
mediatype.Docker1Manifest,
mediatype.OCI1Artifact,
},
}
req := &reghttp.Req{
MetaKind: reqmeta.Head,
Host: r.Registry,
Method: "HEAD",
Repository: r.Repository,
Path: "manifests/" + tagOrDigest,
Headers: headers,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to request manifest head %s: %w", r.CommonName(), err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 200 {
return nil, fmt.Errorf("failed to request manifest head %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
return manifest.New(
manifest.WithRef(r),
manifest.WithHeader(resp.HTTPResponse().Header),
)
}
// ManifestPut uploads a manifest to a registry
func (reg *Reg) ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...scheme.ManifestOpts) error {
var tagOrDigest string
if r.Digest != "" {
tagOrDigest = r.Digest
} else if r.Tag != "" {
tagOrDigest = r.Tag
} else {
reg.slog.Warn("Manifest put requires a tag",
slog.String("ref", r.Reference))
return errs.ErrMissingTag
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
// create the request body
mj, err := m.MarshalJSON()
if err != nil {
reg.slog.Warn("Error marshaling manifest",
slog.String("ref", r.Reference),
slog.String("err", err.Error()))
return fmt.Errorf("error marshalling manifest for %s: %w", r.CommonName(), err)
}
// limit length
if reg.manifestMaxPush > 0 && int64(len(mj)) > reg.manifestMaxPush {
return fmt.Errorf("manifest too large, calculated %d, limit %d: %s%.0w", len(mj), reg.manifestMaxPush, r.CommonName(), errs.ErrSizeLimitExceeded)
}
// if the ref provides a digest, verify it matches the manifest being pushed
desc := m.GetDescriptor()
if err := desc.Digest.Validate(); err != nil {
return fmt.Errorf("invalid digest for manifest: %s: %w", string(desc.Digest), err)
}
if r.Digest != "" && desc.Digest.String() != r.Digest {
// Digest algorithm may have changed, try recreating the manifest with the provided ref.
// This will fail if the ref digest does not match the manifest.
m, err = manifest.New(manifest.WithRef(r), manifest.WithRaw(mj))
if err != nil {
return fmt.Errorf("failed to rebuilding manifest with ref \"%s\": %w", r.CommonName(), err)
}
}
// build/send request
expectTags := []string{}
headers := http.Header{
"Content-Type": []string{manifest.GetMediaType(m)},
}
q := url.Values{}
if tagOrDigest == r.Tag && m.GetDescriptor().Digest.Algorithm() != digest.Canonical {
// TODO(bmitch): EXPERIMENTAL support for pushing tags with a digest: <https://github.com/opencontainers/distribution-spec/pull/600>
tagOrDigest = m.GetDescriptor().Digest.String()
q.Add("tag", r.Tag)
expectTags = append(expectTags, r.Tag)
}
req := &reghttp.Req{
MetaKind: reqmeta.Manifest,
Host: r.Registry,
NoMirrors: true,
Method: "PUT",
Repository: r.Repository,
Path: "manifests/" + tagOrDigest,
Query: q,
Headers: headers,
BodyLen: int64(len(mj)),
BodyBytes: mj,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return fmt.Errorf("failed to put manifest %s: %w", r.CommonName(), err)
}
err = resp.Close()
if err != nil {
return fmt.Errorf("failed to close request: %w", err)
}
if resp.HTTPResponse().StatusCode != 201 {
return fmt.Errorf("failed to put manifest %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
// if Docker-Content-Digest header was returned, verify the digest matches
if dig := resp.HTTPResponse().Header.Get("Docker-Content-Digest"); dig != "" && dig != m.GetDescriptor().Digest.String() {
return fmt.Errorf("failed to put manifest, unexpected digest returned, expected %s, received %s", m.GetDescriptor().Digest.String(), dig)
}
// if pushing tags by digest fails, fall back to pushing individual tags
respTags := []string{}
for _, hv := range resp.HTTPResponse().Header.Values("OCI-Tag") {
for sv := range strings.SplitSeq(hv, ",") {
respTags = append(respTags, strings.TrimSpace(sv))
}
}
for _, t := range expectTags {
if slices.Contains(respTags, t) {
continue
}
q := url.Values{}
if m.GetDescriptor().Digest.Algorithm() != digest.Canonical {
// TODO(bmitch): EXPERIMENTAL parameter, registry support and OCI spec change needed: <https://github.com/opencontainers/distribution-spec/pull/543>
q.Add(paramManifestDigest, m.GetDescriptor().Digest.String())
}
req := &reghttp.Req{
MetaKind: reqmeta.Manifest,
Host: r.Registry,
NoMirrors: true,
Method: "PUT",
Repository: r.Repository,
Path: "manifests/" + t,
Query: q,
Headers: headers,
BodyLen: int64(len(mj)),
BodyBytes: mj,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return fmt.Errorf("failed to put manifest %s: %w", r.CommonName(), err)
}
err = resp.Close()
if err != nil {
return fmt.Errorf("failed to close request: %w", err)
}
if resp.HTTPResponse().StatusCode != 201 {
return fmt.Errorf("failed to put manifest %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
// if Docker-Content-Digest header was returned, verify the digest matches
if dig := resp.HTTPResponse().Header.Get("Docker-Content-Digest"); dig != "" && dig != m.GetDescriptor().Digest.String() {
return fmt.Errorf("failed to put manifest, unexpected digest returned, expected %s, received %s", m.GetDescriptor().Digest.String(), dig)
}
}
rCache := r.SetDigest(m.GetDescriptor().Digest.String())
reg.cacheMan.Set(rCache, m)
// update referrers if defined on this manifest
if mr, ok := m.(manifest.Subjecter); ok {
mDesc, err := mr.GetSubject()
if err != nil {
return err
}
if mDesc != nil && mDesc.Digest.String() != "" {
rSubj := r.SetDigest(mDesc.Digest.String())
reg.cacheRL.Delete(rSubj)
if mDesc.Digest.String() != resp.HTTPResponse().Header.Get(OCISubjectHeader) {
err = reg.referrerPut(ctx, r, m)
if err != nil {
return err
}
}
}
}
return nil
}
+39
View File
@@ -0,0 +1,39 @@
package reg
import (
"context"
"fmt"
"github.com/regclient/regclient/internal/reghttp"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/types/ping"
"github.com/regclient/regclient/types/ref"
)
// Ping queries the /v2/ API of the registry to verify connectivity and access.
func (reg *Reg) Ping(ctx context.Context, r ref.Ref) (ping.Result, error) {
ret := ping.Result{}
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
NoMirrors: true,
Method: "GET",
Path: "",
}
resp, err := reg.reghttp.Do(ctx, req)
if resp != nil && resp.HTTPResponse() != nil {
ret.Header = resp.HTTPResponse().Header
}
if err != nil {
return ret, fmt.Errorf("failed to ping registry %s: %w", r.Registry, err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 200 {
return ret, fmt.Errorf("failed to ping registry %s: %w",
r.Registry, reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
return ret, nil
}
+354
View File
@@ -0,0 +1,354 @@
package reg
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"github.com/regclient/regclient/internal/httplink"
"github.com/regclient/regclient/internal/reghttp"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/mediatype"
v1 "github.com/regclient/regclient/types/oci/v1"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/referrer"
"github.com/regclient/regclient/types/warning"
)
const OCISubjectHeader = "OCI-Subject"
// ReferrerList returns a list of referrers to a given reference.
// The reference must include the digest. Use [regclient.ReferrerList] to resolve the platform or tag.
func (reg *Reg) ReferrerList(ctx context.Context, rSubject ref.Ref, opts ...scheme.ReferrerOpts) (referrer.ReferrerList, error) {
config := scheme.ReferrerConfig{}
for _, opt := range opts {
opt(&config)
}
var r ref.Ref
if config.SrcRepo.IsSet() {
r = config.SrcRepo.SetDigest(rSubject.Digest)
} else {
r = rSubject.SetDigest(rSubject.Digest)
}
rl := referrer.New(rSubject)
if rSubject.Digest == "" {
return rl, fmt.Errorf("digest required to query referrers %s", rSubject.CommonName())
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
found := false
// try cache
rl, err := reg.cacheRL.Get(r)
if err == nil {
found = true
}
// try referrers API
if !found {
referrerEnabled, ok := reg.featureGet("referrer", r.Registry, r.Repository)
if !ok || referrerEnabled {
// attempt to call the referrer API
rl, err = reg.referrerListByAPI(ctx, r, config)
if !ok {
// save the referrer API state
reg.featureSet("referrer", r.Registry, r.Repository, err == nil)
}
if err == nil {
if config.MatchOpt.ArtifactType == "" {
// only cache if successful and artifactType is not filtered
reg.cacheRL.Set(r, rl)
}
found = true
}
}
}
// fall back to tag
if !found {
rl, err = reg.referrerListByTag(ctx, r)
if err == nil {
reg.cacheRL.Set(r, rl)
}
}
rl.Subject = rSubject
if config.SrcRepo.IsSet() {
rl.Source = config.SrcRepo
}
if err != nil {
return rl, err
}
// apply client side filters and return result
rl = scheme.ReferrerFilter(config, rl)
return rl, nil
}
func (reg *Reg) referrerListByAPI(ctx context.Context, r ref.Ref, config scheme.ReferrerConfig) (referrer.ReferrerList, error) {
rl := referrer.New(r)
var link *url.URL
// loop for paging
for {
rlAdd, linkNext, err := reg.referrerListByAPIPage(ctx, r, config, link)
if err != nil {
return rl, err
}
err = rl.Merge(rlAdd)
if err != nil {
return rl, err
}
if linkNext == nil {
break
}
link = linkNext
}
return rl, nil
}
func (reg *Reg) referrerListByAPIPage(ctx context.Context, r ref.Ref, config scheme.ReferrerConfig, link *url.URL) (referrer.ReferrerList, *url.URL, error) {
rl := referrer.New(r)
query := url.Values{}
if config.MatchOpt.ArtifactType != "" {
query.Set("artifactType", config.MatchOpt.ArtifactType)
}
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
Method: "GET",
Repository: r.Repository,
}
if link == nil {
req.Path = "referrers/" + r.Digest
req.Query = query
req.IgnoreErr = true
}
if link != nil {
req.DirectURL = link
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return rl, nil, fmt.Errorf("failed to get referrers %s: %w", r.CommonName(), err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 200 {
return rl, nil, fmt.Errorf("failed to get referrers %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
// read manifest
rawBody, err := io.ReadAll(resp)
if err != nil {
return rl, nil, fmt.Errorf("error reading referrers for %s: %w", r.CommonName(), err)
}
m, err := manifest.New(
manifest.WithRef(r.SetDigest("")),
manifest.WithHeader(resp.HTTPResponse().Header),
manifest.WithRaw(rawBody),
)
if err != nil {
return rl, nil, err
}
ociML, ok := m.GetOrig().(v1.Index)
if !ok {
return rl, nil, fmt.Errorf("unexpected manifest type for referrers: %s, %w", m.GetDescriptor().MediaType, errs.ErrUnsupportedMediaType)
}
rl.Manifest = m
rl.Descriptors = ociML.Manifests
rl.Annotations = ociML.Annotations
// lookup next link
respHead := resp.HTTPResponse().Header
links, err := httplink.Parse(respHead.Values("Link"))
if err != nil {
return rl, nil, err
}
next, err := links.Get("rel", "next")
if err != nil {
// no next link
link = nil
} else {
link = resp.HTTPResponse().Request.URL
if link == nil {
return rl, nil, fmt.Errorf("referrers list failed to get URL of previous request")
}
link, err = link.Parse(next.URI)
if err != nil {
return rl, nil, fmt.Errorf("referrers list failed to parse Link: %w", err)
}
}
return rl, link, nil
}
func (reg *Reg) referrerListByTag(ctx context.Context, r ref.Ref) (referrer.ReferrerList, error) {
rl := referrer.New(r)
rlTag, err := referrer.FallbackTag(r)
if err != nil {
return rl, err
}
m, err := reg.ManifestGet(ctx, rlTag)
if err != nil {
if errors.Is(err, errs.ErrNotFound) {
// empty list, initialize a new manifest
rl.Manifest, err = manifest.New(manifest.WithOrig(v1.Index{
Versioned: v1.IndexSchemaVersion,
MediaType: mediatype.OCI1ManifestList,
}))
if err != nil {
return rl, err
}
return rl, nil
}
return rl, err
}
ociML, ok := m.GetOrig().(v1.Index)
if !ok {
return rl, fmt.Errorf("manifest is not an OCI index: %s", rlTag.CommonName())
}
// return resulting index
rl.Manifest = m
rl.Descriptors = ociML.Manifests
rl.Annotations = ociML.Annotations
rl.Tags = append(rl.Tags, rlTag.Tag)
return rl, nil
}
// referrerDelete deletes a referrer associated with a manifest
func (reg *Reg) referrerDelete(ctx context.Context, r ref.Ref, m manifest.Manifest) error {
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
// get subject field
mSubject, ok := m.(manifest.Subjecter)
if !ok {
return fmt.Errorf("manifest does not support the subject field: %w", errs.ErrUnsupportedMediaType)
}
subject, err := mSubject.GetSubject()
if err != nil {
return err
}
// validate/set subject descriptor
if subject == nil || subject.Digest == "" {
return fmt.Errorf("refers is not set%.0w", errs.ErrNotFound)
}
// remove from cache
rSubject := r.SetDigest(subject.Digest.String())
reg.cacheRL.Delete(rSubject)
// if referrer API is available, nothing to do, return
if reg.referrerPing(ctx, rSubject) {
return nil
}
// fallback to using tag schema for refers
rl, err := reg.referrerListByTag(ctx, rSubject)
if err != nil {
return err
}
err = rl.Delete(m)
if err != nil {
return err
}
// push updated referrer list by tag
rlTag, err := referrer.FallbackTag(rSubject)
if err != nil {
return err
}
if rl.IsEmpty() {
err = reg.TagDelete(ctx, rlTag)
if err == nil {
return nil
}
// if delete is not supported, fall back to pushing empty list
}
return reg.ManifestPut(ctx, rlTag, rl.Manifest)
}
// referrerPut pushes a new referrer associated with a manifest
func (reg *Reg) referrerPut(ctx context.Context, r ref.Ref, m manifest.Manifest) error {
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
// get subject field
mSubject, ok := m.(manifest.Subjecter)
if !ok {
return fmt.Errorf("manifest does not support the subject field: %w", errs.ErrUnsupportedMediaType)
}
subject, err := mSubject.GetSubject()
if err != nil {
return err
}
// validate/set subject descriptor
if subject == nil || subject.Digest == "" {
return fmt.Errorf("subject is not set%.0w", errs.ErrNotFound)
}
// lock to avoid internal race conditions between pulling and pushing tag
reg.muRefTag.Lock()
defer reg.muRefTag.Unlock()
// fallback to using tag schema for refers
rSubject := r.SetDigest(subject.Digest.String())
rl, err := reg.referrerListByTag(ctx, rSubject)
if err != nil {
return err
}
err = rl.Add(m)
if err != nil {
return err
}
// ensure the referrer list does not have a subject itself (avoiding circular locks)
if ms, ok := rl.Manifest.(manifest.Subjecter); ok {
mDesc, err := ms.GetSubject()
if err != nil {
return err
}
if mDesc != nil && mDesc.Digest != "" {
return fmt.Errorf("fallback referrers manifest should not have a subject: %s", rSubject.CommonName())
}
}
// push updated referrer list by tag
rlTag, err := referrer.FallbackTag(rSubject)
if err != nil {
return err
}
if len(rl.Tags) == 0 {
rl.Tags = []string{rlTag.Tag}
}
err = reg.ManifestPut(ctx, rlTag, rl.Manifest)
if err == nil {
reg.cacheRL.Set(rSubject, rl)
}
return err
}
// referrerPing verifies the registry supports the referrers API
func (reg *Reg) referrerPing(ctx context.Context, r ref.Ref) bool {
referrerEnabled, ok := reg.featureGet("referrer", r.Registry, r.Repository)
if ok {
return referrerEnabled
}
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
Method: "GET",
Repository: r.Repository,
Path: "referrers/" + r.Digest,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
reg.featureSet("referrer", r.Registry, r.Repository, false)
return false
}
_ = resp.Close()
result := resp.HTTPResponse().StatusCode == 200
reg.featureSet("referrer", r.Registry, r.Repository, result)
return result
}
+272
View File
@@ -0,0 +1,272 @@
// Package reg implements the OCI registry scheme used by most images (host:port/repo:tag)
package reg
import (
"log/slog"
"net/http"
"sync"
"time"
"github.com/regclient/regclient/config"
"github.com/regclient/regclient/internal/cache"
"github.com/regclient/regclient/internal/pqueue"
"github.com/regclient/regclient/internal/reghttp"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/referrer"
)
const (
// blobChunkMinHeader is returned by registries requesting a minimum chunk size
blobChunkMinHeader = "OCI-Chunk-Min-Length"
// defaultBlobChunk 1M chunks, this is allocated in a memory buffer
defaultBlobChunk = 1024 * 1024
// defaultBlobChunkLimit 1G chunks, prevents a memory exhaustion attack
defaultBlobChunkLimit = 1024 * 1024 * 1024
// defaultBlobMax is disabled to support registries without chunked upload support
defaultBlobMax = -1
// defaultManifestMaxPull limits the largest manifest that will be pulled
defaultManifestMaxPull = 1024 * 1024 * 8
// defaultManifestMaxPush limits the largest manifest that will be pushed
defaultManifestMaxPush = 1024 * 1024 * 4
// paramBlobDigestAlgo specifies the query parameter to request a specific digest algorithm.
// TODO(bmitch): EXPERIMENTAL field, registry support and OCI spec update needed
paramBlobDigestAlgo = "digest-algorithm"
// paramManifestDigest specifies the query parameter to specify the digest of a manifest pushed by tag.
// TODO(bmitch): EXPERIMENTAL field, registry support and OCI spec update needed
paramManifestDigest = "digest"
)
// Reg is used for interacting with remote registry servers
type Reg struct {
reghttp *reghttp.Client
reghttpOpts []reghttp.Opts
slog *slog.Logger
hosts map[string]*config.Host
hostDefault *config.Host
features map[featureKey]*featureVal
blobChunkSize int64
blobChunkLimit int64
blobMaxPut int64
manifestMaxPull int64
manifestMaxPush int64
cacheMan *cache.Cache[ref.Ref, manifest.Manifest]
cacheRL *cache.Cache[ref.Ref, referrer.ReferrerList]
muHost sync.Mutex
muRefTag sync.Mutex
}
type featureKey struct {
kind string
reg string
repo string
}
type featureVal struct {
enabled bool
expire time.Time
}
var featureExpire = time.Minute * time.Duration(5)
// Opts provides options to access registries
type Opts func(*Reg)
// New returns a Reg pointer with any provided options
func New(opts ...Opts) *Reg {
r := Reg{
reghttpOpts: []reghttp.Opts{},
blobChunkSize: defaultBlobChunk,
blobChunkLimit: defaultBlobChunkLimit,
blobMaxPut: defaultBlobMax,
manifestMaxPull: defaultManifestMaxPull,
manifestMaxPush: defaultManifestMaxPush,
hosts: map[string]*config.Host{},
features: map[featureKey]*featureVal{},
}
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithConfigHostFn(r.hostGet))
for _, opt := range opts {
opt(&r)
}
r.reghttp = reghttp.NewClient(r.reghttpOpts...)
return &r
}
// Throttle is used to limit concurrency
func (reg *Reg) Throttle(r ref.Ref, put bool) []*pqueue.Queue[reqmeta.Data] {
tList := []*pqueue.Queue[reqmeta.Data]{}
host := reg.hostGet(r.Registry)
t := reg.reghttp.GetThrottle(r.Registry)
if t != nil {
tList = append(tList, t)
}
if !put {
for _, mirror := range host.Mirrors {
t := reg.reghttp.GetThrottle(mirror)
if t != nil {
tList = append(tList, t)
}
}
}
return tList
}
func (reg *Reg) hostGet(hostname string) *config.Host {
reg.muHost.Lock()
defer reg.muHost.Unlock()
if _, ok := reg.hosts[hostname]; !ok {
newHost := config.HostNewDefName(reg.hostDefault, hostname)
// check for normalized hostname
if newHost.Name != hostname {
hostname = newHost.Name
if h, ok := reg.hosts[hostname]; ok {
return h
}
}
reg.hosts[hostname] = newHost
}
return reg.hosts[hostname]
}
// featureGet returns enabled and ok
func (reg *Reg) featureGet(kind, registry, repo string) (bool, bool) {
reg.muHost.Lock()
defer reg.muHost.Unlock()
if v, ok := reg.features[featureKey{kind: kind, reg: registry, repo: repo}]; ok {
if time.Now().Before(v.expire) {
return v.enabled, true
}
}
return false, false
}
func (reg *Reg) featureSet(kind, registry, repo string, enabled bool) {
reg.muHost.Lock()
reg.features[featureKey{kind: kind, reg: registry, repo: repo}] = &featureVal{enabled: enabled, expire: time.Now().Add(featureExpire)}
reg.muHost.Unlock()
}
// WithBlobSize overrides default blob sizes
func WithBlobSize(size, max int64) Opts {
return func(r *Reg) {
if size > 0 {
r.blobChunkSize = size
}
if max != 0 {
r.blobMaxPut = max
}
}
}
// WithBlobLimit overrides default blob limit
func WithBlobLimit(limit int64) Opts {
return func(r *Reg) {
if limit > 0 {
r.blobChunkLimit = limit
}
if r.blobMaxPut > 0 && r.blobMaxPut < limit {
r.blobMaxPut = limit
}
}
}
// WithCache defines a cache used for various requests
func WithCache(timeout time.Duration, count int) Opts {
return func(r *Reg) {
cm := cache.New[ref.Ref, manifest.Manifest](cache.WithAge(timeout), cache.WithCount(count))
r.cacheMan = &cm
crl := cache.New[ref.Ref, referrer.ReferrerList](cache.WithAge(timeout), cache.WithCount(count))
r.cacheRL = &crl
}
}
// WithCerts adds certificates
func WithCerts(certs [][]byte) Opts {
return func(r *Reg) {
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithCerts(certs))
}
}
// WithCertDirs adds certificate directories for host specific certs
func WithCertDirs(dirs []string) Opts {
return func(r *Reg) {
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithCertDirs(dirs))
}
}
// WithCertFiles adds certificates by filename
func WithCertFiles(files []string) Opts {
return func(r *Reg) {
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithCertFiles(files))
}
}
// WithConfigHostDefault provides default settings for hosts.
func WithConfigHostDefault(ch *config.Host) Opts {
return func(r *Reg) {
r.hostDefault = ch
}
}
// WithConfigHosts adds host configs for credentials
func WithConfigHosts(configHosts []*config.Host) Opts {
return func(r *Reg) {
for _, host := range configHosts {
if host.Name == "" {
continue
}
r.hosts[host.Name] = host
}
}
}
// WithDelay initial time to wait between retries (increased with exponential backoff)
func WithDelay(delayInit time.Duration, delayMax time.Duration) Opts {
return func(r *Reg) {
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithDelay(delayInit, delayMax))
}
}
// WithHTTPClient uses a specific http client with retryable requests
func WithHTTPClient(hc *http.Client) Opts {
return func(r *Reg) {
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithHTTPClient(hc))
}
}
// WithManifestMax sets the push and pull limits for manifests
func WithManifestMax(push, pull int64) Opts {
return func(r *Reg) {
r.manifestMaxPush = push
r.manifestMaxPull = pull
}
}
// WithRetryLimit restricts the number of retries (defaults to 5)
func WithRetryLimit(l int) Opts {
return func(r *Reg) {
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithRetryLimit(l))
}
}
// WithSlog injects a slog Logger configuration
func WithSlog(slog *slog.Logger) Opts {
return func(r *Reg) {
r.slog = slog
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithLog(slog))
}
}
// WithTransport uses a specific http transport with retryable requests
func WithTransport(t *http.Transport) Opts {
return func(r *Reg) {
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithTransport(t))
}
}
// WithUserAgent sets a user agent header
func WithUserAgent(ua string) Opts {
return func(r *Reg) {
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithUserAgent(ua))
}
}
+20
View File
@@ -0,0 +1,20 @@
//go:build !wasm
package reg
import (
"log/slog"
"github.com/sirupsen/logrus"
"github.com/regclient/regclient/internal/reghttp"
"github.com/regclient/regclient/internal/sloghandle"
)
// WithLog injects a logrus Logger configuration
func WithLog(log *logrus.Logger) Opts {
return func(r *Reg) {
r.slog = slog.New(sloghandle.Logrus(log))
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithLog(r.slog))
}
}
+79
View File
@@ -0,0 +1,79 @@
package reg
import (
"context"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"strconv"
"github.com/regclient/regclient/internal/reghttp"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/mediatype"
"github.com/regclient/regclient/types/repo"
)
// RepoList returns a list of repositories on a registry
// Note the underlying "_catalog" API is not supported on many cloud registries
func (reg *Reg) RepoList(ctx context.Context, hostname string, opts ...scheme.RepoOpts) (*repo.RepoList, error) {
config := scheme.RepoConfig{}
for _, opt := range opts {
opt(&config)
}
query := url.Values{}
if config.Last != "" {
query.Set("last", config.Last)
}
if config.Limit > 0 {
query.Set("n", strconv.Itoa(config.Limit))
}
headers := http.Header{
"Accept": []string{"application/json"},
}
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: hostname,
NoMirrors: true,
Method: "GET",
Path: "_catalog",
NoPrefix: true,
Query: query,
Headers: headers,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to list repositories for %s: %w", hostname, err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 200 {
return nil, fmt.Errorf("failed to list repositories for %s: %w", hostname, reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
respBody, err := io.ReadAll(resp)
if err != nil {
reg.slog.Warn("Failed to read repo list",
slog.String("err", err.Error()),
slog.String("host", hostname))
return nil, fmt.Errorf("failed to read repo list for %s: %w", hostname, err)
}
mt := mediatype.Base(resp.HTTPResponse().Header.Get("Content-Type"))
rl, err := repo.New(
repo.WithMT(mt),
repo.WithRaw(respBody),
repo.WithHost(hostname),
repo.WithHeaders(resp.HTTPResponse().Header),
)
if err != nil {
reg.slog.Warn("Failed to unmarshal repo list",
slog.String("err", err.Error()),
slog.String("body", string(respBody)),
slog.String("host", hostname))
return nil, fmt.Errorf("failed to parse repo list for %s: %w", hostname, err)
}
return rl, nil
}
+343
View File
@@ -0,0 +1,343 @@
package reg
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"strconv"
"time"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/internal/httplink"
"github.com/regclient/regclient/internal/reghttp"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/docker/schema2"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/mediatype"
v1 "github.com/regclient/regclient/types/oci/v1"
"github.com/regclient/regclient/types/platform"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/tag"
"github.com/regclient/regclient/types/warning"
)
// TagDelete removes a tag from a repository.
// It first attempts the newer OCI API to delete by tag name (not widely supported).
// If the OCI API fails, it falls back to pushing a unique empty manifest and deleting that.
func (reg *Reg) TagDelete(ctx context.Context, r ref.Ref) error {
var tempManifest manifest.Manifest
if r.Tag == "" {
return errs.ErrMissingTag
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
// attempt to delete the tag directly, available in OCI distribution-spec, and Hub API
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
NoMirrors: true,
Method: "DELETE",
Repository: r.Repository,
Path: "manifests/" + r.Tag,
IgnoreErr: true, // do not trigger backoffs if this fails
}
resp, err := reg.reghttp.Do(ctx, req)
if resp != nil {
defer resp.Close()
}
if err == nil && resp != nil && resp.HTTPResponse().StatusCode == 202 {
return nil
}
// ignore errors, fallback to creating a temporary manifest to replace the tag and deleting that manifest
// lookup the current manifest media type
curManifest, err := reg.ManifestHead(ctx, r)
if err != nil && errors.Is(err, errs.ErrUnsupportedAPI) {
curManifest, err = reg.ManifestGet(ctx, r)
}
if err != nil {
return err
}
// create empty image config with single label
// Note, this should be MediaType specific, but it appears that docker uses OCI for the config
now := time.Now()
conf := v1.Image{
Created: &now,
Config: v1.ImageConfig{
Labels: map[string]string{
"delete-tag": r.Tag,
"delete-date": now.String(),
},
},
Platform: platform.Platform{
OS: "linux",
Architecture: "amd64",
},
History: []v1.History{
{
Created: &now,
CreatedBy: "# regclient",
Comment: "empty JSON blob",
},
},
RootFS: v1.RootFS{
Type: "layers",
DiffIDs: []digest.Digest{
descriptor.EmptyDigest,
},
},
}
confB, err := json.Marshal(conf)
if err != nil {
return err
}
digester := digest.Canonical.Digester()
confBuf := bytes.NewBuffer(confB)
_, err = confBuf.WriteTo(digester.Hash())
if err != nil {
return err
}
confDigest := digester.Digest()
// create manifest with config, matching the original tag manifest type
switch manifest.GetMediaType(curManifest) {
case mediatype.OCI1Manifest, mediatype.OCI1ManifestList:
tempManifest, err = manifest.New(manifest.WithOrig(v1.Manifest{
Versioned: v1.ManifestSchemaVersion,
MediaType: mediatype.OCI1Manifest,
Config: descriptor.Descriptor{
MediaType: mediatype.OCI1ImageConfig,
Digest: confDigest,
Size: int64(len(confB)),
},
Layers: []descriptor.Descriptor{
{
MediaType: mediatype.OCI1Layer,
Size: int64(len(descriptor.EmptyData)),
Digest: descriptor.EmptyDigest,
},
},
}))
if err != nil {
return err
}
default: // default to the docker v2 schema
tempManifest, err = manifest.New(manifest.WithOrig(schema2.Manifest{
Versioned: schema2.ManifestSchemaVersion,
Config: descriptor.Descriptor{
MediaType: mediatype.Docker2ImageConfig,
Digest: confDigest,
Size: int64(len(confB)),
},
Layers: []descriptor.Descriptor{
{
MediaType: mediatype.Docker2LayerGzip,
Size: int64(len(descriptor.EmptyData)),
Digest: descriptor.EmptyDigest,
},
},
}))
if err != nil {
return err
}
}
reg.slog.Debug("Sending dummy manifest to replace tag",
slog.String("ref", r.Reference))
// push empty layer
_, err = reg.BlobPut(ctx, r, descriptor.Descriptor{Digest: descriptor.EmptyDigest, Size: int64(len(descriptor.EmptyData))}, bytes.NewReader(descriptor.EmptyData))
if err != nil {
return err
}
// push config
_, err = reg.BlobPut(ctx, r, descriptor.Descriptor{Digest: confDigest, Size: int64(len(confB))}, bytes.NewReader(confB))
if err != nil {
return fmt.Errorf("failed sending dummy config to delete %s: %w", r.CommonName(), err)
}
// push manifest to tag
err = reg.ManifestPut(ctx, r, tempManifest)
if err != nil {
return fmt.Errorf("failed sending dummy manifest to delete %s: %w", r.CommonName(), err)
}
// delete manifest by digest
r = r.AddDigest(tempManifest.GetDescriptor().Digest.String())
reg.slog.Debug("Deleting dummy manifest",
slog.String("ref", r.Reference),
slog.String("digest", r.Digest))
err = reg.ManifestDelete(ctx, r)
if err != nil {
return fmt.Errorf("failed deleting dummy manifest for %s: %w", r.CommonName(), err)
}
return nil
}
// TagList returns a listing to tags from the repository
func (reg *Reg) TagList(ctx context.Context, r ref.Ref, opts ...scheme.TagOpts) (*tag.List, error) {
var config scheme.TagConfig
for _, opt := range opts {
opt(&config)
}
tl, err := reg.tagListOCI(ctx, r, config)
if err != nil {
return tl, err
}
for {
// if limit reached, stop searching
if config.Limit > 0 && len(tl.Tags) >= config.Limit {
break
}
tlHead, err := tl.RawHeaders()
if err != nil {
return tl, err
}
links, err := httplink.Parse(tlHead.Values("Link"))
if err != nil {
return tl, err
}
next, err := links.Get("rel", "next")
// if Link header with rel="next" is defined
if err == nil {
link := tl.GetURL()
if link == nil {
return tl, fmt.Errorf("tag list, failed to get URL of previous request")
}
link, err = link.Parse(next.URI)
if err != nil {
return tl, fmt.Errorf("tag list failed to parse Link: %w", err)
}
tlAdd, err := reg.tagListLink(ctx, r, config, link)
if err != nil {
return tl, fmt.Errorf("tag list failed to get Link: %w", err)
}
err = tl.Append(tlAdd)
if err != nil {
return tl, fmt.Errorf("tag list failed to append entries: %w", err)
}
} else {
// do not automatically expand tags with OCI methods,
// OCI registries should send all possible entries up to the specified limit
break
}
}
return tl, nil
}
func (reg *Reg) tagListOCI(ctx context.Context, r ref.Ref, config scheme.TagConfig) (*tag.List, error) {
query := url.Values{}
if config.Last != "" {
query.Set("last", config.Last)
}
if config.Limit > 0 {
query.Set("n", strconv.Itoa(config.Limit))
}
headers := http.Header{
"Accept": []string{"application/json"},
}
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
Method: "GET",
Repository: r.Repository,
Path: "tags/list",
Query: query,
Headers: headers,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to list tags for %s: %w", r.CommonName(), err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 200 {
return nil, fmt.Errorf("failed to list tags for %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
respBody, err := io.ReadAll(resp)
if err != nil {
reg.slog.Warn("Failed to read tag list",
slog.String("err", err.Error()),
slog.String("ref", r.CommonName()))
return nil, fmt.Errorf("failed to read tags for %s: %w", r.CommonName(), err)
}
tl, err := tag.New(
tag.WithRef(r),
tag.WithRaw(respBody),
tag.WithResp(resp.HTTPResponse()),
)
if err != nil {
reg.slog.Warn("Failed to unmarshal tag list",
slog.String("err", err.Error()),
slog.String("body", string(respBody)),
slog.String("ref", r.CommonName()))
return tl, fmt.Errorf("failed to unmarshal tag list for %s: %w", r.CommonName(), err)
}
return tl, nil
}
func (reg *Reg) tagListLink(ctx context.Context, r ref.Ref, _ scheme.TagConfig, link *url.URL) (*tag.List, error) {
headers := http.Header{
"Accept": []string{"application/json"},
}
req := &reghttp.Req{
MetaKind: reqmeta.Query,
Host: r.Registry,
Method: "GET",
DirectURL: link,
Repository: r.Repository,
Headers: headers,
}
resp, err := reg.reghttp.Do(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to list tags for %s: %w", r.CommonName(), err)
}
defer resp.Close()
if resp.HTTPResponse().StatusCode != 200 {
return nil, fmt.Errorf("failed to list tags for %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
}
respBody, err := io.ReadAll(resp)
if err != nil {
reg.slog.Warn("Failed to read tag list",
slog.String("err", err.Error()),
slog.String("ref", r.CommonName()))
return nil, fmt.Errorf("failed to read tags for %s: %w", r.CommonName(), err)
}
tl, err := tag.New(
tag.WithRef(r),
tag.WithRaw(respBody),
tag.WithResp(resp.HTTPResponse()),
)
if err != nil {
reg.slog.Warn("Failed to unmarshal tag list",
slog.String("err", err.Error()),
slog.String("body", string(respBody)),
slog.String("ref", r.CommonName()))
return tl, fmt.Errorf("failed to unmarshal tag list for %s: %w", r.CommonName(), err)
}
return tl, nil
}
+236
View File
@@ -0,0 +1,236 @@
// Package scheme defines the interface for various reference schemes.
package scheme
import (
"context"
"io"
"github.com/regclient/regclient/internal/pqueue"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/types/blob"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/ping"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/referrer"
"github.com/regclient/regclient/types/tag"
)
// API is used to interface between different methods to store images.
type API interface {
// BlobDelete removes a blob from the repository.
BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error
// BlobGet retrieves a blob, returning a reader.
BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error)
// BlobHead verifies the existence of a blob, the reader contains the headers but no body to read.
BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error)
// BlobMount attempts to perform a server side copy of the blob.
BlobMount(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor) error
// BlobPut sends a blob to the repository, returns the digest and size when successful.
BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error)
// ManifestDelete removes a manifest, including all tags that point to that manifest.
ManifestDelete(ctx context.Context, r ref.Ref, opts ...ManifestOpts) error
// ManifestGet retrieves a manifest from a repository.
ManifestGet(ctx context.Context, r ref.Ref) (manifest.Manifest, error)
// ManifestHead gets metadata about the manifest (existence, digest, mediatype, size).
ManifestHead(ctx context.Context, r ref.Ref) (manifest.Manifest, error)
// ManifestPut sends a manifest to the repository.
ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...ManifestOpts) error
// Ping verifies access to a registry or equivalent.
Ping(ctx context.Context, r ref.Ref) (ping.Result, error)
// ReferrerList returns a list of referrers to a given reference.
ReferrerList(ctx context.Context, r ref.Ref, opts ...ReferrerOpts) (referrer.ReferrerList, error)
// TagDelete removes a tag from the repository.
TagDelete(ctx context.Context, r ref.Ref) error
// TagList returns a list of tags from the repository.
TagList(ctx context.Context, r ref.Ref, opts ...TagOpts) (*tag.List, error)
}
// Closer is used to check if a scheme implements the Close API.
type Closer interface {
Close(ctx context.Context, r ref.Ref) error
}
// GCLocker is used to indicate locking is available for GC management.
type GCLocker interface {
// GCLock a reference to prevent GC from triggering during a put, locks are not exclusive.
GCLock(r ref.Ref)
// GCUnlock a reference to allow GC (once all locks are released).
// The reference should be closed after this step and unlock should only be called once per each Lock call.
GCUnlock(r ref.Ref)
}
// Throttler is used to indicate the scheme implements Throttle.
type Throttler interface {
Throttle(r ref.Ref, put bool) []*pqueue.Queue[reqmeta.Data]
}
// ManifestConfig is used by schemes to import [ManifestOpts].
type ManifestConfig struct {
CheckReferrers bool
Child bool // used when pushing a child of a manifest list, skips indexing in ocidir
Manifest manifest.Manifest
}
// ManifestOpts is used to set options on manifest APIs.
type ManifestOpts func(*ManifestConfig)
// WithManifestCheckReferrers is used when deleting a manifest.
// It indicates the manifest should be fetched and referrers should be deleted if defined.
func WithManifestCheckReferrers() ManifestOpts {
return func(config *ManifestConfig) {
config.CheckReferrers = true
}
}
// WithManifestChild indicates the API call is on a child manifest.
// This is used internally when copying multi-platform manifests.
// This bypasses tracking of an untagged digest in ocidir which is needed for garbage collection.
func WithManifestChild() ManifestOpts {
return func(config *ManifestConfig) {
config.Child = true
}
}
// WithManifest is used to pass the manifest to a method to avoid an extra GET request.
// This is used on a delete to check for referrers.
func WithManifest(m manifest.Manifest) ManifestOpts {
return func(mc *ManifestConfig) {
mc.Manifest = m
}
}
// ReferrerConfig is used by schemes to import [ReferrerOpts].
type ReferrerConfig struct {
MatchOpt descriptor.MatchOpt // filter/sort results
Platform string // get referrers for a specific platform
SlowSearch bool // search every manifest in an OCI Layout index.json file
SrcRepo ref.Ref // repo used to query referrers
}
// ReferrerOpts is used to set options on referrer APIs.
type ReferrerOpts func(*ReferrerConfig)
// WithReferrerMatchOpt filters results using [descriptor.MatchOpt].
func WithReferrerMatchOpt(mo descriptor.MatchOpt) ReferrerOpts {
return func(config *ReferrerConfig) {
config.MatchOpt = config.MatchOpt.Merge(mo)
}
}
// WithReferrerPlatform gets referrers for a single platform from a multi-platform manifest.
// Note that this is implemented by [regclient.ReferrerList] and not the individual scheme implementations.
func WithReferrerPlatform(p string) ReferrerOpts {
return func(config *ReferrerConfig) {
config.Platform = p
}
}
// WithReferrerSlowSearch looks for referrers in ways that increases compatibility but takes longer.
// When used with an OCI Layout, this searches every manifest for a subject for ORAS compatibility.
func WithReferrerSlowSearch() ReferrerOpts {
return func(config *ReferrerConfig) {
config.SlowSearch = true
}
}
// WithReferrerSource pulls referrers from a separate source.
// Note that this is implemented by [regclient.ReferrerList] and not the individual scheme implementations.
func WithReferrerSource(r ref.Ref) ReferrerOpts {
return func(config *ReferrerConfig) {
config.SrcRepo = r
}
}
// WithReferrerAT filters by a specific artifactType value.
//
// Deprecated: replace with [WithReferrerMatchOpt].
//
//go:fix inline
func WithReferrerAT(at string) ReferrerOpts {
return WithReferrerMatchOpt(descriptor.MatchOpt{ArtifactType: at})
}
// WithReferrerAnnotations filters by a list of annotations, all of which must match.
//
// Deprecated: replace with [WithReferrerMatchOpt].
//
//go:fix inline
func WithReferrerAnnotations(annotations map[string]string) ReferrerOpts {
return WithReferrerMatchOpt(descriptor.MatchOpt{Annotations: annotations})
}
// WithReferrerSort orders the resulting referrers listing according to a specified annotation.
//
// Deprecated: replace with [WithReferrerMatchOpt].
//
//go:fix inline
func WithReferrerSort(annotation string, desc bool) ReferrerOpts {
return WithReferrerMatchOpt(descriptor.MatchOpt{SortAnnotation: annotation, SortDesc: desc})
}
// ReferrerFilter filters the referrer list according to the config.
func ReferrerFilter(config ReferrerConfig, rlIn referrer.ReferrerList) referrer.ReferrerList {
return referrer.ReferrerList{
Subject: rlIn.Subject,
Source: rlIn.Source,
Manifest: rlIn.Manifest,
Annotations: rlIn.Annotations,
Tags: rlIn.Tags,
Descriptors: descriptor.DescriptorListFilter(rlIn.Descriptors, config.MatchOpt),
}
}
// RepoConfig is used by schemes to import [RepoOpts].
type RepoConfig struct {
Limit int
Last string
}
// RepoOpts is used to set options on repo APIs.
type RepoOpts func(*RepoConfig)
// WithRepoLimit passes a maximum number of repositories to return to the repository list API.
// Registries may ignore this.
func WithRepoLimit(l int) RepoOpts {
return func(config *RepoConfig) {
config.Limit = l
}
}
// WithRepoLast passes the last received repository for requesting the next batch of repositories.
// Registries may ignore this.
func WithRepoLast(l string) RepoOpts {
return func(config *RepoConfig) {
config.Last = l
}
}
// TagConfig is used by schemes to import [TagOpts].
type TagConfig struct {
Limit int
Last string
}
// TagOpts is used to set options on tag APIs.
type TagOpts func(*TagConfig)
// WithTagLimit passes a maximum number of tags to return to the tag list API.
// Registries may ignore this.
func WithTagLimit(limit int) TagOpts {
return func(t *TagConfig) {
t.Limit = limit
}
}
// WithTagLast passes the last received tag for requesting the next batch of tags.
// Registries may ignore this.
func WithTagLast(last string) TagOpts {
return func(t *TagConfig) {
t.Last = last
}
}
+40
View File
@@ -0,0 +1,40 @@
package regclient
import (
"context"
"fmt"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/tag"
)
// TagDelete deletes a tag from the registry. Since there's no API for this,
// you'd want to normally just delete the manifest. However multiple tags may
// point to the same manifest, so instead you must:
// 1. Make a manifest, for this we put a few labels and timestamps to be unique.
// 2. Push that manifest to the tag.
// 3. Delete the digest for that new manifest that is only used by that tag.
func (rc *RegClient) TagDelete(ctx context.Context, r ref.Ref) error {
if !r.IsSet() {
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return err
}
return schemeAPI.TagDelete(ctx, r)
}
// TagList returns a tag list from a repository
func (rc *RegClient) TagList(ctx context.Context, r ref.Ref, opts ...scheme.TagOpts) (*tag.List, error) {
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
return schemeAPI.TagList(ctx, r, opts...)
}
+72
View File
@@ -0,0 +1,72 @@
// Content in this file comes from OCI
// Copyright 2016 The Linux Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
const (
// AnnotationCreated is the annotation key for the date and time on which the image was built (date-time string as defined by RFC 3339).
AnnotationCreated = "org.opencontainers.image.created"
// AnnotationAuthors is the annotation key for the contact details of the people or organization responsible for the image (freeform string).
AnnotationAuthors = "org.opencontainers.image.authors"
// AnnotationURL is the annotation key for the URL to find more information on the image.
AnnotationURL = "org.opencontainers.image.url"
// AnnotationDocumentation is the annotation key for the URL to get documentation on the image.
AnnotationDocumentation = "org.opencontainers.image.documentation"
// AnnotationSource is the annotation key for the URL to get source code for building the image.
AnnotationSource = "org.opencontainers.image.source"
// AnnotationVersion is the annotation key for the version of the packaged software.
// The version MAY match a label or tag in the source code repository.
// The version MAY be Semantic versioning-compatible.
AnnotationVersion = "org.opencontainers.image.version"
// AnnotationRevision is the annotation key for the source control revision identifier for the packaged software.
AnnotationRevision = "org.opencontainers.image.revision"
// AnnotationVendor is the annotation key for the name of the distributing entity, organization or individual.
AnnotationVendor = "org.opencontainers.image.vendor"
// AnnotationLicenses is the annotation key for the license(s) under which contained software is distributed as an SPDX License Expression.
AnnotationLicenses = "org.opencontainers.image.licenses"
// AnnotationRefName is the annotation key for the name of the reference for a target.
// SHOULD only be considered valid when on descriptors on `index.json` within image layout.
AnnotationRefName = "org.opencontainers.image.ref.name"
// AnnotationTitle is the annotation key for the human-readable title of the image.
AnnotationTitle = "org.opencontainers.image.title"
// AnnotationDescription is the annotation key for the human-readable description of the software packaged in the image.
AnnotationDescription = "org.opencontainers.image.description"
// AnnotationBaseImageDigest is the annotation key for the digest of the image's base image.
AnnotationBaseImageDigest = "org.opencontainers.image.base.digest"
// AnnotationBaseImageName is the annotation key for the image reference of the image's base image.
AnnotationBaseImageName = "org.opencontainers.image.base.name"
// AnnotationArtifactCreated is the annotation key for the date and time on which the artifact was built, conforming to RFC 3339.
AnnotationArtifactCreated = "org.opencontainers.artifact.created"
// AnnotationArtifactDescription is the annotation key for the human readable description for the artifact.
AnnotationArtifactDescription = "org.opencontainers.artifact.description"
// AnnotationReferrersFiltersApplied is the annotation key for the comma separated list of filters applied by the registry in the referrers listing.
AnnotationReferrersFiltersApplied = "org.opencontainers.referrers.filtersApplied"
)
+103
View File
@@ -0,0 +1,103 @@
// Package blob is the underlying type for pushing and pulling blobs.
package blob
import (
"io"
"net/http"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/types/descriptor"
v1 "github.com/regclient/regclient/types/oci/v1"
"github.com/regclient/regclient/types/ref"
)
// Blob interface is used for returning blobs.
type Blob interface {
// GetDescriptor returns the descriptor associated with the blob.
GetDescriptor() descriptor.Descriptor
// RawBody returns the raw content of the blob.
RawBody() ([]byte, error)
// RawHeaders returns the headers received from the registry.
RawHeaders() http.Header
// Response returns the response associated with the blob.
Response() *http.Response
// Digest returns the provided or calculated digest of the blob.
//
// Deprecated: Digest should be replaced by GetDescriptor().Digest.
Digest() digest.Digest
// Length returns the provided or calculated length of the blob.
//
// Deprecated: Length should be replaced by GetDescriptor().Size.
Length() int64
// MediaType returns the Content-Type header received from the registry.
//
// Deprecated: MediaType should be replaced by GetDescriptor().MediaType.
MediaType() string
}
type blobConfig struct {
desc descriptor.Descriptor
header http.Header
image *v1.Image
r ref.Ref
rdr io.Reader
resp *http.Response
rawBody []byte
}
// Opts is used for options to create a new blob.
type Opts func(*blobConfig)
// WithDesc specifies the descriptor associated with the blob.
func WithDesc(d descriptor.Descriptor) Opts {
return func(bc *blobConfig) {
bc.desc = d
}
}
// WithHeader defines the headers received when pulling a blob.
func WithHeader(header http.Header) Opts {
return func(bc *blobConfig) {
bc.header = header
}
}
// WithImage provides the OCI Image config needed for config blobs.
func WithImage(image v1.Image) Opts {
return func(bc *blobConfig) {
bc.image = &image
}
}
// WithRawBody defines the raw blob contents for OCIConfig.
func WithRawBody(raw []byte) Opts {
return func(bc *blobConfig) {
bc.rawBody = raw
}
}
// WithReader defines the reader for a new blob.
func WithReader(rc io.Reader) Opts {
return func(bc *blobConfig) {
bc.rdr = rc
}
}
// WithRef specifies the reference where the blob was pulled from.
func WithRef(r ref.Ref) Opts {
return func(bc *blobConfig) {
bc.r = r
}
}
// WithResp includes the http response, which is used to extract the headers and reader.
func WithResp(resp *http.Response) Opts {
return func(bc *blobConfig) {
bc.resp = resp
if bc.header == nil && resp != nil {
bc.header = resp.Header
}
}
}
+68
View File
@@ -0,0 +1,68 @@
package blob
import (
"net/http"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/ref"
)
// Common was previously an interface. A type alias is provided for upgrades.
type Common = *BCommon
// BCommon is a common struct for all blobs which includes various shared methods.
type BCommon struct {
r ref.Ref
desc descriptor.Descriptor
blobSet bool
rawHeader http.Header
resp *http.Response
}
// GetDescriptor returns the descriptor associated with the blob.
func (c *BCommon) GetDescriptor() descriptor.Descriptor {
return c.desc
}
// Digest returns the provided or calculated digest of the blob.
//
// Deprecated: Digest should be replaced by GetDescriptor().Digest, see [GetDescriptor].
//
//go:fix inline
func (c *BCommon) Digest() digest.Digest {
return c.desc.Digest
}
// Length returns the provided or calculated length of the blob.
//
// Deprecated: Length should be replaced by GetDescriptor().Size, see [GetDescriptor].
//
//go:fix inline
func (c *BCommon) Length() int64 {
return c.desc.Size
}
// MediaType returns the Content-Type header received from the registry.
//
// Deprecated: MediaType should be replaced by GetDescriptor().MediaType, see [GetDescriptor].
//
//go:fix inline
func (c *BCommon) MediaType() string {
return c.desc.MediaType
}
// RawHeaders returns the headers received from the registry.
func (c *BCommon) RawHeaders() http.Header {
return c.rawHeader
}
// Response returns the response associated with the blob.
func (c *BCommon) Response() *http.Response {
return c.resp
}
+127
View File
@@ -0,0 +1,127 @@
package blob
import (
"encoding/json"
"fmt"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/regclient/regclient/types/mediatype"
v1 "github.com/regclient/regclient/types/oci/v1"
)
// OCIConfig was previously an interface. A type alias is provided for upgrading.
type OCIConfig = *BOCIConfig
// BOCIConfig includes an OCI Image Config struct that may be extracted from or pushed to a blob.
type BOCIConfig struct {
BCommon
rawBody []byte
image v1.Image
}
// NewOCIConfig creates a new BOCIConfig.
// When created from an existing blob, a BOCIConfig will be created using BReader.ToOCIConfig().
func NewOCIConfig(opts ...Opts) *BOCIConfig {
bc := blobConfig{}
for _, opt := range opts {
opt(&bc)
}
if bc.image != nil && len(bc.rawBody) == 0 {
var err error
bc.rawBody, err = json.Marshal(bc.image)
if err != nil {
bc.rawBody = []byte{}
}
}
if len(bc.rawBody) > 0 {
if bc.image == nil {
bc.image = &v1.Image{}
err := json.Unmarshal(bc.rawBody, bc.image)
if err != nil {
bc.image = nil
}
}
// force descriptor to match raw body, even if we generated the raw body
bc.desc.Digest = bc.desc.DigestAlgo().FromBytes(bc.rawBody)
bc.desc.Size = int64(len(bc.rawBody))
if bc.desc.MediaType == "" {
bc.desc.MediaType = mediatype.OCI1ImageConfig
}
}
b := BOCIConfig{
BCommon: BCommon{
desc: bc.desc,
r: bc.r,
rawHeader: bc.header,
resp: bc.resp,
},
rawBody: bc.rawBody,
}
if bc.image != nil {
b.image = *bc.image
b.blobSet = true
}
return &b
}
// GetConfig returns OCI config.
func (oc *BOCIConfig) GetConfig() v1.Image {
return oc.image
}
// RawBody returns the original body from the request.
func (oc *BOCIConfig) RawBody() ([]byte, error) {
var err error
if !oc.blobSet {
return []byte{}, fmt.Errorf("Blob is not defined")
}
if len(oc.rawBody) == 0 {
oc.rawBody, err = json.Marshal(oc.image)
}
return oc.rawBody, err
}
// SetConfig updates the config, including raw body and descriptor.
func (oc *BOCIConfig) SetConfig(image v1.Image) {
oc.image = image
oc.rawBody, _ = json.Marshal(oc.image)
if oc.desc.MediaType == "" {
oc.desc.MediaType = mediatype.OCI1ImageConfig
}
oc.desc.Digest = oc.desc.DigestAlgo().FromBytes(oc.rawBody)
oc.desc.Size = int64(len(oc.rawBody))
oc.blobSet = true
}
// MarshalJSON passes through the marshalling to the underlying image if rawBody is not available.
func (oc *BOCIConfig) MarshalJSON() ([]byte, error) {
if !oc.blobSet {
return []byte{}, fmt.Errorf("Blob is not defined")
}
if len(oc.rawBody) > 0 {
return oc.rawBody, nil
}
return json.Marshal(oc.image)
}
// UnmarshalJSON extracts json content and populates the content.
func (oc *BOCIConfig) UnmarshalJSON(data []byte) error {
image := v1.Image{}
err := json.Unmarshal(data, &image)
if err != nil {
return err
}
oc.image = image
oc.rawBody = make([]byte, len(data))
copy(oc.rawBody, data)
if oc.desc.MediaType == "" {
oc.desc.MediaType = mediatype.OCI1ImageConfig
}
oc.desc.Digest = oc.desc.DigestAlgo().FromBytes(oc.rawBody)
oc.desc.Size = int64(len(oc.rawBody))
oc.blobSet = true
return nil
}
+210
View File
@@ -0,0 +1,210 @@
package blob
import (
"fmt"
"io"
"strconv"
"sync"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/internal/limitread"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/mediatype"
)
// Reader was previously an interface. A type alias is provided for upgrading.
type Reader = *BReader
// BReader is used to read blobs.
type BReader struct {
BCommon
readBytes int64
reader io.Reader
origRdr io.Reader
digester digest.Digester
mu sync.Mutex
}
// NewReader creates a new BReader.
func NewReader(opts ...Opts) *BReader {
bc := blobConfig{}
for _, opt := range opts {
opt(&bc)
}
if bc.resp != nil {
// extract headers and reader if other fields not passed
if bc.header == nil {
bc.header = bc.resp.Header
}
if bc.rdr == nil {
bc.rdr = bc.resp.Body
}
}
if bc.header != nil {
// extract fields from header if descriptor not passed
if bc.desc.MediaType == "" {
bc.desc.MediaType = mediatype.Base(bc.header.Get("Content-Type"))
}
if bc.desc.Size == 0 {
cl, _ := strconv.Atoi(bc.header.Get("Content-Length"))
bc.desc.Size = int64(cl)
}
if bc.desc.Digest == "" {
bc.desc.Digest, _ = digest.Parse(bc.header.Get("Docker-Content-Digest"))
}
}
br := BReader{
BCommon: BCommon{
r: bc.r,
desc: bc.desc,
rawHeader: bc.header,
resp: bc.resp,
},
origRdr: bc.rdr,
}
if bc.rdr != nil {
br.blobSet = true
br.digester = br.desc.DigestAlgo().Digester()
rdr := bc.rdr
if br.desc.Size > 0 {
rdr = &limitread.LimitRead{
Reader: rdr,
Limit: br.desc.Size,
}
}
br.reader = io.TeeReader(rdr, br.digester.Hash())
}
return &br
}
// Close attempts to close the reader and populates/validates the digest.
func (r *BReader) Close() error {
if r == nil || r.origRdr == nil {
return nil
}
// attempt to close if available in original reader
bc, ok := r.origRdr.(io.Closer)
if !ok {
return nil
}
return bc.Close()
}
// RawBody returns the original body from the request.
func (r *BReader) RawBody() ([]byte, error) {
return io.ReadAll(r)
}
// Read passes through the read operation while computing the digest and tracking the size.
func (r *BReader) Read(p []byte) (int, error) {
if r == nil || r.reader == nil {
return 0, fmt.Errorf("blob has no reader: %w", io.ErrUnexpectedEOF)
}
r.mu.Lock()
defer r.mu.Unlock()
size, err := r.reader.Read(p)
r.readBytes = r.readBytes + int64(size)
if err == io.EOF {
// check/save size
if r.desc.Size == 0 {
r.desc.Size = r.readBytes
} else if r.readBytes < r.desc.Size {
err = fmt.Errorf("%w [expected %d, received %d]: %w", errs.ErrShortRead, r.desc.Size, r.readBytes, err)
} else if r.readBytes > r.desc.Size {
err = fmt.Errorf("%w [expected %d, received %d]: %w", errs.ErrSizeLimitExceeded, r.desc.Size, r.readBytes, err)
}
// check/save digest
if r.desc.Digest.Validate() != nil {
r.desc.Digest = r.digester.Digest()
} else if r.desc.Digest != r.digester.Digest() {
err = fmt.Errorf("%w [expected %s, calculated %s]: %w", errs.ErrDigestMismatch, r.desc.Digest.String(), r.digester.Digest().String(), err)
}
}
return size, err
}
// Seek passes through the seek operation, reseting or invalidating the digest
func (r *BReader) Seek(offset int64, whence int) (int64, error) {
if r == nil || r.origRdr == nil {
return 0, fmt.Errorf("blob has no reader")
}
r.mu.Lock()
defer r.mu.Unlock()
if offset == 0 && whence == io.SeekCurrent {
return r.readBytes, nil
}
// cannot do an arbitrary seek and still digest without a lot more complication
if offset != 0 || whence != io.SeekStart {
return r.readBytes, fmt.Errorf("unable to seek to arbitrary position")
}
rdrSeek, ok := r.origRdr.(io.Seeker)
if !ok {
return r.readBytes, fmt.Errorf("Seek unsupported")
}
o, err := rdrSeek.Seek(offset, whence)
if err != nil || o != 0 {
return r.readBytes, err
}
// reset internal offset and digest calculation
rdr := r.origRdr
if r.desc.Size > 0 {
rdr = &limitread.LimitRead{
Reader: rdr,
Limit: r.desc.Size,
}
}
r.digester = r.desc.DigestAlgo().Digester()
r.reader = io.TeeReader(rdr, r.digester.Hash())
r.readBytes = 0
return 0, nil
}
// ToOCIConfig converts a BReader to a BOCIConfig.
func (r *BReader) ToOCIConfig() (*BOCIConfig, error) {
if r == nil || !r.blobSet {
return nil, fmt.Errorf("blob is not defined")
}
if r.readBytes != 0 {
return nil, fmt.Errorf("unable to convert after read has been performed")
}
blobBody, err := io.ReadAll(r)
errC := r.Close()
if err != nil {
return nil, fmt.Errorf("error reading image config for %s: %w", r.r.CommonName(), err)
}
if errC != nil {
return nil, fmt.Errorf("error closing blob reader: %w", err)
}
return NewOCIConfig(
WithDesc(r.desc),
WithHeader(r.rawHeader),
WithRawBody(blobBody),
WithRef(r.r),
WithResp(r.resp),
), nil
}
// ToTarReader converts a BReader to a BTarReader
func (r *BReader) ToTarReader() (*BTarReader, error) {
if r == nil || !r.blobSet {
return nil, fmt.Errorf("blob is not defined")
}
r.mu.Lock()
defer r.mu.Unlock()
if r.readBytes != 0 {
return nil, fmt.Errorf("unable to convert after read has been performed")
}
return NewTarReader(
WithDesc(r.desc),
WithHeader(r.rawHeader),
WithRef(r.r),
WithResp(r.resp),
WithReader(r.reader),
), nil
}
+191
View File
@@ -0,0 +1,191 @@
package blob
import (
"archive/tar"
"errors"
"fmt"
"io"
"path/filepath"
"strings"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/internal/limitread"
"github.com/regclient/regclient/pkg/archive"
"github.com/regclient/regclient/types/errs"
)
// TarReader was previously an interface. A type alias is provided for upgrading.
type TarReader = *BTarReader
// BTarReader is used to read individual files from an image layer.
type BTarReader struct {
BCommon
origRdr io.Reader
reader io.Reader
digester digest.Digester
tr *tar.Reader
}
// NewTarReader creates a BTarReader.
// Typically a BTarReader will be created using BReader.ToTarReader().
func NewTarReader(opts ...Opts) *BTarReader {
bc := blobConfig{}
for _, opt := range opts {
opt(&bc)
}
tr := BTarReader{
BCommon: BCommon{
desc: bc.desc,
r: bc.r,
rawHeader: bc.header,
resp: bc.resp,
},
origRdr: bc.rdr,
}
if bc.rdr != nil {
tr.blobSet = true
tr.digester = tr.desc.DigestAlgo().Digester()
rdr := bc.rdr
if tr.desc.Size > 0 {
rdr = &limitread.LimitRead{
Reader: rdr,
Limit: tr.desc.Size,
}
}
tr.reader = io.TeeReader(rdr, tr.digester.Hash())
}
return &tr
}
// Close attempts to close the reader and populates/validates the digest.
func (tr *BTarReader) Close() error {
// attempt to close if available in original reader
if trc, ok := tr.origRdr.(io.Closer); ok && trc != nil {
return trc.Close()
}
return nil
}
// GetTarReader returns the tar.Reader for the blob.
func (tr *BTarReader) GetTarReader() (*tar.Reader, error) {
if tr.reader == nil {
return nil, fmt.Errorf("blob has no reader defined")
}
if tr.tr == nil {
dr, err := archive.Decompress(tr.reader)
if err != nil {
return nil, err
}
tr.tr = tar.NewReader(dr)
}
return tr.tr, nil
}
// RawBody returns the original body from the request.
func (tr *BTarReader) RawBody() ([]byte, error) {
if !tr.blobSet {
return []byte{}, fmt.Errorf("Blob is not defined")
}
if tr.tr != nil {
return []byte{}, fmt.Errorf("RawBody cannot be returned after TarReader returned")
}
b, err := io.ReadAll(tr.reader)
if err != nil {
return b, err
}
if tr.digester != nil {
dig := tr.digester.Digest()
tr.digester = nil
if tr.desc.Digest.String() != "" && dig != tr.desc.Digest {
return b, fmt.Errorf("%w, expected %s, received %s", errs.ErrDigestMismatch, tr.desc.Digest.String(), dig.String())
}
tr.desc.Digest = dig
}
err = tr.Close()
return b, err
}
// ReadFile parses the tar to find a file.
func (tr *BTarReader) ReadFile(filename string) (*tar.Header, io.Reader, error) {
if strings.HasPrefix(filename, ".wh.") {
return nil, nil, fmt.Errorf(".wh. prefix is reserved for whiteout files")
}
// normalize filenames,
filename = filepath.Clean(filename)
if filename[0] == '/' {
filename = filename[1:]
}
// get reader
rdr, err := tr.GetTarReader()
if err != nil {
return nil, nil, err
}
// loop through files until whiteout or target file is found
whiteout := false
for {
th, err := rdr.Next()
if err != nil {
// break on eof, everything else is an error
if errors.Is(err, io.EOF) {
break
}
return nil, nil, err
}
thFile := filepath.Clean(th.Name)
if thFile[0] == '/' {
thFile = thFile[1:]
}
// found the target file
if thFile == filename {
return th, rdr, nil
}
// check/track whiteout file
name := filepath.Base(th.Name)
if !whiteout && strings.HasPrefix(name, ".wh.") && tarCmpWhiteout(th.Name, filename) {
// continue searching after finding a whiteout file
// a new file may be created in the same layer
whiteout = true
}
}
// EOF encountered
if whiteout {
return nil, nil, errs.ErrFileDeleted
}
if tr.digester != nil {
_, _ = io.Copy(io.Discard, tr.reader) // process/digest any trailing bytes from reader
dig := tr.digester.Digest()
tr.digester = nil
if tr.desc.Digest.String() != "" && dig != tr.desc.Digest {
return nil, nil, fmt.Errorf("%w, expected %s, received %s", errs.ErrDigestMismatch, tr.desc.Digest.String(), dig.String())
}
tr.desc.Digest = dig
}
return nil, nil, errs.ErrFileNotFound
}
func tarCmpWhiteout(whFile, tgtFile string) bool {
whSplit := strings.Split(whFile, "/")
tgtSplit := strings.Split(tgtFile, "/")
// the -1 handles the opaque whiteout
if len(whSplit)-1 > len(tgtSplit) {
return false
}
// verify the path matches up to the whiteout
for i := range whSplit[:len(whSplit)-1] {
if whSplit[i] != tgtSplit[i] {
return false
}
}
i := len(whSplit) - 1
// opaque whiteout of entire directory
if whSplit[i] == ".wh..wh..opq" {
return true
}
// compare whiteout name to next path entry
if i > len(tgtSplit)-1 {
return false
}
whName := strings.TrimPrefix(whSplit[i], ".wh.")
return whName == tgtSplit[i]
}
+29
View File
@@ -0,0 +1,29 @@
package types
type CallbackState int
const (
CallbackUndef CallbackState = iota
CallbackSkipped
CallbackStarted
CallbackActive
CallbackFinished
CallbackArchived
)
type CallbackKind int
const (
CallbackManifest CallbackKind = iota
CallbackBlob
)
func (k CallbackKind) String() string {
switch k {
case CallbackBlob:
return "blob"
case CallbackManifest:
return "manifest"
}
return "unknown"
}
+40
View File
@@ -0,0 +1,40 @@
package types
import "github.com/regclient/regclient/types/descriptor"
type (
// Descriptor is used in manifests to refer to content by media type, size, and digest.
//
// Deprecated: replace with [descriptor.Descriptor].
//go:fix inline
Descriptor = descriptor.Descriptor
// MatchOpt defines conditions for a match descriptor.
//
// Deprecated: replace with [descriptor.MatchOpt].
//go:fix inline
MatchOpt = descriptor.MatchOpt
)
var (
// EmptyData is the content of the empty JSON descriptor. See [mediatype.OCI1Empty].
//
// Deprecated: replace with [descriptor.EmptyData].
//go:fix inline
EmptyData = descriptor.EmptyData
// EmptyDigest is the digest of the empty JSON descriptor. See [mediatype.OCI1Empty].
//
// Deprecated: replace with [descriptor.EmptyDigest].
//go:fix inline
EmptyDigest = descriptor.EmptyDigest
// DescriptorListFilter returns a list of descriptors from the list matching the search options.
// When opt.SortAnnotation is set, the order of descriptors with matching annotations is undefined.
//
// Deprecated: replace with [descriptor.DescriptorListFilter]
//go:fix inline
DescriptorListFilter = descriptor.DescriptorListFilter
// DescriptorListSearch returns the first descriptor from the list matching the search options.
//
// Deprecated: replace with [descriptor.DescriptorListSearch]
//go:fix inline
DescriptorListSearch = descriptor.DescriptorListSearch
)
+349
View File
@@ -0,0 +1,349 @@
// Package descriptor defines the OCI descriptor data structure used in manifests to reference content addressable data.
package descriptor
import (
"fmt"
"maps"
"sort"
"strings"
"text/tabwriter"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/internal/units"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/mediatype"
"github.com/regclient/regclient/types/platform"
)
// Descriptor is used in manifests to refer to content by media type, size, and digest.
type Descriptor struct {
// MediaType describe the type of the content.
MediaType string `json:"mediaType"`
// Digest uniquely identifies the content.
Digest digest.Digest `json:"digest"`
// Size in bytes of content.
Size int64 `json:"size"`
// URLs contains the source URLs of this content.
URLs []string `json:"urls,omitempty"`
// Annotations contains arbitrary metadata relating to the targeted content.
Annotations map[string]string `json:"annotations,omitempty"`
// Data is an embedding of the targeted content. This is encoded as a base64
// string when marshalled to JSON (automatically, by encoding/json). If
// present, Data can be used directly to avoid fetching the targeted content.
Data []byte `json:"data,omitempty"`
// Platform describes the platform which the image in the manifest runs on.
// This should only be used when referring to a manifest.
Platform *platform.Platform `json:"platform,omitempty"`
// ArtifactType is the media type of the artifact this descriptor refers to.
ArtifactType string `json:"artifactType,omitempty"`
// digestAlgo is the preferred digest algorithm for when the digest is unset.
digestAlgo digest.Algorithm
}
var (
// EmptyData is the content of the empty JSON descriptor. See [mediatype.OCI1Empty].
EmptyData = []byte("{}")
// EmptyDigest is the digest of the empty JSON descriptor. See [mediatype.OCI1Empty].
EmptyDigest = digest.SHA256.FromBytes(EmptyData)
mtToOCI map[string]string
)
func init() {
mtToOCI = map[string]string{
mediatype.Docker2ManifestList: mediatype.OCI1ManifestList,
mediatype.Docker2Manifest: mediatype.OCI1Manifest,
mediatype.Docker2ImageConfig: mediatype.OCI1ImageConfig,
mediatype.Docker2Layer: mediatype.OCI1Layer,
mediatype.Docker2LayerGzip: mediatype.OCI1LayerGzip,
mediatype.Docker2LayerZstd: mediatype.OCI1LayerZstd,
mediatype.OCI1ManifestList: mediatype.OCI1ManifestList,
mediatype.OCI1Manifest: mediatype.OCI1Manifest,
mediatype.OCI1ImageConfig: mediatype.OCI1ImageConfig,
mediatype.OCI1Layer: mediatype.OCI1Layer,
mediatype.OCI1LayerGzip: mediatype.OCI1LayerGzip,
mediatype.OCI1LayerZstd: mediatype.OCI1LayerZstd,
}
}
// DigestAlgo returns the algorithm for computing the digest.
// This prefers the algorithm used by the digest when set, falling back to the preferred digest algorithm, and finally the canonical algorithm.
func (d Descriptor) DigestAlgo() digest.Algorithm {
if d.Digest != "" && d.Digest.Validate() == nil {
return d.Digest.Algorithm()
}
if d.digestAlgo != "" && d.digestAlgo.Available() {
return d.digestAlgo
}
return digest.Canonical
}
// DigestAlgoPrefer sets the preferred digest algorithm for when the digest is unset.
func (d *Descriptor) DigestAlgoPrefer(algo digest.Algorithm) error {
if !algo.Available() {
return fmt.Errorf("digest algorithm is not available: %s%.0w", algo.String(), errs.ErrUnsupported)
}
d.digestAlgo = algo
return nil
}
// GetData decodes the Data field from the descriptor if available
func (d Descriptor) GetData() ([]byte, error) {
// verify length
if int64(len(d.Data)) != d.Size {
return nil, errs.ErrParsingFailed
}
// generate and verify digest
if d.Digest != d.DigestAlgo().FromBytes(d.Data) {
return nil, errs.ErrParsingFailed
}
// return data
return d.Data, nil
}
// Equal indicates the two descriptors are identical, effectively a DeepEqual.
func (d Descriptor) Equal(d2 Descriptor) bool {
if !d.Same(d2) {
return false
}
if d.MediaType != d2.MediaType {
return false
}
if d.ArtifactType != d2.ArtifactType {
return false
}
if d.Platform == nil || d2.Platform == nil {
if d.Platform != nil || d2.Platform != nil {
return false
}
} else if !platform.Match(*d.Platform, *d2.Platform) {
return false
}
if d.URLs == nil || d2.URLs == nil {
if d.URLs != nil || d2.URLs != nil {
return false
}
} else if len(d.URLs) != len(d2.URLs) {
return false
} else {
for i := range d.URLs {
if d.URLs[i] != d2.URLs[i] {
return false
}
}
}
if d.Annotations == nil || d2.Annotations == nil {
if d.Annotations != nil || d2.Annotations != nil {
return false
}
} else if len(d.Annotations) != len(d2.Annotations) {
return false
} else {
for i := range d.Annotations {
if d.Annotations[i] != d2.Annotations[i] {
return false
}
}
}
return true
}
// Same indicates two descriptors point to the same CAS object.
// This verifies the digest, media type, and size all match.
func (d Descriptor) Same(d2 Descriptor) bool {
if d.Digest != d2.Digest || d.Size != d2.Size {
return false
}
// loosen the check on media type since this can be converted from a build
if d.MediaType != d2.MediaType && (mtToOCI[d.MediaType] != mtToOCI[d2.MediaType] || mtToOCI[d.MediaType] == "") {
return false
}
return true
}
func (d Descriptor) MarshalPrettyTW(tw *tabwriter.Writer, prefix string) error {
fmt.Fprintf(tw, "%sDigest:\t%s\n", prefix, string(d.Digest))
fmt.Fprintf(tw, "%sMediaType:\t%s\n", prefix, d.MediaType)
if d.ArtifactType != "" {
fmt.Fprintf(tw, "%sArtifactType:\t%s\n", prefix, d.ArtifactType)
}
switch d.MediaType {
case mediatype.Docker1Manifest, mediatype.Docker1ManifestSigned,
mediatype.Docker2Manifest, mediatype.Docker2ManifestList,
mediatype.OCI1Manifest, mediatype.OCI1ManifestList:
// skip printing size for descriptors to manifests
default:
if d.Size > 100000 {
fmt.Fprintf(tw, "%sSize:\t%s\n", prefix, units.HumanSize(float64(d.Size)))
} else {
fmt.Fprintf(tw, "%sSize:\t%dB\n", prefix, d.Size)
}
}
if p := d.Platform; p != nil && p.OS != "" {
fmt.Fprintf(tw, "%sPlatform:\t%s\n", prefix, p.String())
if p.OSVersion != "" {
fmt.Fprintf(tw, "%sOSVersion:\t%s\n", prefix, p.OSVersion)
}
if len(p.OSFeatures) > 0 {
fmt.Fprintf(tw, "%sOSFeatures:\t%s\n", prefix, strings.Join(p.OSFeatures, ", "))
}
}
if len(d.URLs) > 0 {
fmt.Fprintf(tw, "%sURLs:\t%s\n", prefix, strings.Join(d.URLs, ", "))
}
if d.Annotations != nil {
fmt.Fprintf(tw, "%sAnnotations:\t\n", prefix)
for k, v := range d.Annotations {
fmt.Fprintf(tw, "%s %s:\t%s\n", prefix, k, v)
}
}
return nil
}
// MatchOpt defines conditions for a match descriptor.
type MatchOpt struct {
Platform *platform.Platform // Platform to match including compatible platforms (darwin/arm64 matches linux/arm64)
ArtifactType string // Match ArtifactType in the descriptor
Annotations map[string]string // Match each of the specified annotations and their value, an empty value verifies the key is set
SortAnnotation string // Sort the results by an annotation, string based comparison, descriptors without the annotation are sorted last
SortDesc bool // Set to true to sort in descending order
}
// Merge applies changes to a MatchOpt, overwriting existing values, and returning a new MatchOpt.
func (mo MatchOpt) Merge(changes MatchOpt) MatchOpt {
ret := MatchOpt{
ArtifactType: changes.ArtifactType,
SortAnnotation: changes.SortAnnotation,
SortDesc: changes.SortDesc,
}
if ret.ArtifactType == "" {
ret.ArtifactType = mo.ArtifactType
}
if changes.Platform != nil {
p := *changes.Platform
ret.Platform = &p
} else if mo.Platform != nil {
p := *mo.Platform
ret.Platform = &p
}
if ret.SortAnnotation == "" {
ret.SortAnnotation = mo.SortAnnotation
}
if !ret.SortDesc {
ret.SortDesc = mo.SortDesc
}
if len(mo.Annotations) > 0 {
ret.Annotations = maps.Clone(mo.Annotations)
}
if len(changes.Annotations) > 0 {
if ret.Annotations == nil {
ret.Annotations = changes.Annotations
} else {
maps.Copy(ret.Annotations, changes.Annotations)
}
}
return ret
}
// Match returns true if the descriptor matches the options, including compatible platforms.
func (d Descriptor) Match(opt MatchOpt) bool {
if opt.ArtifactType != "" && d.ArtifactType != opt.ArtifactType {
return false
}
if len(opt.Annotations) > 0 {
if d.Annotations == nil {
return false
}
for k, v := range opt.Annotations {
if dv, ok := d.Annotations[k]; !ok || (v != "" && v != dv) {
return false
}
}
}
if opt.Platform != nil {
if d.Platform == nil {
return false
}
if !platform.Compatible(*opt.Platform, *d.Platform) {
return false
}
}
return true
}
// DescriptorListFilter returns a list of descriptors from the list matching the search options.
// When opt.SortAnnotation is set, the order of descriptors with matching annotations is undefined.
func DescriptorListFilter(dl []Descriptor, opt MatchOpt) []Descriptor {
ret := []Descriptor{}
for _, d := range dl {
if d.Match(opt) {
ret = append(ret, d)
}
}
if opt.SortAnnotation != "" {
sort.Slice(ret, func(i, j int) bool {
// if annotations are not defined, sort to the very end
if ret[i].Annotations == nil {
return false
}
if _, ok := ret[i].Annotations[opt.SortAnnotation]; !ok {
return false
}
if ret[j].Annotations == nil {
return true
}
if _, ok := ret[j].Annotations[opt.SortAnnotation]; !ok {
return true
}
// else sort by string
if strings.Compare(ret[i].Annotations[opt.SortAnnotation], ret[j].Annotations[opt.SortAnnotation]) < 0 {
return !opt.SortDesc
}
return opt.SortDesc
})
}
return ret
}
// DescriptorListSearch returns the first descriptor from the list matching the search options.
func DescriptorListSearch(dl []Descriptor, opt MatchOpt) (Descriptor, error) {
if opt.ArtifactType != "" || opt.SortAnnotation != "" || len(opt.Annotations) > 0 {
dl = DescriptorListFilter(dl, opt)
}
var ret Descriptor
var retPlat platform.Platform
if len(dl) == 0 {
return ret, errs.ErrNotFound
}
if opt.Platform == nil {
return dl[0], nil
}
found := false
comp := platform.NewCompare(*opt.Platform)
for _, d := range dl {
if d.Platform == nil {
continue
}
if comp.Better(*d.Platform, retPlat) {
found = true
ret = d
retPlat = *d.Platform
}
}
if !found {
return ret, errs.ErrNotFound
}
return ret, nil
}
+3
View File
@@ -0,0 +1,3 @@
// Package types defines various types that have no other internal imports
// This allows them to be used between other packages without creating import loops
package types
+135
View File
@@ -0,0 +1,135 @@
// Package schema1 defines the manifest and json marshal/unmarshal for docker schema1
package schema1
import (
"encoding/json"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/docker/libtrust"
"github.com/opencontainers/go-digest"
"github.com/regclient/regclient/types/docker"
"github.com/regclient/regclient/types/mediatype"
)
var (
// ManifestSchemaVersion provides a pre-initialized version structure schema1 manifests.
ManifestSchemaVersion = docker.Versioned{
SchemaVersion: 1,
MediaType: mediatype.Docker1Manifest,
}
// ManifestSignedSchemaVersion provides a pre-initialized version structure schema1 signed manifests.
ManifestSignedSchemaVersion = docker.Versioned{
SchemaVersion: 1,
MediaType: mediatype.Docker1ManifestSigned,
}
)
// FSLayer is a container struct for BlobSums defined in an image manifest
type FSLayer struct {
// BlobSum is the tarsum of the referenced filesystem image layer
BlobSum digest.Digest `json:"blobSum"`
}
// History stores unstructured v1 compatibility information
type History struct {
// V1Compatibility is the raw v1 compatibility information
V1Compatibility string `json:"v1Compatibility"`
}
// Manifest defines the schema v1 docker manifest
type Manifest struct {
docker.Versioned
// Name is the name of the image's repository
Name string `json:"name"`
// Tag is the tag of the image specified by this manifest
Tag string `json:"tag"`
// Architecture is the host architecture on which this image is intended to run
Architecture string `json:"architecture"`
// FSLayers is a list of filesystem layer blobSums contained in this image
FSLayers []FSLayer `json:"fsLayers"`
// History is a list of unstructured historical data for v1 compatibility
History []History `json:"history"`
}
// SignedManifest provides an envelope for a signed image manifest, including the format sensitive raw bytes.
type SignedManifest struct {
Manifest
// Canonical is the canonical byte representation of the ImageManifest, without any attached signatures.
// The manifest byte representation cannot change or it will have to be re-signed.
Canonical []byte `json:"-"`
// all contains the byte representation of the Manifest including signatures and is returned by Payload()
all []byte
}
// UnmarshalJSON populates a new SignedManifest struct from JSON data.
func (sm *SignedManifest) UnmarshalJSON(b []byte) error {
sm.all = make([]byte, len(b))
// store manifest and signatures in all
copy(sm.all, b)
jsig, err := libtrust.ParsePrettySignature(b, "signatures")
if err != nil {
return err
}
// Resolve the payload in the manifest.
bytes, err := jsig.Payload()
if err != nil {
return err
}
// sm.Canonical stores the canonical manifest JSON
sm.Canonical = make([]byte, len(bytes))
copy(sm.Canonical, bytes)
// Unmarshal canonical JSON into Manifest object
var manifest Manifest
if err := json.Unmarshal(sm.Canonical, &manifest); err != nil {
return err
}
sm.Manifest = manifest
return nil
}
// MarshalJSON returns the contents of raw.
// If Raw is nil, marshals the inner contents.
// Applications requiring a marshaled signed manifest should simply use Raw directly, since the the content produced by json.Marshal will be compacted and will fail signature checks.
func (sm *SignedManifest) MarshalJSON() ([]byte, error) {
if len(sm.all) > 0 {
return sm.all, nil
}
// If the raw data is not available, just dump the inner content.
return json.Marshal(&sm.Manifest)
}
// TODO: verify Payload and Signatures methods are required
// Payload returns the signed content of the signed manifest.
func (sm SignedManifest) Payload() (string, []byte, error) {
return mediatype.Docker1ManifestSigned, sm.all, nil
}
// Signatures returns the signatures as provided by (*libtrust.JSONSignature).Signatures.
// The byte slices are opaque jws signatures.
func (sm *SignedManifest) Signatures() ([][]byte, error) {
jsig, err := libtrust.ParsePrettySignature(sm.all, "signatures")
if err != nil {
return nil, err
}
// Resolve the payload in the manifest.
return jsig.Signatures()
}
+2
View File
@@ -0,0 +1,2 @@
// Package schema2 contains structs for Docker schema v2 manifests.
package schema2

Some files were not shown because too many files have changed in this diff Show More