Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9b7952bdf4 | |||
| 8839ad17ba | |||
| abe3d4d4dd | |||
| 57ce169160 |
@@ -0,0 +1 @@
|
||||
vendor/** linguist-generated=true
|
||||
@@ -19,5 +19,6 @@ var (
|
||||
Minor bool
|
||||
NoDomainChecks bool
|
||||
Patch bool
|
||||
PinDigests bool
|
||||
ShowUnchanged bool
|
||||
)
|
||||
|
||||
@@ -98,7 +98,7 @@ func GetMainAppImage(recipe recipe.Recipe) (string, error) {
|
||||
}
|
||||
for _, service := range config.Services {
|
||||
if service.Name == "app" {
|
||||
img, err := reference.ParseNormalizedNamed(service.Image)
|
||||
img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -322,7 +322,7 @@ func GetImageVersions(recipe recipePkg.Recipe) (map[string]string, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
img, err := reference.ParseNormalizedNamed(service.Image)
|
||||
img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
if err != nil {
|
||||
return services, err
|
||||
}
|
||||
|
||||
+30
-3
@@ -130,7 +130,7 @@ interface.`),
|
||||
}
|
||||
|
||||
for _, service := range config.Services {
|
||||
img, err := reference.ParseNormalizedNamed(service.Image)
|
||||
img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -293,12 +293,31 @@ interface.`),
|
||||
}
|
||||
}
|
||||
if upgradeTag != "skip" {
|
||||
ok, err := recipe.UpdateTag(image, upgradeTag)
|
||||
var ok bool
|
||||
var err error
|
||||
var resolvedDigest string
|
||||
|
||||
if internal.PinDigests {
|
||||
digest, dErr := client.GetImageDigest(img, upgradeTag)
|
||||
if dErr != nil {
|
||||
log.Fatal(i18n.G("failed to resolve digest for %s:%s: %s", image, upgradeTag, dErr.Error()))
|
||||
}
|
||||
resolvedDigest = digest
|
||||
|
||||
ok, err = recipe.UpdatePinnedTag(image, upgradeTag, digest)
|
||||
} else {
|
||||
ok, err = recipe.UpdateTag(image, upgradeTag)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if ok {
|
||||
log.Info(i18n.G("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image))
|
||||
if internal.PinDigests {
|
||||
log.Info(i18n.G("tag upgraded and pinned from %s to %s@%s for %s", tag.String(), upgradeTag, resolvedDigest, image))
|
||||
} else {
|
||||
log.Info(i18n.G("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if !internal.NoInput {
|
||||
@@ -417,4 +436,12 @@ func init() {
|
||||
false,
|
||||
i18n.G("commit changes"),
|
||||
)
|
||||
|
||||
RecipeUpgradeCommand.Flags().BoolVarP(
|
||||
&internal.PinDigests,
|
||||
i18n.G("pindigests"),
|
||||
i18n.GC("p", "pin container image by digest"),
|
||||
false,
|
||||
i18n.G("pin the container image version by manifest digest (image:tag@sha256:...)"),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ require (
|
||||
github.com/moby/sys/signal v0.7.1
|
||||
github.com/moby/term v0.5.2
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/regclient/regclient v0.11.5
|
||||
github.com/schollz/progressbar/v3 v3.19.1
|
||||
golang.org/x/term v0.45.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -73,7 +74,7 @@ require (
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/kevinburke/ssh_config v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
@@ -106,6 +107,7 @@ require (
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/skeema/knownhosts v1.3.2 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
|
||||
@@ -124,10 +126,10 @@ require (
|
||||
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.50.0 // indirect
|
||||
golang.org/x/crypto v0.52.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/net v0.54.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
@@ -151,7 +153,7 @@ require (
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/sergi/go-diff v1.4.0 // indirect
|
||||
github.com/spf13/cobra v1.10.1
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/theupdateframework/notary v0.7.0 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
|
||||
@@ -427,6 +427,8 @@ github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
|
||||
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
|
||||
github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
|
||||
github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
|
||||
@@ -584,8 +586,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
@@ -699,6 +701,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
|
||||
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/olareg/olareg v0.2.1 h1:RPHGIaqlVWPbKAsOYUj7e2WfEEW5M7F8I6hKMrwd4jU=
|
||||
github.com/olareg/olareg v0.2.1/go.mod h1:dhr8QetC7U7jJ2m93oxDhEEOKCRbPgOK1oGyKfB4QNo=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@@ -796,6 +800,8 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/regclient/regclient v0.11.5 h1:OHRsXO0F3qHGfa4HEUv+EkMH9NXNcCTBKjNzyC/UhIA=
|
||||
github.com/regclient/regclient v0.11.5/go.mod h1:DZUOfIT14WFTK2Pj4vjd93avy9O4Fdpjrf9ir23TbRE=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
@@ -859,6 +865,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/sudo-bmitch/oci-digest v0.1.2 h1:are0qzWTsFZGZ3Uvdi9OSztJszSWaab6iqquMEEB7rw=
|
||||
github.com/sudo-bmitch/oci-digest v0.1.2/go.mod h1:SH6l5OIe0islKBZBedjiPOeET/0QwGL+/oYfQt51uQo=
|
||||
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
||||
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
|
||||
@@ -870,6 +878,8 @@ github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnH
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
|
||||
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
@@ -966,8 +976,8 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -1043,8 +1053,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
|
||||
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
|
||||
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -1155,8 +1165,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
||||
@@ -9,8 +9,12 @@ import (
|
||||
"github.com/containers/image/docker"
|
||||
"github.com/containers/image/types"
|
||||
"github.com/distribution/reference"
|
||||
"github.com/regclient/regclient"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
var rc = regclient.New()
|
||||
|
||||
// GetRegistryTags retrieves all tags of an image from a container registry.
|
||||
func GetRegistryTags(img reference.Named) ([]string, error) {
|
||||
var tags []string
|
||||
@@ -28,3 +32,23 @@ func GetRegistryTags(img reference.Named) ([]string, error) {
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// GetImageDigest resolves the content digest (sha256:...) for a specific
|
||||
// image:tag reference by querying the registry manifest. This gives us a
|
||||
// digest to pin to while keeping support for multi-arch deployments: when
|
||||
// the tag points at a manifest list/OCI index, the digest returned is the
|
||||
// digest of that top-level index, so the pin still resolves correctly
|
||||
// regardless of the deploying host's architecture.
|
||||
func GetImageDigest(img reference.Named, tag string) (string, error) {
|
||||
r, err := ref.New(fmt.Sprintf("%s:%s", img.Name(), tag))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parsing reference %s:%s: %w", img.Name(), tag, err)
|
||||
}
|
||||
|
||||
m, err := rc.ManifestHead(context.Background(), r)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetching manifest for %s:%s: %w", img.Name(), tag, err)
|
||||
}
|
||||
|
||||
return m.GetDescriptor().Digest.String(), nil
|
||||
}
|
||||
|
||||
+2
-2
@@ -158,7 +158,7 @@ func GetImagesForStack(cl *dockerClient.Client, app appPkg.App) (map[string]stri
|
||||
if service.Spec.TaskTemplate.ContainerSpec != nil {
|
||||
imageName := service.Spec.TaskTemplate.ContainerSpec.Image
|
||||
|
||||
imageParsed, err := reference.ParseNormalizedNamed(imageName)
|
||||
imageParsed, err := reference.ParseNormalizedNamed(formatter.StripDigest(imageName))
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
continue
|
||||
@@ -281,7 +281,7 @@ func GatherImagesForDeploy(cl *dockerClient.Client, app appPkg.App, compose *com
|
||||
newImages := make(map[string]string)
|
||||
|
||||
for _, service := range compose.Services {
|
||||
imageParsed, err := reference.ParseNormalizedNamed(service.Image)
|
||||
imageParsed, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
continue
|
||||
|
||||
@@ -34,6 +34,11 @@ func SmallSHA(hash string) string {
|
||||
return hash[:8]
|
||||
}
|
||||
|
||||
// StripDigest removes the digest suffix from an image reference (e.g., "image:tag@sha256:..." -> "image:tag").
|
||||
func StripDigest(image string) string {
|
||||
return strings.Split(image, "@")[0]
|
||||
}
|
||||
|
||||
// RemoveSha remove image sha from a string that are added in some docker outputs
|
||||
func RemoveSha(str string) string {
|
||||
return strings.Split(str, "@")[0]
|
||||
|
||||
@@ -6,6 +6,13 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestStripDigest(t *testing.T) {
|
||||
assert.Equal(t, "ubuntu:latest", StripDigest("ubuntu:latest@sha256:1234567890abcdef"))
|
||||
assert.Equal(t, "ubuntu:latest", StripDigest("ubuntu:latest"))
|
||||
assert.Equal(t, "my-repo/my-image:v1", StripDigest("my-repo/my-image:v1@sha256:abcdef1234567890"))
|
||||
assert.Equal(t, "my-repo/my-image", StripDigest("my-repo/my-image@sha256:abcdef1234567890"))
|
||||
}
|
||||
|
||||
func TestBoldDirtyDefault(t *testing.T) {
|
||||
assert.Equal(t, "foo", BoldDirtyDefault("foo"))
|
||||
}
|
||||
|
||||
+4
-3
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"coopcloud.tech/abra/pkg/formatter"
|
||||
"coopcloud.tech/abra/pkg/i18n"
|
||||
"coopcloud.tech/abra/pkg/log"
|
||||
"coopcloud.tech/abra/pkg/recipe"
|
||||
@@ -321,7 +322,7 @@ func LintAllImagesTagged(recipe recipe.Recipe) (bool, error) {
|
||||
return false, err
|
||||
}
|
||||
for _, service := range config.Services {
|
||||
img, err := reference.ParseNormalizedNamed(service.Image)
|
||||
img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -339,7 +340,7 @@ func LintNoUnstableTags(recipe recipe.Recipe) (bool, error) {
|
||||
return false, err
|
||||
}
|
||||
for _, service := range config.Services {
|
||||
img, err := reference.ParseNormalizedNamed(service.Image)
|
||||
img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -366,7 +367,7 @@ func LintSemverLikeTags(recipe recipe.Recipe) (bool, error) {
|
||||
return false, err
|
||||
}
|
||||
for _, service := range config.Services {
|
||||
img, err := reference.ParseNormalizedNamed(service.Image)
|
||||
img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
+68
-2
@@ -122,6 +122,7 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
|
||||
|
||||
log.Debug(i18n.G("considering %s config(s) for tag update", strings.Join(composeFiles, ", ")))
|
||||
|
||||
updated := false
|
||||
for _, composeFile := range composeFiles {
|
||||
opts := stack.Deploy{Composefiles: []string{composeFile}}
|
||||
|
||||
@@ -140,7 +141,7 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
|
||||
continue // may be a compose.$optional.yml file
|
||||
}
|
||||
|
||||
img, _ := reference.ParseNormalizedNamed(service.Image)
|
||||
img, _ := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -166,6 +167,10 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
|
||||
|
||||
old := fmt.Sprintf("%s:%s", composeImage, composeTag)
|
||||
new := fmt.Sprintf("%s:%s", composeImage, tag)
|
||||
if old == new {
|
||||
continue
|
||||
}
|
||||
|
||||
replacedBytes := strings.Replace(string(bytes), old, new, -1)
|
||||
|
||||
log.Debug(i18n.G("updating %s to %s in %s", old, new, compose.Filename))
|
||||
@@ -173,11 +178,72 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
|
||||
if err := os.WriteFile(compose.Filename, []byte(replacedBytes), 0o764); err != nil {
|
||||
return false, err
|
||||
}
|
||||
updated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
// UpdatePinnedTag updates an image reference to a pinned version (image:tag@sha256:...) in-place on local compose files.
|
||||
func (r Recipe) UpdatePinnedTag(image, tag, digest string) (bool, error) {
|
||||
glob := fmt.Sprintf("%s/compose**yml", r.Dir)
|
||||
image = formatter.StripTagMeta(image)
|
||||
|
||||
composeFiles, err := filepath.Glob(glob)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
updated := false
|
||||
for _, composeFile := range composeFiles {
|
||||
opts := stack.Deploy{Composefiles: []string{composeFile}}
|
||||
|
||||
sampleEnv, err := r.SampleEnv()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
compose, err := loader.LoadComposefile(opts, sampleEnv)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
for _, service := range compose.Services {
|
||||
if service.Image == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
img, _ := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
composeImage := formatter.StripTagMeta(reference.Path(img))
|
||||
|
||||
if image == composeImage {
|
||||
bytes, err := ioutil.ReadFile(composeFile)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
old := service.Image
|
||||
new := fmt.Sprintf("%s:%s@%s", composeImage, tag, digest)
|
||||
|
||||
if old == new {
|
||||
continue
|
||||
}
|
||||
|
||||
replacedBytes := strings.Replace(string(bytes), old, new, -1)
|
||||
|
||||
log.Debug(i18n.G("updating pinned reference %s to %s in %s", old, new, compose.Filename))
|
||||
|
||||
if err := os.WriteFile(compose.Filename, []byte(replacedBytes), 0o764); err != nil {
|
||||
return false, err
|
||||
}
|
||||
updated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
// UpdateLabel updates a label in-place on file system local compose files.
|
||||
|
||||
+1
-1
@@ -426,7 +426,7 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, []string, error) {
|
||||
versionMeta := make(map[string]ServiceMeta)
|
||||
for _, service := range config.Services {
|
||||
|
||||
img, err := reference.ParseNormalizedNamed(service.Image)
|
||||
img, err := reference.ParseNormalizedNamed(formatter.StripDigest(service.Image))
|
||||
if err != nil {
|
||||
log.Debug(i18n.G("failed to parse image for %s in %s: %s", service.Name, tag, err))
|
||||
warnMsg = append(warnMsg, i18n.G("skipping tag %s: invalid image reference in service %s: %s", tag, service.Name, err))
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
* -text
|
||||
*.bin -text -diff
|
||||
*.md text eol=lf
|
||||
|
||||
+700
-700
File diff suppressed because it is too large
Load Diff
+78
-78
@@ -1,79 +1,79 @@
|
||||
# Finite State Entropy
|
||||
|
||||
This package provides Finite State Entropy encoding and decoding.
|
||||
|
||||
Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS))
|
||||
encoding provides a fast near-optimal symbol encoding/decoding
|
||||
for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
|
||||
|
||||
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
|
||||
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
|
||||
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
|
||||
|
||||
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
|
||||
|
||||
## News
|
||||
|
||||
* Feb 2018: First implementation released. Consider this beta software for now.
|
||||
|
||||
# Usage
|
||||
|
||||
This package provides a low level interface that allows to compress single independent blocks.
|
||||
|
||||
Each block is separate, and there is no built in integrity checks.
|
||||
This means that the caller should keep track of block sizes and also do checksums if needed.
|
||||
|
||||
Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
|
||||
You must provide input and will receive the output and maybe an error.
|
||||
|
||||
These error values can be returned:
|
||||
|
||||
| Error | Description |
|
||||
|---------------------|-----------------------------------------------------------------------------|
|
||||
| `<nil>` | Everything ok, output is returned |
|
||||
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
|
||||
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
|
||||
| `(error)` | An internal error occurred. |
|
||||
|
||||
As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
|
||||
|
||||
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object
|
||||
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
|
||||
object can be used for both.
|
||||
|
||||
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
|
||||
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
|
||||
|
||||
Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
|
||||
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
|
||||
your input was likely corrupted.
|
||||
|
||||
It is important to note that a successful decoding does *not* mean your output matches your original input.
|
||||
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
|
||||
|
||||
For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
|
||||
|
||||
# Performance
|
||||
|
||||
A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.
|
||||
All compression functions are currently only running on the calling goroutine so only one core will be used per block.
|
||||
|
||||
The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
|
||||
is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be
|
||||
beneficial to transpose all your input values down by 64.
|
||||
|
||||
With moderate block sizes around 64k speed are typically 200MB/s per core for compression and
|
||||
around 300MB/s decompression speed.
|
||||
|
||||
The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s.
|
||||
|
||||
# Plans
|
||||
|
||||
At one point, more internals will be exposed to facilitate more "expert" usage of the components.
|
||||
|
||||
A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).
|
||||
|
||||
# Contributing
|
||||
|
||||
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
|
||||
# Finite State Entropy
|
||||
|
||||
This package provides Finite State Entropy encoding and decoding.
|
||||
|
||||
Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS))
|
||||
encoding provides a fast near-optimal symbol encoding/decoding
|
||||
for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
|
||||
|
||||
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
|
||||
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
|
||||
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
|
||||
|
||||
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
|
||||
|
||||
## News
|
||||
|
||||
* Feb 2018: First implementation released. Consider this beta software for now.
|
||||
|
||||
# Usage
|
||||
|
||||
This package provides a low level interface that allows to compress single independent blocks.
|
||||
|
||||
Each block is separate, and there is no built in integrity checks.
|
||||
This means that the caller should keep track of block sizes and also do checksums if needed.
|
||||
|
||||
Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
|
||||
You must provide input and will receive the output and maybe an error.
|
||||
|
||||
These error values can be returned:
|
||||
|
||||
| Error | Description |
|
||||
|---------------------|-----------------------------------------------------------------------------|
|
||||
| `<nil>` | Everything ok, output is returned |
|
||||
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
|
||||
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
|
||||
| `(error)` | An internal error occurred. |
|
||||
|
||||
As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
|
||||
|
||||
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object
|
||||
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
|
||||
object can be used for both.
|
||||
|
||||
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
|
||||
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
|
||||
|
||||
Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
|
||||
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
|
||||
your input was likely corrupted.
|
||||
|
||||
It is important to note that a successful decoding does *not* mean your output matches your original input.
|
||||
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
|
||||
|
||||
For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
|
||||
|
||||
# Performance
|
||||
|
||||
A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.
|
||||
All compression functions are currently only running on the calling goroutine so only one core will be used per block.
|
||||
|
||||
The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
|
||||
is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be
|
||||
beneficial to transpose all your input values down by 64.
|
||||
|
||||
With moderate block sizes around 64k speed are typically 200MB/s per core for compression and
|
||||
around 300MB/s decompression speed.
|
||||
|
||||
The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s.
|
||||
|
||||
# Plans
|
||||
|
||||
At one point, more internals will be exposed to facilitate more "expert" usage of the components.
|
||||
|
||||
A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).
|
||||
|
||||
# Contributing
|
||||
|
||||
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
|
||||
changes will likely not be accepted. If in doubt open an issue before writing the PR.
|
||||
+89
-89
@@ -1,89 +1,89 @@
|
||||
# Huff0 entropy compression
|
||||
|
||||
This package provides Huff0 encoding and decoding as used in zstd.
|
||||
|
||||
[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders),
|
||||
a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU
|
||||
(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
|
||||
|
||||
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
|
||||
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
|
||||
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
|
||||
|
||||
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
|
||||
|
||||
## News
|
||||
|
||||
This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
|
||||
|
||||
This ensures that most functionality is well tested.
|
||||
|
||||
# Usage
|
||||
|
||||
This package provides a low level interface that allows to compress single independent blocks.
|
||||
|
||||
Each block is separate, and there is no built in integrity checks.
|
||||
This means that the caller should keep track of block sizes and also do checksums if needed.
|
||||
|
||||
Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and
|
||||
[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
|
||||
You must provide input and will receive the output and maybe an error.
|
||||
|
||||
These error values can be returned:
|
||||
|
||||
| Error | Description |
|
||||
|---------------------|-----------------------------------------------------------------------------|
|
||||
| `<nil>` | Everything ok, output is returned |
|
||||
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
|
||||
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
|
||||
| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) |
|
||||
| `(error)` | An internal error occurred. |
|
||||
|
||||
|
||||
As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
|
||||
|
||||
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object
|
||||
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
|
||||
object can be used for both.
|
||||
|
||||
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
|
||||
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
|
||||
|
||||
The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.
|
||||
|
||||
## Tables and re-use
|
||||
|
||||
Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results.
|
||||
|
||||
The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy)
|
||||
that controls this behaviour. See the documentation for details. This can be altered between each block.
|
||||
|
||||
Do however note that this information is *not* stored in the output block and it is up to the users of the package to
|
||||
record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
|
||||
based on the boolean reported back from the CompressXX call.
|
||||
|
||||
If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the
|
||||
[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
|
||||
|
||||
## Decompressing
|
||||
|
||||
The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
|
||||
This will initialize the decoding tables.
|
||||
You can supply the complete block to `ReadTable` and it will return the data part of the block
|
||||
which can be given to the decompressor.
|
||||
|
||||
Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X)
|
||||
or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
|
||||
|
||||
For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
|
||||
|
||||
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
|
||||
your input was likely corrupted.
|
||||
|
||||
It is important to note that a successful decoding does *not* mean your output matches your original input.
|
||||
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
|
||||
|
||||
# Contributing
|
||||
|
||||
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
|
||||
changes will likely not be accepted. If in doubt open an issue before writing the PR.
|
||||
# Huff0 entropy compression
|
||||
|
||||
This package provides Huff0 encoding and decoding as used in zstd.
|
||||
|
||||
[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders),
|
||||
a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU
|
||||
(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
|
||||
|
||||
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
|
||||
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
|
||||
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
|
||||
|
||||
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
|
||||
|
||||
## News
|
||||
|
||||
This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
|
||||
|
||||
This ensures that most functionality is well tested.
|
||||
|
||||
# Usage
|
||||
|
||||
This package provides a low level interface that allows to compress single independent blocks.
|
||||
|
||||
Each block is separate, and there is no built in integrity checks.
|
||||
This means that the caller should keep track of block sizes and also do checksums if needed.
|
||||
|
||||
Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and
|
||||
[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
|
||||
You must provide input and will receive the output and maybe an error.
|
||||
|
||||
These error values can be returned:
|
||||
|
||||
| Error | Description |
|
||||
|---------------------|-----------------------------------------------------------------------------|
|
||||
| `<nil>` | Everything ok, output is returned |
|
||||
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
|
||||
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
|
||||
| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) |
|
||||
| `(error)` | An internal error occurred. |
|
||||
|
||||
|
||||
As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
|
||||
|
||||
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object
|
||||
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
|
||||
object can be used for both.
|
||||
|
||||
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
|
||||
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
|
||||
|
||||
The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.
|
||||
|
||||
## Tables and re-use
|
||||
|
||||
Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results.
|
||||
|
||||
The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy)
|
||||
that controls this behaviour. See the documentation for details. This can be altered between each block.
|
||||
|
||||
Do however note that this information is *not* stored in the output block and it is up to the users of the package to
|
||||
record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
|
||||
based on the boolean reported back from the CompressXX call.
|
||||
|
||||
If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the
|
||||
[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
|
||||
|
||||
## Decompressing
|
||||
|
||||
The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
|
||||
This will initialize the decoding tables.
|
||||
You can supply the complete block to `ReadTable` and it will return the data part of the block
|
||||
which can be given to the decompressor.
|
||||
|
||||
Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X)
|
||||
or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
|
||||
|
||||
For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
|
||||
|
||||
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
|
||||
your input was likely corrupted.
|
||||
|
||||
It is important to note that a successful decoding does *not* mean your output matches your original input.
|
||||
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
|
||||
|
||||
# Contributing
|
||||
|
||||
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
|
||||
changes will likely not be accepted. If in doubt open an issue before writing the PR.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26@sha256:b900de91b15b2e2953d930ece1d0ecff0a1590ab2006088d20dcf0f56f1e979f
|
||||
FROM golang:1.25@sha256:31c1e53dfc1cc2d269deec9c83f58729fa3c53dc9a576f6426109d1e319e9e9a
|
||||
|
||||
ENV GOOS=linux
|
||||
ENV GOARCH=arm
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26@sha256:b900de91b15b2e2953d930ece1d0ecff0a1590ab2006088d20dcf0f56f1e979f
|
||||
FROM golang:1.25@sha256:31c1e53dfc1cc2d269deec9c83f58729fa3c53dc9a576f6426109d1e319e9e9a
|
||||
|
||||
ENV GOOS=linux
|
||||
ENV GOARCH=arm64
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
*
|
||||
!.git/
|
||||
!build/root.tgz
|
||||
!cmd/
|
||||
!config/
|
||||
!internal/
|
||||
!mod/
|
||||
!pkg/
|
||||
!regclient/
|
||||
!scheme/
|
||||
!types/
|
||||
!vendor/
|
||||
!go.*
|
||||
!*.go
|
||||
!Makefile
|
||||
@@ -0,0 +1,5 @@
|
||||
artifacts/
|
||||
bin/
|
||||
output/
|
||||
vendor/
|
||||
.regctl_conf_ci.json
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
# all lists use a `-`
|
||||
MD004:
|
||||
style: dash
|
||||
|
||||
# allow tabs in code blocks (for Go)
|
||||
MD010:
|
||||
code_blocks: false
|
||||
|
||||
# disable line length, prefer one sentence per line for PRs
|
||||
MD013: false
|
||||
|
||||
# emphasis with underscore (`_emphasis_`)
|
||||
MD049:
|
||||
style: "underscore"
|
||||
|
||||
# bold with asterisk (`**bold**`)
|
||||
MD050:
|
||||
style: "asterisk"
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
GoVersionOverride = "1.26.3"
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
{"name":"docker-arg-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
|
||||
{"name":"docker-arg-alpine-tag","key":"docker.io/library/alpine","version":"3.23.4"}
|
||||
{"name":"docker-arg-ecr","key":"https://github.com/awslabs/amazon-ecr-credential-helper.git","version":"v0.12.0"}
|
||||
{"name":"docker-arg-gcr","key":"https://github.com/GoogleCloudPlatform/docker-credential-gcr.git","version":"v2.1.32"}
|
||||
{"name":"docker-arg-go-digest","key":"docker.io/library/golang:1.26.3-alpine","version":"sha256:91eda9776261207ea25fd06b5b7fed8d397dd2c0a283e77f2ab6e91bfa71079d"}
|
||||
{"name":"docker-arg-go-tag","key":"docker.io/library/golang","version":"1.26.3"}
|
||||
{"name":"docker-arg-lunajson","key":"https://github.com/grafi-tt/lunajson.git:master","version":"e3a9666eb1275741e887e29926b144f8daee3bef"}
|
||||
{"name":"docker-arg-semver","key":"https://github.com/kikito/semver.lua.git:master","version":"a4b708ba243208d46e575da870af969dca46a94d"}
|
||||
{"name":"gha-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
|
||||
{"name":"gha-alpine-tag-base","key":"docker.io/library/alpine","version":"3"}
|
||||
{"name":"gha-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.23.4"}
|
||||
{"name":"gha-cosign-version","key":"https://github.com/sigstore/cosign.git","version":"v3.0.6"}
|
||||
{"name":"gha-golang-matrix","key":"golang-matrix","version":"[\"1.25\", \"1.26\"]"}
|
||||
{"name":"gha-golang-release","key":"golang-latest","version":"1.26"}
|
||||
{"name":"gha-syft-version","key":"docker.io/anchore/syft","version":"v1.44.0"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/actions/checkout.git:v6.0.2","version":"de0fac2e4500dabe0009e67214ff5f5447ce83dd"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/actions/setup-go.git:v6.4.0","version":"4a3601121dd01d1626a1e23e37211e3254c1c06c"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/actions/stale.git:v10.3.0","version":"eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/actions/upload-artifact.git:v7.0.1","version":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/anchore/sbom-action.git:v0.24.0","version":"e22c389904149dbc22b58101806040fa8d37a610"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/docker/build-push-action.git:v7.2.0","version":"f9f3042f7e2789586610d6e8b85c8f03e5195baf"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/docker/login-action.git:v4.2.0","version":"650006c6eb7dba73a995cc03b0b2d7f5ca915bee"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/docker/setup-buildx-action.git:v4.1.0","version":"d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/regclient/actions.git:main","version":"c70ad64367908075211b10dcd2ab9fad4bfa1816"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/sigstore/cosign-installer.git:v4.1.2","version":"6f9f17788090df1f26f669e9d70d6ae9567deba6"}
|
||||
{"name":"gha-uses-commit","key":"https://github.com/softprops/action-gh-release.git:v3.0.0","version":"b4309332981a82ec1c5618f44dd2e27cc8bfbfda"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/actions/checkout.git","version":"v6.0.2"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/actions/setup-go.git","version":"v6.4.0"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/actions/stale.git","version":"v10.3.0"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/actions/upload-artifact.git","version":"v7.0.1"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/anchore/sbom-action.git","version":"v0.24.0"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/docker/build-push-action.git","version":"v7.2.0"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/docker/login-action.git","version":"v4.2.0"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/docker/setup-buildx-action.git","version":"v4.1.0"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/sigstore/cosign-installer.git","version":"v4.1.2"}
|
||||
{"name":"gha-uses-semver","key":"https://github.com/softprops/action-gh-release.git","version":"v3.0.0"}
|
||||
{"name":"go-mod-golang-release","key":"golang-oldest","version":"1.25.0"}
|
||||
{"name":"makefile-ci-distribution","key":"docker.io/library/registry","version":"3.1.1"}
|
||||
{"name":"makefile-ci-zot","key":"ghcr.io/project-zot/zot-linux-amd64","version":"v2.1.17"}
|
||||
{"name":"makefile-go-vulncheck","key":"https://go.googlesource.com/vuln.git","version":"v1.3.0"}
|
||||
{"name":"makefile-gofumpt","key":"https://github.com/mvdan/gofumpt.git","version":"v0.10.0"}
|
||||
{"name":"makefile-gomajor","key":"https://github.com/icholy/gomajor.git","version":"v0.15.0"}
|
||||
{"name":"makefile-gosec","key":"https://github.com/securego/gosec.git","version":"v2.26.1"}
|
||||
{"name":"makefile-markdown-lint","key":"docker.io/davidanson/markdownlint-cli2","version":"v0.22.1"}
|
||||
{"name":"makefile-osv-scanner","key":"https://github.com/google/osv-scanner.git","version":"v2.3.8"}
|
||||
{"name":"makefile-staticcheck","key":"https://github.com/dominikh/go-tools.git","version":"v0.7.0"}
|
||||
{"name":"makefile-syft-container-digest","key":"anchore/syft:v1.44.0","version":"sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6"}
|
||||
{"name":"makefile-syft-container-tag","key":"anchore/syft","version":"v1.44.0"}
|
||||
{"name":"makefile-syft-version","key":"docker.io/anchore/syft","version":"v1.44.0"}
|
||||
{"name":"osv-golang-release","key":"docker.io/library/golang","version":"1.26.3"}
|
||||
{"name":"shell-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
|
||||
{"name":"shell-alpine-tag-base","key":"docker.io/library/alpine","version":"3"}
|
||||
{"name":"shell-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.23.4"}
|
||||
+346
@@ -0,0 +1,346 @@
|
||||
files:
|
||||
"build/Dockerfile*":
|
||||
processors:
|
||||
- docker-arg-alpine-tag
|
||||
- docker-arg-alpine-digest
|
||||
- docker-arg-go-tag
|
||||
- docker-arg-go-digest
|
||||
- docker-arg-ecr
|
||||
- docker-arg-gcr
|
||||
- docker-arg-lunajson
|
||||
- docker-arg-semver
|
||||
"build/oci-image.sh":
|
||||
processors:
|
||||
- shell-alpine-tag-base
|
||||
- shell-alpine-tag-comment
|
||||
- shell-alpine-digest
|
||||
".github/workflows/*.yml":
|
||||
processors:
|
||||
- gha-golang-matrix
|
||||
- gha-golang-release
|
||||
- gha-uses-vx
|
||||
- gha-uses-semver
|
||||
- gha-uses-commit
|
||||
- gha-syft-version
|
||||
- gha-cosign-version
|
||||
- gha-alpine-tag-base
|
||||
- gha-alpine-tag-comment
|
||||
- gha-alpine-digest
|
||||
"Makefile":
|
||||
processors:
|
||||
- makefile-gofumpt
|
||||
- makefile-gomajor
|
||||
- makefile-go-vulncheck
|
||||
- makefile-markdown-lint
|
||||
- makefile-gosec
|
||||
- makefile-osv-scanner
|
||||
- makefile-staticcheck
|
||||
- makefile-syft-version
|
||||
- makefile-syft-container-tag
|
||||
- makefile-syft-container-digest
|
||||
- makefile-ci-distribution
|
||||
- makefile-ci-zot
|
||||
"go.mod":
|
||||
processors:
|
||||
- go-mod-golang-release
|
||||
".osv-scanner.toml":
|
||||
processors:
|
||||
- osv-golang-release
|
||||
|
||||
x-processor-tmpl:
|
||||
git-commit: &git-commit
|
||||
key: "{{ .SourceArgs.url }}:{{ .SourceArgs.ref }}"
|
||||
scan: "regexp"
|
||||
source: "git-commit"
|
||||
filter:
|
||||
expr: "^{{ .SourceArgs.ref }}$"
|
||||
git-tag-semver: &git-tag-semver
|
||||
key: "{{ .SourceArgs.url }}"
|
||||
scan: "regexp"
|
||||
source: "git-tag"
|
||||
filter:
|
||||
expr: '^v?\d+\.\d+\.\d+$'
|
||||
sort:
|
||||
method: "semver"
|
||||
registry-digest: ®istry-digest
|
||||
key: "{{ .SourceArgs.image }}"
|
||||
scan: "regexp"
|
||||
source: "registry-digest"
|
||||
registry-tag-semver: ®istry-tag-semver
|
||||
key: "{{ .SourceArgs.repo }}"
|
||||
scan: "regexp"
|
||||
source: "registry-tag"
|
||||
filter:
|
||||
expr: '^v?\d+\.\d+\.\d+$'
|
||||
sort:
|
||||
method: "semver"
|
||||
|
||||
processors:
|
||||
docker-arg-alpine-tag:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^ARG ALPINE_VER=(?P<Version>v?\d+\.\d+\.\d+)@(?P<SHA>sha256:[0-9a-f]+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/alpine"
|
||||
docker-arg-alpine-digest:
|
||||
<<: *registry-digest
|
||||
scanArgs:
|
||||
regexp: '^ARG ALPINE_VER=(?P<Tag>v?\d+\.\d+\.\d+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
|
||||
sourceArgs:
|
||||
image: "docker.io/library/alpine:{{.ScanMatch.Tag}}"
|
||||
docker-arg-go-tag:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^ARG GO_VER=(?P<Version>[a-z0-9\-\.]+)-alpine@(?P<SHA>sha256:[0-9a-f]+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/golang"
|
||||
docker-arg-go-digest:
|
||||
<<: *registry-digest
|
||||
scanArgs:
|
||||
regexp: '^ARG GO_VER=(?P<Tag>[a-z0-9\-\.]+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
|
||||
sourceArgs:
|
||||
image: "docker.io/library/golang:{{.ScanMatch.Tag}}"
|
||||
docker-arg-ecr:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^ARG ECR_HELPER_VER=(?P<Version>v?\d+\.\d+\.\d+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/awslabs/amazon-ecr-credential-helper.git"
|
||||
# get the version for the ecr-login nested package in the repo
|
||||
filter:
|
||||
expr: '^ecr-login/v?\d+\.\d+\.\d+$'
|
||||
# sort and output only the version number without the package name prefix
|
||||
sort:
|
||||
method: "semver"
|
||||
template: '{{ index (split . "/") 1 }}'
|
||||
template: '{{ index (split .Version "/") 1 }}'
|
||||
docker-arg-gcr:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^ARG GCR_HELPER_VER=(?P<Version>v?\d+\.\d+\.\d+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/GoogleCloudPlatform/docker-credential-gcr.git"
|
||||
docker-arg-lunajson:
|
||||
<<: *git-commit
|
||||
scanArgs:
|
||||
regexp: '^ARG LUNAJSON_COMMIT=(?P<Version>[0-9a-f]+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/grafi-tt/lunajson.git"
|
||||
ref: master
|
||||
docker-arg-semver:
|
||||
<<: *git-commit
|
||||
scanArgs:
|
||||
regexp: '^ARG SEMVER_COMMIT=(?P<Version>[0-9a-f]+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/kikito/semver.lua.git"
|
||||
ref: master
|
||||
|
||||
gha-alpine-digest:
|
||||
<<: *registry-digest
|
||||
scanArgs:
|
||||
regexp: '^\s*ALPINE_DIGEST: "(?P<Version>sha256:[0-9a-f]+)"\s*#\s*(?P<Tag>\d+\.\d+\.\d+)\s*$'
|
||||
sourceArgs:
|
||||
image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}"
|
||||
gha-alpine-tag-base:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^\s*ALPINE_NAME: "alpine:(?P<Version>v?\d+)"\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/alpine"
|
||||
# only return the major version number in the tag to support detecting a change in the base image
|
||||
template: '{{ index ( split .Version "." ) 0 }}'
|
||||
gha-alpine-tag-comment:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^\s*ALPINE_DIGEST: "(?P<Digest>sha256:[0-9a-f]+)"\s*#\s*(?P<Version>v?\d+\.\d+\.\d+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/alpine"
|
||||
gha-cosign-version:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^\s*cosign-release: "(?P<Version>v?[0-9\.]+)"\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/sigstore/cosign.git"
|
||||
filter:
|
||||
expr: '^v?3\.\d+\.\d+$' # pin to v3, v4 will remove support for older clients
|
||||
gha-golang-matrix:
|
||||
<<: *registry-tag-semver
|
||||
key: "golang-matrix"
|
||||
scanArgs:
|
||||
regexp: '^\s*gover: (?P<Version>\[["0-9, \.]+\])\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/golang"
|
||||
filter:
|
||||
expr: '^v?\d+\.\d+$'
|
||||
template: '["{{ index .VerMap ( index .VerList 1 ) }}", "{{ index .VerMap ( index .VerList 0 ) }}"]'
|
||||
gha-golang-release:
|
||||
<<: *registry-tag-semver
|
||||
key: "golang-latest"
|
||||
scanArgs:
|
||||
regexp: '^\s*RELEASE_GO_VER: "(?P<Version>v?[0-9\.]+)"\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/golang"
|
||||
filter:
|
||||
expr: '^v?\d+\.\d+$'
|
||||
gha-syft-version:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^\s*syft-version: "(?P<Version>v?[0-9\.]+)"\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/anchore/syft"
|
||||
gha-uses-vx:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Commit>[0-9a-f]+)\s+#\s+(?P<Version>v?\d+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
|
||||
filter:
|
||||
expr: '^v?\d+$'
|
||||
gha-uses-semver:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Commit>[0-9a-f]+)\s+#\s+(?P<Version>v?\d+\.\d+\.\d+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
|
||||
gha-uses-commit:
|
||||
<<: *git-commit
|
||||
scanArgs:
|
||||
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Version>[0-9a-f]+)\s+#\s+(?P<Ref>[\w\d\.]+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
|
||||
ref: "{{ .ScanMatch.Ref }}"
|
||||
|
||||
go-mod-golang-release:
|
||||
<<: *registry-tag-semver
|
||||
key: "golang-oldest"
|
||||
scanArgs:
|
||||
regexp: '^go (?P<Version>[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/golang"
|
||||
filter:
|
||||
expr: '^\d+\.\d+$'
|
||||
template: '{{ index .VerMap ( index .VerList 1 ) }}.0'
|
||||
|
||||
makefile-ci-distribution:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^CI_DISTRIBUTION_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/registry"
|
||||
makefile-ci-zot:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^CI_ZOT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "ghcr.io/project-zot/zot-linux-amd64"
|
||||
makefile-gofumpt:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^GOFUMPT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/mvdan/gofumpt.git"
|
||||
makefile-gomajor:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^GOMAJOR_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/icholy/gomajor.git"
|
||||
makefile-gosec:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^GOSEC_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/securego/gosec.git"
|
||||
makefile-go-vulncheck:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^GO_VULNCHECK_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://go.googlesource.com/vuln.git"
|
||||
makefile-markdown-lint:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^MARKDOWN_LINT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/davidanson/markdownlint-cli2"
|
||||
makefile-osv-scanner:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^OSV_SCANNER_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/google/osv-scanner.git"
|
||||
makefile-staticcheck:
|
||||
<<: *git-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^STATICCHECK_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
url: "https://github.com/dominikh/go-tools.git"
|
||||
filter:
|
||||
# repo also has dated tags, ignore versions without a preceding "v"
|
||||
expr: '^v\d+\.\d+\.\d+$'
|
||||
makefile-syft-container-tag:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^SYFT_CONTAINER\?=(?P<Repo>[^:]*):(?P<Version>v?[0-9\.]+)@(?P<Digest>sha256:[0-9a-f]+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "{{ .ScanMatch.Repo }}"
|
||||
makefile-syft-container-digest:
|
||||
<<: *registry-digest
|
||||
scanArgs:
|
||||
regexp: '^SYFT_CONTAINER\?=(?P<Image>[^:]*):(?P<Tag>v?[0-9\.]+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
|
||||
sourceArgs:
|
||||
image: "{{ .ScanMatch.Image }}:{{.ScanMatch.Tag}}"
|
||||
makefile-syft-version:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^SYFT_VERSION\?=(?P<Version>v[0-9\.]+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/anchore/syft"
|
||||
|
||||
osv-golang-release:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^GoVersionOverride = "(?P<Version>v?[0-9\.]+)"\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/golang"
|
||||
|
||||
shell-alpine-tag-base:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^\s*ALPINE_NAME="alpine:(?P<Version>v?\d+)"\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/alpine"
|
||||
# only return the major version number in the tag to support detecting a change in the base image
|
||||
template: '{{ index ( split .Version "." ) 0 }}'
|
||||
shell-alpine-tag-comment:
|
||||
<<: *registry-tag-semver
|
||||
scanArgs:
|
||||
regexp: '^\s*ALPINE_DIGEST="(?P<Digest>sha256:[0-9a-f]+)"\s*#\s*(?P<Version>v?\d+\.\d+\.\d+)\s*$'
|
||||
sourceArgs:
|
||||
repo: "docker.io/library/alpine"
|
||||
shell-alpine-digest:
|
||||
<<: *registry-digest
|
||||
scanArgs:
|
||||
regexp: '^\s*ALPINE_DIGEST="(?P<Version>sha256:[0-9a-f]+)"\s*#\s*(?P<Tag>\d+\.\d+\.\d+)\s*$'
|
||||
sourceArgs:
|
||||
image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}"
|
||||
|
||||
scans:
|
||||
regexp:
|
||||
type: "regexp"
|
||||
|
||||
sources:
|
||||
git-commit:
|
||||
type: "git"
|
||||
args:
|
||||
type: "commit"
|
||||
git-tag:
|
||||
type: "git"
|
||||
args:
|
||||
type: "tag"
|
||||
registry-digest:
|
||||
type: "registry"
|
||||
registry-tag:
|
||||
type: "registry"
|
||||
args:
|
||||
type: "tag"
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
|
||||
# Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our
|
||||
community a harassment-free experience for everyone, regardless of age, body
|
||||
size, visible or invisible disability, ethnicity, sex characteristics, gender
|
||||
identity and expression, level of experience, education, socio-economic status,
|
||||
nationality, personal appearance, race, caste, color, religion, or sexual
|
||||
identity and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming,
|
||||
diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to a positive environment for our
|
||||
community include:
|
||||
|
||||
- Demonstrating empathy and kindness toward other people
|
||||
- Being respectful of differing opinions, viewpoints, and experiences
|
||||
- Giving and gracefully accepting constructive feedback
|
||||
- Accepting responsibility and apologizing to those affected by our mistakes,
|
||||
and learning from the experience
|
||||
- Focusing on what is best not just for us as individuals, but for the overall
|
||||
community
|
||||
|
||||
Examples of unacceptable behavior include:
|
||||
|
||||
- The use of sexualized language or imagery, and sexual attention or advances of
|
||||
any kind
|
||||
- Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
- Public or private harassment
|
||||
- Publishing others' private information, such as a physical or email address,
|
||||
without their explicit permission
|
||||
- Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of
|
||||
acceptable behavior and will take appropriate and fair corrective action in
|
||||
response to any behavior that they deem inappropriate, threatening, offensive,
|
||||
or harmful.
|
||||
|
||||
Community leaders have the right and responsibility to remove, edit, or reject
|
||||
comments, commits, code, wiki edits, issues, and other contributions that are
|
||||
not aligned to this Code of Conduct, and will communicate reasons for moderation
|
||||
decisions when appropriate.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when
|
||||
an individual is officially representing the community in public spaces.
|
||||
Examples of representing our community include using an official e-mail address,
|
||||
posting via an official social media account, or acting as an appointed
|
||||
representative at an online or offline event.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement at
|
||||
<git@bmitch.net> or slack (I'm found on the CNCF, Docker, OCI, and OpenSSF
|
||||
slacks).
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
reporter of any incident.
|
||||
|
||||
## Enforcement Guidelines
|
||||
|
||||
Community leaders will follow these Community Impact Guidelines in determining
|
||||
the consequences for any action they deem in violation of this Code of Conduct:
|
||||
|
||||
### 1. Correction
|
||||
|
||||
**Community Impact**: Use of inappropriate language or other behavior deemed
|
||||
unprofessional or unwelcome in the community.
|
||||
|
||||
**Consequence**: A private, written warning from community leaders, providing
|
||||
clarity around the nature of the violation and an explanation of why the
|
||||
behavior was inappropriate. A public apology may be requested.
|
||||
|
||||
### 2. Warning
|
||||
|
||||
**Community Impact**: A violation through a single incident or series of
|
||||
actions.
|
||||
|
||||
**Consequence**: A warning with consequences for continued behavior. No
|
||||
interaction with the people involved, including unsolicited interaction with
|
||||
those enforcing the Code of Conduct, for a specified period of time. This
|
||||
includes avoiding interactions in community spaces as well as external channels
|
||||
like social media. Violating these terms may lead to a temporary or permanent
|
||||
ban.
|
||||
|
||||
### 3. Temporary Ban
|
||||
|
||||
**Community Impact**: A serious violation of community standards, including
|
||||
sustained inappropriate behavior.
|
||||
|
||||
**Consequence**: A temporary ban from any sort of interaction or public
|
||||
communication with the community for a specified period of time. No public or
|
||||
private interaction with the people involved, including unsolicited interaction
|
||||
with those enforcing the Code of Conduct, is allowed during this period.
|
||||
Violating these terms may lead to a permanent ban.
|
||||
|
||||
### 4. Permanent Ban
|
||||
|
||||
**Community Impact**: Demonstrating a pattern of violation of community
|
||||
standards, including sustained inappropriate behavior, harassment of an
|
||||
individual, or aggression toward or disparagement of classes of individuals.
|
||||
|
||||
**Consequence**: A permanent ban from any sort of public interaction within the
|
||||
community.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
||||
version 2.1, available at
|
||||
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
|
||||
|
||||
Community Impact Guidelines were inspired by
|
||||
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
|
||||
|
||||
For answers to common questions about this code of conduct, see the FAQ at
|
||||
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
|
||||
[https://www.contributor-covenant.org/translations][translations].
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
|
||||
[Mozilla CoC]: https://github.com/mozilla/diversity
|
||||
[FAQ]: https://www.contributor-covenant.org/faq
|
||||
[translations]: https://www.contributor-covenant.org/translations
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
# Contributing
|
||||
|
||||
## Reporting security issues
|
||||
|
||||
Please see [SECURITY.md](security.md) for the process to report security issues.
|
||||
|
||||
## Reporting other issues
|
||||
|
||||
Please search for similar issues and if none are seen, report an issue at [github.com/regclient/regclient/issues](https://github.com/regclient/regclient/issues).
|
||||
|
||||
## Code style
|
||||
|
||||
This project attempts to follow these principles:
|
||||
|
||||
- Code is canonical Go, following styles and patterns commonly used by the Go community.
|
||||
- Dependencies outside of the Go standard library should be minimized.
|
||||
- Dependencies should be pinned to a specific digest and tracked by Go or version-check.
|
||||
- Unit tests are strongly encouraged with a focus on test coverage of the successful path and common errors.
|
||||
- Linters and other style formatting tools are used, please run `make all` before committing any changes.
|
||||
|
||||
## LLM Policy
|
||||
|
||||
This project expects all contributions to be developed by a human or created with a reproducible tool.
|
||||
Developers using an AI/LLM tool to generate their contribution are expected to fully understand the entire contribution and the logic behind its design.
|
||||
Contributions that appear to have been generated by an AI/LLM without a human review may result in a ban from future contributions to the project.
|
||||
|
||||
## Pull requests
|
||||
|
||||
PRs are welcome following the below guides:
|
||||
|
||||
- For anything beyond a minor fix, opening an issue is suggested to discuss possible solutions.
|
||||
- Changes should be rebased on the main branch.
|
||||
- Changes should be squashed to a single commit per logical change.
|
||||
|
||||
All changes must be signed (`git commit -s`) to indicate you agree to the [Developer Certificate or Origin](https://developercertificate.org/):
|
||||
|
||||
```text
|
||||
Developer Certificate of Origin
|
||||
Version 1.1
|
||||
|
||||
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
|
||||
|
||||
Everyone is permitted to copy and distribute verbatim copies of this
|
||||
license document, but changing it is not allowed.
|
||||
|
||||
|
||||
Developer's Certificate of Origin 1.1
|
||||
|
||||
By making a contribution to this project, I certify that:
|
||||
|
||||
(a) The contribution was created in whole or in part by me and I
|
||||
have the right to submit it under the open source license
|
||||
indicated in the file; or
|
||||
|
||||
(b) The contribution is based upon previous work that, to the best
|
||||
of my knowledge, is covered under an appropriate open source
|
||||
license and I have the right under that license to submit that
|
||||
work with modifications, whether created in whole or in part
|
||||
by me, under the same open source license (unless I am
|
||||
permitted to submit under a different license), as indicated
|
||||
in the file; or
|
||||
|
||||
(c) The contribution was provided directly to me by some other
|
||||
person who certified (a), (b) or (c) and I have not modified
|
||||
it.
|
||||
|
||||
(d) I understand and agree that this project and the contribution
|
||||
are public and that a record of the contribution (including all
|
||||
personal information I submit with it, including my sign-off) is
|
||||
maintained indefinitely and may be redistributed consistent with
|
||||
this project or the open source license(s) involved.
|
||||
```
|
||||
|
||||
The sign-off will include the following message in your commit:
|
||||
|
||||
```text
|
||||
Signed-off-by: Your Name <your-email@example.org>
|
||||
```
|
||||
|
||||
This needs to be your real name, no aliases please.
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2020 The regclient Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
+285
@@ -0,0 +1,285 @@
|
||||
COMMANDS?=regctl regsync regbot
|
||||
BINARIES?=$(addprefix bin/,$(COMMANDS))
|
||||
IMAGES?=$(addprefix docker-,$(COMMANDS))
|
||||
ARTIFACT_PLATFORMS?=linux-amd64 linux-arm64 linux-ppc64le linux-s390x linux-riscv64 darwin-amd64 darwin-arm64 windows-amd64.exe freebsd-amd64
|
||||
ARTIFACTS?=$(foreach cmd,$(addprefix artifacts/,$(COMMANDS)),$(addprefix $(cmd)-,$(ARTIFACT_PLATFORMS)))
|
||||
IMAGE_PLATFORMS?=linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/riscv64
|
||||
VCS_REPO?="https://github.com/regclient/regclient.git"
|
||||
VCS_REF?=$(shell git rev-list -1 HEAD)
|
||||
ifneq ($(shell git status --porcelain 2>/dev/null),)
|
||||
VCS_REF := $(VCS_REF)-dirty
|
||||
endif
|
||||
VCS_VERSION?=$(shell vcs_describe="$$(git describe --all)"; \
|
||||
vcs_version="(devel)"; \
|
||||
if [ "$${vcs_describe}" != "$${vcs_describe#tags/}" ]; then \
|
||||
vcs_version="$${vcs_describe#tags/}"; \
|
||||
elif [ "$${vcs_describe}" != "$${vcs_describe#heads/}" ]; then \
|
||||
vcs_version="$${vcs_describe#heads/}"; \
|
||||
if [ "main" = "$${vcs_version}" ]; then vcs_version=edge; fi; \
|
||||
fi; \
|
||||
echo "$${vcs_version}" | sed -r 's#/+#-#g')
|
||||
VCS_TAG?=$(shell git describe --tags --abbrev=0 2>/dev/null || true)
|
||||
VCS_SEC?=$(shell git log -1 --format=%ct)
|
||||
VCS_DATE?=$(shell date -d "@$(VCS_SEC)" +%Y-%m-%dT%H:%M:%SZ --utc)
|
||||
LD_FLAGS?=-s -w -extldflags -static -buildid= -X \"github.com/regclient/regclient/internal/version.vcsTag=$(VCS_TAG)\"
|
||||
GO_BUILD_FLAGS?=-trimpath -ldflags "$(LD_FLAGS)"
|
||||
DOCKERFILE_EXT?=$(shell if docker build --help 2>/dev/null | grep -q -- '--progress'; then echo ".buildkit"; fi)
|
||||
DOCKER_ARGS?=--build-arg "VCS_REF=$(VCS_REF)" --build-arg "VCS_VERSION=$(VCS_VERSION)" --build-arg "SOURCE_DATE_EPOCH=$(VCS_SEC)" --build-arg "BUILD_DATE=$(VCS_DATE)"
|
||||
GOPATH?=$(shell go env GOPATH)
|
||||
PWD:=$(shell pwd)
|
||||
VER_BUMP?=$(shell command -v version-bump 2>/dev/null)
|
||||
VER_BUMP_CONTAINER?=sudobmitch/version-bump:edge
|
||||
ifeq "$(strip $(VER_BUMP))" ''
|
||||
VER_BUMP=docker run --rm \
|
||||
-v "$(shell pwd)/:$(shell pwd)/" -w "$(shell pwd)" \
|
||||
-u "$(shell id -u):$(shell id -g)" \
|
||||
$(VER_BUMP_CONTAINER)
|
||||
endif
|
||||
MARKDOWN_LINT_VER?=v0.22.1
|
||||
GOFUMPT_VER?=v0.10.0
|
||||
GOMAJOR_VER?=v0.15.0
|
||||
GOSEC_VER?=v2.26.1
|
||||
GO_VULNCHECK_VER?=v1.3.0
|
||||
OSV_SCANNER_VER?=v2.3.8
|
||||
SYFT?=$(shell command -v syft 2>/dev/null)
|
||||
SYFT_CMD_VER:=$(shell [ -x "$(SYFT)" ] && echo "v$$($(SYFT) version | awk '/^Version: / {print $$2}')" || echo "0")
|
||||
SYFT_VERSION?=v1.44.0
|
||||
SYFT_CONTAINER?=anchore/syft:v1.44.0@sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6
|
||||
ifneq "$(SYFT_CMD_VER)" "$(SYFT_VERSION)"
|
||||
SYFT=docker run --rm \
|
||||
-v "$(shell pwd)/:$(shell pwd)/" -w "$(shell pwd)" \
|
||||
-u "$(shell id -u):$(shell id -g)" \
|
||||
$(SYFT_CONTAINER)
|
||||
endif
|
||||
STATICCHECK_VER?=v0.7.0
|
||||
CI_DISTRIBUTION_VER?=3.1.1
|
||||
CI_ZOT_VER?=v2.1.17
|
||||
|
||||
.PHONY: .FORCE
|
||||
.FORCE:
|
||||
|
||||
.PHONY: all
|
||||
all: fmt gofumpt gofix goimports vet test lint binaries ## Full build of Go binaries (including fmt, vet, test, and lint)
|
||||
|
||||
.PHONY: fmt
|
||||
fmt: ## go fmt
|
||||
go fmt ./...
|
||||
|
||||
.PHONY: gofumpt
|
||||
gofumpt: $(GOPATH)/bin/gofumpt ## gofumpt is a stricter alternative to go fmt
|
||||
gofumpt -l -w .
|
||||
|
||||
.PHONY: gofix
|
||||
gofix: ## go fix
|
||||
go fix ./...
|
||||
|
||||
goimports: $(GOPATH)/bin/goimports
|
||||
$(GOPATH)/bin/goimports -w -format-only -local github.com/regclient .
|
||||
|
||||
.PHONY: vet
|
||||
vet: ## go vet
|
||||
go vet ./...
|
||||
|
||||
.PHONY: test
|
||||
test: ## go test
|
||||
go test -cover -race ./...
|
||||
|
||||
.PHONY: lint
|
||||
lint: lint-go lint-goimports lint-md lint-gosec ## Run all linting
|
||||
|
||||
.PHONY: lint-go
|
||||
lint-go: $(GOPATH)/bin/gofumpt $(GOPATH)/bin/staticcheck .FORCE ## Run linting for Go
|
||||
$(GOPATH)/bin/staticcheck -checks all ./...
|
||||
$(GOPATH)/bin/gofumpt -l -d .
|
||||
errors=$$(go fix -diff ./...); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi
|
||||
|
||||
lint-goimports: $(GOPATH)/bin/goimports
|
||||
@if [ -n "$$($(GOPATH)/bin/goimports -l -format-only -local github.com/regclient .)" ]; then \
|
||||
echo $(GOPATH)/bin/goimports -d -format-only -local github.com/regclient .; \
|
||||
$(GOPATH)/bin/goimports -d -format-only -local github.com/regclient .; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# excluding types/platform pending resultion to https://github.com/securego/gosec/issues/1116
|
||||
.PHONY: lint-gosec
|
||||
lint-gosec: $(GOPATH)/bin/gosec .FORCE ## Run gosec
|
||||
$(GOPATH)/bin/gosec -terse -exclude-dir types/platform ./...
|
||||
|
||||
.PHONY: lint-md
|
||||
lint-md: .FORCE ## Run linting for markdown
|
||||
docker run --rm -v "$(PWD):/workdir:ro" davidanson/markdownlint-cli2:$(MARKDOWN_LINT_VER) \
|
||||
"**/*.md" "#vendor"
|
||||
|
||||
.PHONY: vulnerability-scan
|
||||
vulnerability-scan: osv-scanner vulncheck-go ## Run all vulnerability scanners
|
||||
|
||||
.PHONY: osv-scanner
|
||||
osv-scanner: $(GOPATH)/bin/osv-scanner .FORCE ## Run OSV Scanner
|
||||
$(GOPATH)/bin/osv-scanner scan --config .osv-scanner.toml -r --licenses="Apache-2.0,BSD-3-Clause,MIT,CC-BY-SA-4.0,UNKNOWN" .
|
||||
|
||||
.PHONY: vulncheck-go
|
||||
vulncheck-go: $(GOPATH)/bin/govulncheck .FORCE ## Run govulncheck
|
||||
$(GOPATH)/bin/govulncheck ./...
|
||||
|
||||
.PHONY: vendor
|
||||
vendor: ## Vendor Go modules
|
||||
go mod vendor
|
||||
|
||||
.PHONY: binaries
|
||||
binaries: $(BINARIES) ## Build Go binaries
|
||||
|
||||
bin/%: .FORCE
|
||||
CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o bin/$* ./cmd/$*
|
||||
|
||||
.PHONY: docker
|
||||
docker: $(IMAGES) ## Build Docker images
|
||||
|
||||
docker-%: .FORCE
|
||||
docker build -t regclient/$* -f build/Dockerfile.$*$(DOCKERFILE_EXT) $(DOCKER_ARGS) .
|
||||
docker build -t regclient/$*:alpine -f build/Dockerfile.$*$(DOCKERFILE_EXT) --target release-alpine $(DOCKER_ARGS) .
|
||||
|
||||
.PHONY: oci-image
|
||||
oci-image: $(addprefix oci-image-,$(COMMANDS)) ## Build reproducible images to an OCI Layout
|
||||
|
||||
oci-image-%: bin/regctl .FORCE
|
||||
PATH="$(PWD)/bin:$(PATH)" build/oci-image.sh -r scratch -i "$*" -p "$(IMAGE_PLATFORMS)"
|
||||
PATH="$(PWD)/bin:$(PATH)" build/oci-image.sh -r alpine -i "$*" -p "$(IMAGE_PLATFORMS)" -b "alpine:3"
|
||||
|
||||
.PHONY: test-docker
|
||||
test-docker: $(addprefix test-docker-,$(COMMANDS)) ## Build multi-platform docker images (but do not tag)
|
||||
|
||||
test-docker-%:
|
||||
docker buildx build --platform="$(IMAGE_PLATFORMS)" -f build/Dockerfile.$*.buildkit .
|
||||
docker buildx build --platform="$(IMAGE_PLATFORMS)" -f build/Dockerfile.$*.buildkit --target release-alpine .
|
||||
|
||||
.PHONY: ci
|
||||
ci: ci-distribution ci-zot ## Run CI tests against self hosted registries
|
||||
|
||||
.PHONY: ci-distribution
|
||||
ci-distribution:
|
||||
docker run --rm -d -p 5000 \
|
||||
--label regclient-ci=true --name regclient-ci-distribution \
|
||||
-e "REGISTRY_STORAGE_DELETE_ENABLED=true" \
|
||||
docker.io/library/registry:$(CI_DISTRIBUTION_VER)
|
||||
./build/ci-test.sh -t localhost:$$(docker port regclient-ci-distribution 5000 | head -1 | cut -f2 -d:)/test-ci
|
||||
docker stop regclient-ci-distribution
|
||||
|
||||
.PHONY: ci-zot
|
||||
ci-zot:
|
||||
docker run --rm -d -p 5000 \
|
||||
--label regclient-ci=true --name regclient-ci-zot \
|
||||
-v "$$(pwd)/build/zot-config.json:/etc/zot/config.json:ro" \
|
||||
ghcr.io/project-zot/zot-linux-amd64:$(CI_ZOT_VER)
|
||||
./build/ci-test.sh -t localhost:$$(docker port regclient-ci-zot 5000 | head -1 | cut -f2 -d:)/test-ci
|
||||
docker stop regclient-ci-zot
|
||||
|
||||
.PHONY: artifacts
|
||||
artifacts: $(ARTIFACTS) ## Generate artifacts
|
||||
|
||||
.PHONY: artifact-pre
|
||||
artifact-pre:
|
||||
mkdir -p artifacts
|
||||
|
||||
artifacts/%: artifact-pre .FORCE
|
||||
@set -e; \
|
||||
target="$*"; \
|
||||
command="$${target%%-*}"; \
|
||||
platform_ext="$${target#*-}"; \
|
||||
platform="$${platform_ext%.*}"; \
|
||||
export GOOS="$${platform%%-*}"; \
|
||||
export GOARCH="$${platform#*-}"; \
|
||||
echo export GOOS=$${GOOS}; \
|
||||
echo export GOARCH=$${GOARCH}; \
|
||||
echo go build ${GO_BUILD_FLAGS} -o "$@" ./cmd/$${command}/; \
|
||||
CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o "$@" ./cmd/$${command}/; \
|
||||
$(SYFT) scan -q "file:$@" --source-name "$${command}" -o cyclonedx-json >"artifacts/$${command}-$${platform}.cyclonedx.json"; \
|
||||
$(SYFT) scan -q "file:$@" --source-name "$${command}" -o spdx-json >"artifacts/$${command}-$${platform}.spdx.json"
|
||||
|
||||
.PHONY: clean
|
||||
clean: ## delete generated content
|
||||
[ ! -d artifacts ] || rm -r artifacts
|
||||
[ ! -d bin ] || rm -r bin
|
||||
[ ! -d output ] || rm -r output
|
||||
[ ! -d vendor ] || rm -r vendor
|
||||
|
||||
.PHONY: plugin-user
|
||||
plugin-user:
|
||||
mkdir -p ${HOME}/.docker/cli-plugins/
|
||||
cp docker-plugin/docker-regclient ${HOME}/.docker/cli-plugins/docker-regctl
|
||||
|
||||
.PHONY: plugin-host
|
||||
plugin-host:
|
||||
sudo cp docker-plugin/docker-regclient /usr/libexec/docker/cli-plugins/docker-regctl
|
||||
|
||||
.PHONY: util-golang-major
|
||||
util-golang-major: $(GOPATH)/bin/gomajor ## check for major dependency updates
|
||||
$(GOPATH)/bin/gomajor list
|
||||
|
||||
.PHONY: util-golang-update
|
||||
util-golang-update: ## update go module versions
|
||||
go get -u -t ./...
|
||||
go mod tidy
|
||||
[ ! -d vendor ] || go mod vendor
|
||||
|
||||
.PHONY: util-release-preview
|
||||
util-release-preview: $(GOPATH)/bin/gorelease ## preview changes for next release
|
||||
git checkout main
|
||||
./.github/release.sh -d
|
||||
gorelease
|
||||
|
||||
.PHONY: util-release-run
|
||||
util-release-run: ## generate a new release
|
||||
git checkout main
|
||||
./.github/release.sh
|
||||
|
||||
.PHONY: util-version-check
|
||||
util-version-check: ## check all dependencies for updates
|
||||
$(VER_BUMP) check
|
||||
|
||||
.PHONY: util-version-update
|
||||
util-version-update: ## update versions on all dependencies
|
||||
$(VER_BUMP) update
|
||||
|
||||
$(GOPATH)/bin/gofumpt: .FORCE
|
||||
@[ -f "$(GOPATH)/bin/gofumpt" ] \
|
||||
&& [ "$$($(GOPATH)/bin/gofumpt -version | cut -f 1 -d ' ')" = "$(GOFUMPT_VER)" ] \
|
||||
|| go install mvdan.cc/gofumpt@$(GOFUMPT_VER)
|
||||
|
||||
$(GOPATH)/bin/gomajor: .FORCE
|
||||
@[ -f "$(GOPATH)/bin/gomajor" ] \
|
||||
&& [ "$$($(GOPATH)/bin/gomajor version | grep '^version' | cut -f 2 -d ' ')" = "$(GOMAJOR_VER)" ] \
|
||||
|| go install github.com/icholy/gomajor@$(GOMAJOR_VER)
|
||||
|
||||
$(GOPATH)/bin/goimports: .FORCE
|
||||
@[ -f "$(GOPATH)/bin/goimports" ] && [ "$$(go version | cut -f3 -d' ')" = "$$(go version $(GOPATH)/bin/goimports | cut -f2 -d' ')" ] \
|
||||
|| go install golang.org/x/tools/cmd/goimports@latest
|
||||
|
||||
$(GOPATH)/bin/gorelease: .FORCE
|
||||
@[ -f "$(GOPATH)/bin/gorelease" ] && [ "$$(go version | cut -f3 -d' ')" = "$$(go version $(GOPATH)/bin/gorelease | cut -f2 -d' ')" ] \
|
||||
|| go install golang.org/x/exp/cmd/gorelease@latest
|
||||
|
||||
$(GOPATH)/bin/gosec: .FORCE
|
||||
@[ -f $(GOPATH)/bin/gosec ] \
|
||||
&& [ "$$($(GOPATH)/bin/gosec -version | grep '^Version' | cut -f 2 -d ' ')" = "$(GOSEC_VER)" ] \
|
||||
|| go install -ldflags '-X main.Version=$(GOSEC_VER) -X main.GitTag=$(GOSEC_VER)' \
|
||||
github.com/securego/gosec/v2/cmd/gosec@$(GOSEC_VER)
|
||||
|
||||
$(GOPATH)/bin/staticcheck: .FORCE
|
||||
@[ -f $(GOPATH)/bin/staticcheck ] \
|
||||
&& [ "$$($(GOPATH)/bin/staticcheck -version | cut -f 3 -d ' ' | tr -d '()')" = "$(STATICCHECK_VER)" ] \
|
||||
|| go install "honnef.co/go/tools/cmd/staticcheck@$(STATICCHECK_VER)"
|
||||
|
||||
$(GOPATH)/bin/govulncheck: .FORCE
|
||||
@[ -f $(GOPATH)/bin/govulncheck ] \
|
||||
&& [ $$(go version -m $(GOPATH)/bin/govulncheck | \
|
||||
awk -F ' ' '{ if ($$1 == "mod" && $$2 == "golang.org/x/vuln") { printf "%s\n", $$3 } }') = "$(GO_VULNCHECK_VER)" ] \
|
||||
|| CGO_ENABLED=0 go install "golang.org/x/vuln/cmd/govulncheck@$(GO_VULNCHECK_VER)"
|
||||
|
||||
$(GOPATH)/bin/osv-scanner: .FORCE
|
||||
@[ -f $(GOPATH)/bin/osv-scanner ] \
|
||||
&& [ "$$(osv-scanner --version | awk -F ': ' '{ if ($$1 == "osv-scanner version") { printf "%s\n", $$2 } }')" = "$(OSV_SCANNER_VER)" ] \
|
||||
|| CGO_ENABLED=0 go install "github.com/google/osv-scanner/v2/cmd/osv-scanner@$(OSV_SCANNER_VER)"
|
||||
|
||||
.PHONY: help
|
||||
help: # Display help
|
||||
@awk -F ':|##' '/^[^\t].+?:.*?##/ { printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF }' $(MAKEFILE_LIST)
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
# regclient
|
||||
|
||||
[](https://github.com/regclient/regclient/actions/workflows/go.yml)
|
||||
[](https://github.com/regclient/regclient/actions/workflows/docker.yml)
|
||||
[](https://github.com/regclient/regclient/actions/workflows/version-check.yml)
|
||||
[](https://github.com/regclient/regclient/actions/workflows/vulnscans.yml)
|
||||
|
||||
[](https://pkg.go.dev/github.com/regclient/regclient)
|
||||

|
||||
[](https://goreportcard.com/report/github.com/regclient/regclient)
|
||||
[](https://github.com/regclient/regclient/releases)
|
||||
|
||||
regclient is a client interface to OCI conformant registries and content shipped with the OCI Image Layout.
|
||||
It includes a Go library and several CLI commands.
|
||||
|
||||
## regclient Go Library Features
|
||||
|
||||
- Runs without a container runtime and without privileged access to the local host.
|
||||
- Querying for a tag listing, repository listing, and remotely inspecting the contents of images.
|
||||
- Efficiently copying and retagging images, only pulling layers when required, and without changing the image digest.
|
||||
- Support for multi-platform images.
|
||||
- Support for querying, creating, and copying OCI Artifacts, allowing arbitrary data to be stored in an OCI registry.
|
||||
- Support for packaging OCI Artifacts with an Index of multiple artifacts, which can be used for platform specific artifacts.
|
||||
- Support for querying OCI referrers, copying referrers, and pushing content with an OCI subject field, associating artifacts with other content on the registry.
|
||||
- Support for the “digest tags” used by projects like sigstore/cosign, allowing the content to be included when copying images.
|
||||
- Efficiently query for an image digest.
|
||||
- Efficiently query for pull rate limits used by Docker Hub.
|
||||
- Import and export content into OCI Layouts and Docker formatted tar files.
|
||||
- Support OCI Layouts in all commands as a local disk equivalent of a repository.
|
||||
- Support for deleting tags, manifests, and blobs.
|
||||
- Ability to mutate existing images, including:
|
||||
- Settings annotations or labels
|
||||
- Deleting content from layers
|
||||
- Changing timestamps for reproducibility
|
||||
- Converting between Docker and OCI media types
|
||||
- Replacing the base image layers
|
||||
- Add or remove volumes and exposed ports
|
||||
- Change digest algorithms
|
||||
- Support for registry warning headers, which may be used to notify users of issues with the server or content they are using.
|
||||
- Automatically import logins from the docker CLI, and registry certificates from the docker engine.
|
||||
- Automatic retry, and fallback to a chunked blob push, when network issues are encountered.
|
||||
|
||||
The full Go references is available on [pkg.go.dev](https://pkg.go.dev/github.com/regclient/regclient).
|
||||
|
||||
## regctl Features
|
||||
|
||||
`regctl` is a CLI interface to the `regclient` library.
|
||||
In addition to the features listed for `regclient`, `regctl` adds the following abilities:
|
||||
|
||||
- Generating multi-platform manifests from multiple images that may have been separately built.
|
||||
- Repackage a multi-platform image with only the requested platforms.
|
||||
- Push and pull arbitrary OCI artifacts.
|
||||
- Recursively list all content associated with an image.
|
||||
- Extract files from a layer or image.
|
||||
- Compare images, showing the differences between manifests, the config, and layers.
|
||||
- Formatted output using Go templates.
|
||||
|
||||
The project website includes [usage instructions](https://regclient.org/usage/regctl/) and a [CLI reference](https://regclient.org/cli/regctl/).
|
||||
|
||||
## regsync features
|
||||
|
||||
`regsync` is an image mirroring tool.
|
||||
It will copy images between two locations with the following additional features:
|
||||
|
||||
- Ability to run on a cron schedule, one time synchronization, or only report stale images.
|
||||
- Uses a yaml configuration.
|
||||
- Each source may be an entire registry (not recommended), a repository, or a single image, with the ability to filter repositories and tags.
|
||||
- Support for multi-platform images, OCI referrers, “digest tags”, and copying to or from an OCI Layout (for maintaining a mirror over an air-gap).
|
||||
- Ability to mirror multiple images concurrently.
|
||||
- Support for copying a single platform from multi-platform images.
|
||||
- Ability to backup an existing image before overwriting the tag.
|
||||
- Ability to postpone mirror step when rate limit (used by Docker Hub) is below a threshold.
|
||||
- Can use user’s docker configuration for user credentials and registry certificates.
|
||||
|
||||
The project website includes [usage instructions](https://regclient.org/usage/regsync/) and a [CLI reference](https://regclient.org/cli/regsync/).
|
||||
|
||||
## regbot features
|
||||
|
||||
`regbot` is a scripting tool on top of the `regclient` API with the following features:
|
||||
|
||||
- Ability to run on a cron schedule, one time execution, or test with a dry-run mode.
|
||||
- Uses a yaml configuration.
|
||||
- Scripts are written in Lua and executed directly in Go.
|
||||
- Built-in functions include:
|
||||
- Repository list
|
||||
- Tag list
|
||||
- Image manifest (either head or get, and optional resolving multi-platform reference)
|
||||
- Image config (this includes the creation time, labels, and other details shown in a docker image inspect)
|
||||
- Image rate limit and a wait function to delay the script when rate limit remaining is below a threshold
|
||||
- Image copy
|
||||
- Manifest delete
|
||||
- Tag delete
|
||||
|
||||
The project website includes [usage instructions](https://regclient.org/usage/regbot/) and a [CLI reference](https://regclient.org/cli/regbot/).
|
||||
|
||||
## Development Status
|
||||
|
||||
This project is using v0 version numbers due to Go's backwards compatibility requirements of a v1 release.
|
||||
The library and commands are stable for external use.
|
||||
Minor version updates may contain breaking changes, however effort is made to first deprecate and provide warnings to give users time to move off of older APIs and commands.
|
||||
|
||||
## Installing
|
||||
|
||||
See the [installation instructions](https://regclient.org/install/) on the project website for the various ways to download or build CLI binaries.
|
||||
|
||||
## Usage
|
||||
|
||||
See the [project documentation](https://regclient.org/usage/).
|
||||
|
||||
## Contributors
|
||||
|
||||
<a href="https://github.com/regclient/regclient/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=regclient/regclient" alt="contributor list"/>
|
||||
</a>
|
||||
|
||||
<!-- markdownlint-disable-file MD033 -->
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
# Reporting security issues
|
||||
|
||||
Please report security issues directly in GitHub at <https://github.com/regclient/regclient/security/advisories/new> or alternatively email <git@bmitch.net>.
|
||||
|
||||
We will typically respond within 7 working days of your report. If the issue is confirmed as a vulnerability, we will open a Security Advisory and acknowledge your contributions as part of it. This project follows a 90 day disclosure timeline.
|
||||
+283
@@ -0,0 +1,283 @@
|
||||
package regclient
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/regclient/regclient/internal/pqueue"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types"
|
||||
"github.com/regclient/regclient/types/blob"
|
||||
"github.com/regclient/regclient/types/descriptor"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/warning"
|
||||
)
|
||||
|
||||
const blobCBFreq = time.Millisecond * 100
|
||||
|
||||
type blobOpt struct {
|
||||
callback func(kind types.CallbackKind, instance string, state types.CallbackState, cur, total int64)
|
||||
readerHook func(*blob.BReader) (*blob.BReader, error)
|
||||
}
|
||||
|
||||
// BlobOpts define options for the Image* commands.
|
||||
type BlobOpts func(*blobOpt)
|
||||
|
||||
// BlobWithCallback provides progress data to a callback function.
|
||||
func BlobWithCallback(callback func(kind types.CallbackKind, instance string, state types.CallbackState, cur, total int64)) BlobOpts {
|
||||
return func(opts *blobOpt) {
|
||||
opts.callback = callback
|
||||
}
|
||||
}
|
||||
|
||||
// BlobWithReaderHook is called in [RegClient.BlobCopy] with the blob source.
|
||||
// The returned [blob.BReader] is pushed to the target.
|
||||
// If the hook returns an error, the copy will fail.
|
||||
func BlobWithReaderHook(hook func(*blob.BReader) (*blob.BReader, error)) BlobOpts {
|
||||
return func(opts *blobOpt) {
|
||||
opts.readerHook = hook
|
||||
}
|
||||
}
|
||||
|
||||
// BlobCopy copies a blob between two locations.
|
||||
// If the blob already exists in the target, the copy is skipped.
|
||||
// A server side cross repository blob mount is attempted.
|
||||
func (rc *RegClient) BlobCopy(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor, opts ...BlobOpts) error {
|
||||
if !refSrc.IsSetRepo() {
|
||||
return fmt.Errorf("refSrc is not set: %s%.0w", refSrc.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
if !refTgt.IsSetRepo() {
|
||||
return fmt.Errorf("refTgt is not set: %s%.0w", refTgt.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
var opt blobOpt
|
||||
for _, optFn := range opts {
|
||||
optFn(&opt)
|
||||
}
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
tDesc := d
|
||||
tDesc.URLs = []string{} // ignore URLs when pushing to target
|
||||
if opt.callback != nil {
|
||||
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackStarted, 0, d.Size)
|
||||
}
|
||||
// for the same repository, there's nothing to copy
|
||||
if ref.EqualRepository(refSrc, refTgt) {
|
||||
if opt.callback != nil {
|
||||
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
|
||||
}
|
||||
rc.slog.Debug("Blob copy skipped, same repo",
|
||||
slog.String("src", refSrc.Reference),
|
||||
slog.String("tgt", refTgt.Reference),
|
||||
slog.String("digest", string(d.Digest)))
|
||||
return nil
|
||||
}
|
||||
// check if layer already exists
|
||||
if _, err := rc.BlobHead(ctx, refTgt, tDesc); err == nil {
|
||||
if opt.callback != nil {
|
||||
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
|
||||
}
|
||||
rc.slog.Debug("Blob copy skipped, already exists",
|
||||
slog.String("src", refSrc.Reference),
|
||||
slog.String("tgt", refTgt.Reference),
|
||||
slog.String("digest", string(d.Digest)))
|
||||
return nil
|
||||
}
|
||||
// acquire throttle for both src and tgt to avoid deadlocks
|
||||
tList := []*pqueue.Queue[reqmeta.Data]{}
|
||||
schemeSrcAPI, err := rc.schemeGet(refSrc.Scheme)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
schemeTgtAPI, err := rc.schemeGet(refTgt.Scheme)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tSrc, ok := schemeSrcAPI.(scheme.Throttler); ok {
|
||||
tList = append(tList, tSrc.Throttle(refSrc, false)...)
|
||||
}
|
||||
if tTgt, ok := schemeTgtAPI.(scheme.Throttler); ok {
|
||||
tList = append(tList, tTgt.Throttle(refTgt, true)...)
|
||||
}
|
||||
if len(tList) > 0 {
|
||||
ctxMulti, done, err := pqueue.AcquireMulti[reqmeta.Data](ctx, reqmeta.Data{Kind: reqmeta.Blob, Size: d.Size}, tList...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if done != nil {
|
||||
defer done()
|
||||
}
|
||||
ctx = ctxMulti
|
||||
}
|
||||
|
||||
// try mounting blob from the source repo is the registry is the same
|
||||
if ref.EqualRegistry(refSrc, refTgt) {
|
||||
err := rc.BlobMount(ctx, refSrc, refTgt, d)
|
||||
if err == nil {
|
||||
if opt.callback != nil {
|
||||
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
|
||||
}
|
||||
rc.slog.Debug("Blob copy performed server side with registry mount",
|
||||
slog.String("src", refSrc.Reference),
|
||||
slog.String("tgt", refTgt.Reference),
|
||||
slog.String("digest", string(d.Digest)))
|
||||
return nil
|
||||
}
|
||||
rc.slog.Warn("Failed to mount blob",
|
||||
slog.String("src", refSrc.Reference),
|
||||
slog.String("tgt", refTgt.Reference),
|
||||
slog.String("err", err.Error()))
|
||||
}
|
||||
// fast options failed, download layer from source and push to target
|
||||
blobIO, err := rc.BlobGet(ctx, refSrc, d)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
rc.slog.Warn("Failed to retrieve blob",
|
||||
slog.String("src", refSrc.Reference),
|
||||
slog.String("digest", string(d.Digest)),
|
||||
slog.String("err", err.Error()))
|
||||
}
|
||||
return err
|
||||
}
|
||||
if opt.callback != nil {
|
||||
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackStarted, 0, d.Size)
|
||||
ticker := time.NewTicker(blobCBFreq)
|
||||
done := make(chan bool)
|
||||
defer func() {
|
||||
close(done)
|
||||
ticker.Stop()
|
||||
if ctx.Err() == nil {
|
||||
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackFinished, d.Size, d.Size)
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
offset, err := blobIO.Seek(0, io.SeekCurrent)
|
||||
if err == nil && offset > 0 {
|
||||
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackActive, offset, d.Size)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
if opt.readerHook != nil {
|
||||
blobIO, err = opt.readerHook(blobIO)
|
||||
if err != nil {
|
||||
rc.slog.Warn("Failed to apply reader hook to blob",
|
||||
slog.String("src", refSrc.Reference),
|
||||
slog.String("err", err.Error()))
|
||||
return err
|
||||
}
|
||||
}
|
||||
defer blobIO.Close()
|
||||
if _, err := rc.BlobPut(ctx, refTgt, blobIO.GetDescriptor(), blobIO); err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
rc.slog.Warn("Failed to push blob",
|
||||
slog.String("src", refSrc.Reference),
|
||||
slog.String("tgt", refTgt.Reference),
|
||||
slog.String("err", err.Error()))
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BlobDelete removes a blob from the registry.
|
||||
// This method should only be used to repair a damaged registry.
|
||||
// Typically a server side garbage collection should be used to purge unused blobs.
|
||||
func (rc *RegClient) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error {
|
||||
if !r.IsSetRepo() {
|
||||
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return schemeAPI.BlobDelete(ctx, r, d)
|
||||
}
|
||||
|
||||
// BlobGet retrieves a blob, returning a reader.
|
||||
// This reader must be closed to free up resources that limit concurrent pulls.
|
||||
func (rc *RegClient) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
|
||||
data, err := d.GetData()
|
||||
if err == nil {
|
||||
return blob.NewReader(blob.WithDesc(d), blob.WithRef(r), blob.WithReader(bytes.NewReader(data))), nil
|
||||
}
|
||||
if !r.IsSetRepo() {
|
||||
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return schemeAPI.BlobGet(ctx, r, d)
|
||||
}
|
||||
|
||||
// BlobGetOCIConfig retrieves an OCI config from a blob, automatically extracting the JSON.
|
||||
func (rc *RegClient) BlobGetOCIConfig(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.OCIConfig, error) {
|
||||
if !r.IsSetRepo() {
|
||||
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
b, err := rc.BlobGet(ctx, r, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.ToOCIConfig()
|
||||
}
|
||||
|
||||
// BlobHead is used to verify if a blob exists and is accessible.
|
||||
func (rc *RegClient) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
|
||||
if !r.IsSetRepo() {
|
||||
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return schemeAPI.BlobHead(ctx, r, d)
|
||||
}
|
||||
|
||||
// BlobMount attempts to perform a server side copy/mount of the blob between repositories.
|
||||
func (rc *RegClient) BlobMount(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor) error {
|
||||
if !refSrc.IsSetRepo() {
|
||||
return fmt.Errorf("ref is not set: %s%.0w", refSrc.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
if !refTgt.IsSetRepo() {
|
||||
return fmt.Errorf("ref is not set: %s%.0w", refTgt.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(refSrc.Scheme)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return schemeAPI.BlobMount(ctx, refSrc, refTgt, d)
|
||||
}
|
||||
|
||||
// BlobPut uploads a blob to a repository.
|
||||
// Descriptor is optional, leave size and digest to zero value if unknown.
|
||||
// Reader must also be an [io.Seeker] to support chunked upload fallback.
|
||||
//
|
||||
// This will attempt an anonymous blob mount first which some registries may support.
|
||||
// It will then try doing a full put of the blob without chunking (most widely supported).
|
||||
// If the full put fails, it will fall back to a chunked upload (useful for flaky networks).
|
||||
func (rc *RegClient) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error) {
|
||||
if !r.IsSetRepo() {
|
||||
return descriptor.Descriptor{}, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return descriptor.Descriptor{}, err
|
||||
}
|
||||
return schemeAPI.BlobPut(ctx, r, d, rdr)
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// credHelper wraps a command that manages user credentials.
|
||||
type credHelper struct {
|
||||
prog string
|
||||
env map[string]string
|
||||
}
|
||||
|
||||
func newCredHelper(prog string, env map[string]string) *credHelper {
|
||||
return &credHelper{prog: prog, env: env}
|
||||
}
|
||||
|
||||
func (ch *credHelper) run(arg string, input io.Reader) ([]byte, error) {
|
||||
//#nosec G204 only untrusted arg is a hostname which the executed command should not trust
|
||||
cmd := exec.Command(ch.prog, arg)
|
||||
cmd.Env = os.Environ()
|
||||
if ch.env != nil {
|
||||
for k, v := range ch.env {
|
||||
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
}
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdin = input
|
||||
return cmd.Output()
|
||||
}
|
||||
|
||||
type credStore struct {
|
||||
ServerURL string `json:"ServerURL"`
|
||||
Username string `json:"Username"`
|
||||
Secret string `json:"Secret"` //#nosec G117 exported struct intentionally holds secrets
|
||||
}
|
||||
|
||||
// get requests a credential from the helper for a given host.
|
||||
func (ch *credHelper) get(host *Host) error {
|
||||
hostname := host.Hostname
|
||||
if host.CredHost != "" {
|
||||
hostname = host.CredHost
|
||||
}
|
||||
hostIn := strings.NewReader(hostname)
|
||||
credOut := credStore{
|
||||
Username: host.User,
|
||||
Secret: host.Pass,
|
||||
}
|
||||
outB, err := ch.run("get", hostIn)
|
||||
if err != nil {
|
||||
outS := strings.TrimSpace(string(outB))
|
||||
return fmt.Errorf("error getting credentials, output: %s, error: %w", outS, err)
|
||||
}
|
||||
err = json.NewDecoder(bytes.NewReader(outB)).Decode(&credOut)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading credentials: %w", err)
|
||||
}
|
||||
if credOut.Username == tokenUser {
|
||||
host.User = ""
|
||||
host.Pass = ""
|
||||
host.Token = credOut.Secret
|
||||
} else {
|
||||
host.User = credOut.Username
|
||||
host.Pass = credOut.Secret
|
||||
host.Token = ""
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// list returns a list of hosts supported by the credential helper.
|
||||
func (ch *credHelper) list() ([]Host, error) {
|
||||
credList := map[string]string{}
|
||||
outB, err := ch.run("list", bytes.NewReader([]byte{}))
|
||||
if err != nil {
|
||||
outS := strings.TrimSpace(string(outB))
|
||||
return nil, fmt.Errorf("error getting credential list, output: %s, error: %w", outS, err)
|
||||
}
|
||||
err = json.NewDecoder(bytes.NewReader(outB)).Decode(&credList)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading credential list: %w", err)
|
||||
}
|
||||
hostList := []Host{}
|
||||
for host, user := range credList {
|
||||
if !HostValidate(host) {
|
||||
continue
|
||||
}
|
||||
h := HostNewName(host)
|
||||
h.User = user
|
||||
h.CredHelper = ch.prog
|
||||
hostList = append(hostList, *h)
|
||||
}
|
||||
return hostList, nil
|
||||
}
|
||||
|
||||
// TODO: store method not implemented
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/regclient/regclient/internal/conffile"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
)
|
||||
|
||||
const (
|
||||
// dockerEnv is the environment variable used to look for Docker's config.json.
|
||||
dockerEnv = "DOCKER_CONFIG"
|
||||
// dockerEnvConfig is used to inject the config as an environment variable.
|
||||
dockerEnvConfig = "DOCKER_AUTH_CONFIG"
|
||||
// dockerDir is the directory name for Docker's config (inside the users home directory).
|
||||
dockerDir = ".docker"
|
||||
// dockerConfFile is the name of Docker's config file.
|
||||
dockerConfFile = "config.json"
|
||||
// dockerHelperPre is the prefix of docker credential helpers.
|
||||
dockerHelperPre = "docker-credential-"
|
||||
)
|
||||
|
||||
// dockerConfig is used to parse the ~/.docker/config.json
|
||||
type dockerConfig struct {
|
||||
AuthConfigs map[string]dockerAuthConfig `json:"auths"`
|
||||
HTTPHeaders map[string]string `json:"HttpHeaders,omitempty"`
|
||||
DetachKeys string `json:"detachKeys,omitempty"`
|
||||
CredentialsStore string `json:"credsStore,omitempty"`
|
||||
CredentialHelpers map[string]string `json:"credHelpers,omitempty"`
|
||||
Proxies map[string]dockerProxyConfig `json:"proxies,omitempty"`
|
||||
}
|
||||
|
||||
// dockerProxyConfig contains proxy configuration settings
|
||||
type dockerProxyConfig struct {
|
||||
HTTPProxy string `json:"httpProxy,omitempty"`
|
||||
HTTPSProxy string `json:"httpsProxy,omitempty"`
|
||||
NoProxy string `json:"noProxy,omitempty"`
|
||||
FTPProxy string `json:"ftpProxy,omitempty"`
|
||||
AllProxy string `json:"allProxy,omitempty"`
|
||||
}
|
||||
|
||||
// dockerAuthConfig contains the auths
|
||||
type dockerAuthConfig struct {
|
||||
Username string `json:"username,omitempty"`
|
||||
Password string `json:"password,omitempty"` //#nosec G117 exported struct intentionally holds secrets
|
||||
Auth string `json:"auth,omitempty"`
|
||||
|
||||
ServerAddress string `json:"serveraddress,omitempty"`
|
||||
|
||||
// IdentityToken is used to authenticate the user and get
|
||||
// an access token for the registry.
|
||||
IdentityToken string `json:"identitytoken,omitempty"`
|
||||
|
||||
// RegistryToken is a bearer token to be sent to a registry
|
||||
RegistryToken string `json:"registrytoken,omitempty"`
|
||||
}
|
||||
|
||||
// DockerLoad returns a slice of hosts from the users docker config.
|
||||
// This will search for the config.json in either the DOCKER_CONFIG identified directory or the default .docker directory.
|
||||
// It also includes hosts extracted from the DOCKER_AUTH_CONFIG variable.
|
||||
// If the config file is missing and no value is injected using an environment variable, an empty list is returned.
|
||||
func DockerLoad() ([]Host, error) {
|
||||
hosts := []Host{}
|
||||
errList := []error{}
|
||||
// load from a file
|
||||
cf := conffile.New(
|
||||
conffile.WithHomeDir(dockerDir, dockerConfFile, true),
|
||||
conffile.WithEnvDir(dockerEnv, dockerConfFile),
|
||||
)
|
||||
rdr, err := cf.Open()
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
errList = append(errList, err)
|
||||
} else if err == nil {
|
||||
defer rdr.Close()
|
||||
hostsFile, err := dockerParse(rdr)
|
||||
if err != nil {
|
||||
errList = append(errList, err)
|
||||
} else {
|
||||
hosts = append(hosts, hostsFile...)
|
||||
}
|
||||
}
|
||||
// load from an env var
|
||||
hostsEnv, err := DockerLoadEnv(dockerEnvConfig)
|
||||
if err != nil && !errors.Is(err, errs.ErrNotFound) {
|
||||
errList = append(errList, err)
|
||||
} else if err == nil {
|
||||
hosts = append(hosts, hostsEnv...)
|
||||
}
|
||||
// return the concatenated result, only wrapping an error list if necessary
|
||||
if len(errList) == 1 {
|
||||
return hosts, errList[0]
|
||||
} else {
|
||||
return hosts, errors.Join(errList...)
|
||||
}
|
||||
}
|
||||
|
||||
// DockerLoadFile returns a slice of hosts from a named docker config file.
|
||||
func DockerLoadFile(fname string) ([]Host, error) {
|
||||
//#nosec G304 scoping file operations to a directory is not yet a feature of regclient.
|
||||
rdr, err := os.Open(fname)
|
||||
if err != nil && errors.Is(err, fs.ErrNotExist) {
|
||||
return []Host{}, nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rdr.Close()
|
||||
return dockerParse(rdr)
|
||||
}
|
||||
|
||||
// DockerLoadEnv returns a slice of hosts extracted from the config injected in an environment variable.
|
||||
func DockerLoadEnv(envName string) ([]Host, error) {
|
||||
envVal := os.Getenv(envName)
|
||||
if envVal == "" {
|
||||
return []Host{}, errs.ErrNotFound
|
||||
}
|
||||
return dockerParse(strings.NewReader(envVal))
|
||||
}
|
||||
|
||||
// dockerParse parses a docker config into a slice of Hosts.
|
||||
func dockerParse(rdr io.Reader) ([]Host, error) {
|
||||
dc := dockerConfig{}
|
||||
if err := json.NewDecoder(rdr).Decode(&dc); err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, err
|
||||
}
|
||||
hosts := []Host{}
|
||||
for name, auth := range dc.AuthConfigs {
|
||||
if !HostValidate(name) {
|
||||
continue
|
||||
}
|
||||
h, err := dockerAuthToHost(name, dc, auth)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
hosts = append(hosts, h)
|
||||
}
|
||||
// also include default entries for credential helpers
|
||||
for name, helper := range dc.CredentialHelpers {
|
||||
if !HostValidate(name) {
|
||||
continue
|
||||
}
|
||||
h := HostNewName(name)
|
||||
h.CredHelper = dockerHelperPre + helper
|
||||
if _, ok := dc.AuthConfigs[name]; ok {
|
||||
continue // skip fields with auth config
|
||||
}
|
||||
hosts = append(hosts, *h)
|
||||
}
|
||||
// add credStore entries
|
||||
if dc.CredentialsStore != "" {
|
||||
ch := newCredHelper(dockerHelperPre+dc.CredentialsStore, map[string]string{})
|
||||
csHosts, err := ch.list()
|
||||
if err == nil {
|
||||
hosts = append(hosts, csHosts...)
|
||||
}
|
||||
}
|
||||
return hosts, nil
|
||||
}
|
||||
|
||||
// dockerAuthToHost parses an auth entry from a docker config into a Host.
|
||||
func dockerAuthToHost(name string, conf dockerConfig, auth dockerAuthConfig) (Host, error) {
|
||||
helper := ""
|
||||
if conf.CredentialHelpers != nil && conf.CredentialHelpers[name] != "" {
|
||||
helper = dockerHelperPre + conf.CredentialHelpers[name]
|
||||
}
|
||||
// parse base64 auth into user/pass
|
||||
if auth.Auth != "" {
|
||||
var err error
|
||||
auth.Username, auth.Password, err = decodeAuth(auth.Auth)
|
||||
if err != nil {
|
||||
return Host{}, err
|
||||
}
|
||||
}
|
||||
if (auth.Username == "" || auth.Password == "") && auth.IdentityToken == "" && helper == "" {
|
||||
return Host{}, fmt.Errorf("no credentials found for %s", name)
|
||||
}
|
||||
|
||||
h := HostNewName(name)
|
||||
// ignore unknown names
|
||||
if h.Name != DockerRegistry && !strings.HasSuffix(strings.TrimSuffix(name, "/"), h.Name) {
|
||||
return Host{}, fmt.Errorf("rejecting entry with repository: %s", name)
|
||||
}
|
||||
h.User = auth.Username
|
||||
h.Pass = auth.Password
|
||||
h.Token = auth.IdentityToken
|
||||
h.CredHelper = helper
|
||||
return *h, nil
|
||||
}
|
||||
|
||||
// decodeAuth extracts a base64 encoded user:pass into the username and password.
|
||||
func decodeAuth(authStr string) (string, string, error) {
|
||||
if authStr == "" {
|
||||
return "", "", nil
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(authStr)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
userPass := strings.SplitN(string(decoded), ":", 2)
|
||||
if len(userPass) != 2 {
|
||||
return "", "", fmt.Errorf("invalid auth configuration file")
|
||||
}
|
||||
return userPass[0], strings.Trim(userPass[1], "\x00"), nil
|
||||
}
|
||||
+521
@@ -0,0 +1,521 @@
|
||||
// Package config is used for all regclient configuration settings.
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/regclient/regclient/internal/timejson"
|
||||
)
|
||||
|
||||
// TLSConf specifies whether TLS is enabled and verified for a host.
|
||||
type TLSConf int
|
||||
|
||||
const (
|
||||
// TLSUndefined indicates TLS is not passed, defaults to Enabled.
|
||||
TLSUndefined TLSConf = iota
|
||||
// TLSEnabled uses TLS (https) for the connection.
|
||||
TLSEnabled
|
||||
// TLSInsecure uses TLS but does not verify CA.
|
||||
TLSInsecure
|
||||
// TLSDisabled does not use TLS (http).
|
||||
TLSDisabled
|
||||
)
|
||||
|
||||
const (
|
||||
// DockerRegistry is the name resolved in docker images on Hub.
|
||||
DockerRegistry = "docker.io"
|
||||
// DockerRegistryAuth is the name provided in docker's config for Hub.
|
||||
DockerRegistryAuth = "https://index.docker.io/v1/"
|
||||
// DockerRegistryDNS is the host to connect to for Hub.
|
||||
DockerRegistryDNS = "registry-1.docker.io"
|
||||
// defaultExpire is the default time to expire a credential and force re-authentication.
|
||||
defaultExpire = time.Hour * 1
|
||||
// defaultCredHelperRetry is the time to refresh a credential from a failed credential helper command.
|
||||
defaultCredHelperRetry = time.Second * 5
|
||||
// defaultConcurrent is the default number of concurrent registry connections.
|
||||
defaultConcurrent = 3
|
||||
// defaultReqPerSec is the default maximum frequency to send requests to a registry.
|
||||
defaultReqPerSec = 0
|
||||
// tokenUser is the username returned by credential helpers that indicates the password is an identity token.
|
||||
tokenUser = "<token>"
|
||||
)
|
||||
|
||||
// MarshalJSON converts TLSConf to a json string using MarshalText.
|
||||
func (t TLSConf) MarshalJSON() ([]byte, error) {
|
||||
s, err := t.MarshalText()
|
||||
if err != nil {
|
||||
return []byte(""), err
|
||||
}
|
||||
return json.Marshal(string(s))
|
||||
}
|
||||
|
||||
// MarshalText converts TLSConf to a string.
|
||||
func (t TLSConf) MarshalText() ([]byte, error) {
|
||||
var s string
|
||||
switch t {
|
||||
default:
|
||||
s = ""
|
||||
case TLSEnabled:
|
||||
s = "enabled"
|
||||
case TLSInsecure:
|
||||
s = "insecure"
|
||||
case TLSDisabled:
|
||||
s = "disabled"
|
||||
}
|
||||
return []byte(s), nil
|
||||
}
|
||||
|
||||
// UnmarshalJSON converts TLSConf from a json string.
|
||||
func (t *TLSConf) UnmarshalJSON(b []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(b, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
return t.UnmarshalText([]byte(s))
|
||||
}
|
||||
|
||||
// UnmarshalText converts TLSConf from a string.
|
||||
func (t *TLSConf) UnmarshalText(b []byte) error {
|
||||
switch strings.ToLower(string(b)) {
|
||||
default:
|
||||
return fmt.Errorf("unknown TLS value \"%s\"", b)
|
||||
case "":
|
||||
*t = TLSUndefined
|
||||
case "enabled":
|
||||
*t = TLSEnabled
|
||||
case "insecure":
|
||||
*t = TLSInsecure
|
||||
case "disabled":
|
||||
*t = TLSDisabled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Host defines settings for connecting to a registry.
|
||||
type Host struct {
|
||||
Name string `json:"-" yaml:"registry,omitempty"` // Name of the registry (required) (yaml configs pass this as a field, json provides this from the object key)
|
||||
TLS TLSConf `json:"tls,omitempty" yaml:"tls"` // TLS setting: enabled (default), disabled, insecure
|
||||
RegCert string `json:"regcert,omitempty" yaml:"regcert"` // public pem cert of registry
|
||||
ClientCert string `json:"clientCert,omitempty" yaml:"clientCert"` // public pem cert for client (mTLS)
|
||||
ClientKey string `json:"clientKey,omitempty" yaml:"clientKey"` //#nosec G117 private pem cert for client (mTLS)
|
||||
Hostname string `json:"hostname,omitempty" yaml:"hostname"` // hostname of registry, default is the registry name
|
||||
User string `json:"user,omitempty" yaml:"user"` // username, not used with credHelper
|
||||
Pass string `json:"pass,omitempty" yaml:"pass"` //#nosec G117 password, not used with credHelper
|
||||
Token string `json:"token,omitempty" yaml:"token"` // token, experimental for specific APIs
|
||||
CredHelper string `json:"credHelper,omitempty" yaml:"credHelper"` // credential helper command for requesting logins
|
||||
CredExpire timejson.Duration `json:"credExpire,omitempty" yaml:"credExpire"` // time until credential expires
|
||||
CredHost string `json:"credHost,omitempty" yaml:"credHost"` // used when a helper hostname doesn't match Hostname
|
||||
PathPrefix string `json:"pathPrefix,omitempty" yaml:"pathPrefix"` // used for mirrors defined within a repository namespace
|
||||
Mirrors []string `json:"mirrors,omitempty" yaml:"mirrors"` // list of other Host Names to use as mirrors
|
||||
Priority uint `json:"priority,omitempty" yaml:"priority"` // priority when sorting mirrors, higher priority attempted first
|
||||
RepoAuth bool `json:"repoAuth,omitempty" yaml:"repoAuth"` // tracks a separate auth per repo
|
||||
API string `json:"api,omitempty" yaml:"api"` // Deprecated: registry API to use
|
||||
APIOpts map[string]string `json:"apiOpts,omitempty" yaml:"apiOpts"` // options for APIs
|
||||
BlobChunk int64 `json:"blobChunk,omitempty" yaml:"blobChunk"` // size of each blob chunk
|
||||
BlobMax int64 `json:"blobMax,omitempty" yaml:"blobMax"` // threshold to switch to chunked upload, -1 to disable, 0 for regclient.blobMaxPut
|
||||
ReqPerSec float64 `json:"reqPerSec,omitempty" yaml:"reqPerSec"` // requests per second
|
||||
ReqConcurrent int64 `json:"reqConcurrent,omitempty" yaml:"reqConcurrent"` // concurrent requests, default is defaultConcurrent(3)
|
||||
Scheme string `json:"scheme,omitempty" yaml:"scheme"` // Deprecated: use TLS instead
|
||||
credRefresh time.Time `json:"-" yaml:"-"` // internal use, when to refresh credentials
|
||||
}
|
||||
|
||||
// Cred defines a user credential for accessing a registry.
|
||||
type Cred struct {
|
||||
User, Password, Token string //#nosec G117 exported struct intentionally holds secrets
|
||||
}
|
||||
|
||||
// HostNew creates a default Host entry.
|
||||
func HostNew() *Host {
|
||||
h := Host{
|
||||
TLS: TLSEnabled,
|
||||
APIOpts: map[string]string{},
|
||||
ReqConcurrent: int64(defaultConcurrent),
|
||||
ReqPerSec: float64(defaultReqPerSec),
|
||||
}
|
||||
return &h
|
||||
}
|
||||
|
||||
// HostNewName creates a default Host with a hostname.
|
||||
func HostNewName(name string) *Host {
|
||||
return HostNewDefName(nil, name)
|
||||
}
|
||||
|
||||
// HostNewDefName creates a host using provided defaults and hostname.
|
||||
func HostNewDefName(def *Host, name string) *Host {
|
||||
var h Host
|
||||
if def == nil {
|
||||
h = *HostNew()
|
||||
} else {
|
||||
h = *def
|
||||
// configure required defaults
|
||||
if h.TLS == TLSUndefined {
|
||||
h.TLS = TLSEnabled
|
||||
}
|
||||
if h.APIOpts == nil {
|
||||
h.APIOpts = map[string]string{}
|
||||
}
|
||||
if h.ReqConcurrent == 0 {
|
||||
h.ReqConcurrent = int64(defaultConcurrent)
|
||||
}
|
||||
if h.ReqPerSec == 0 {
|
||||
h.ReqPerSec = float64(defaultReqPerSec)
|
||||
}
|
||||
// copy any fields that are not passed by value
|
||||
if len(h.APIOpts) > 0 {
|
||||
orig := h.APIOpts
|
||||
h.APIOpts = map[string]string{}
|
||||
maps.Copy(h.APIOpts, orig)
|
||||
}
|
||||
if h.Mirrors != nil {
|
||||
orig := h.Mirrors
|
||||
h.Mirrors = make([]string, len(orig))
|
||||
copy(h.Mirrors, orig)
|
||||
}
|
||||
}
|
||||
// configure host
|
||||
scheme, registry, _ := parseName(name)
|
||||
if scheme == "http" {
|
||||
h.TLS = TLSDisabled
|
||||
}
|
||||
// Docker Hub is a special case
|
||||
if registry == DockerRegistry {
|
||||
h.Name = DockerRegistry
|
||||
h.Hostname = DockerRegistryDNS
|
||||
h.CredHost = DockerRegistryAuth
|
||||
return &h
|
||||
}
|
||||
h.Name = registry
|
||||
h.Hostname = registry
|
||||
if name != registry {
|
||||
h.CredHost = name
|
||||
}
|
||||
return &h
|
||||
}
|
||||
|
||||
// HostValidate returns true if the scheme is missing or a known value, and the path is not set.
|
||||
func HostValidate(name string) bool {
|
||||
scheme, _, path := parseName(name)
|
||||
return path == "" && (scheme == "https" || scheme == "http")
|
||||
}
|
||||
|
||||
// GetCred returns the credential, fetching from a credential helper if needed.
|
||||
func (host *Host) GetCred() Cred {
|
||||
// refresh from credHelper if needed
|
||||
if host.CredHelper != "" && (host.credRefresh.IsZero() || time.Now().After(host.credRefresh)) {
|
||||
host.refreshHelper()
|
||||
}
|
||||
return Cred{User: host.User, Password: host.Pass, Token: host.Token}
|
||||
}
|
||||
|
||||
func (host *Host) refreshHelper() {
|
||||
if host.CredHelper == "" {
|
||||
return
|
||||
}
|
||||
if host.CredExpire <= 0 {
|
||||
host.CredExpire = timejson.Duration(defaultExpire)
|
||||
}
|
||||
// run a cred helper, calling get method
|
||||
ch := newCredHelper(host.CredHelper, map[string]string{})
|
||||
err := ch.get(host)
|
||||
if err != nil {
|
||||
host.credRefresh = time.Now().Add(defaultCredHelperRetry)
|
||||
} else {
|
||||
host.credRefresh = time.Now().Add(time.Duration(host.CredExpire))
|
||||
}
|
||||
}
|
||||
|
||||
// IsZero returns true if the struct is set to the zero value or the result of [HostNew].
|
||||
func (host Host) IsZero() bool {
|
||||
if (host.TLS != TLSUndefined && host.TLS != TLSEnabled) ||
|
||||
host.RegCert != "" ||
|
||||
host.ClientCert != "" ||
|
||||
host.ClientKey != "" ||
|
||||
(host.Hostname != "" && host.Hostname != host.Name) ||
|
||||
host.User != "" ||
|
||||
host.Pass != "" ||
|
||||
host.Token != "" ||
|
||||
host.CredHelper != "" ||
|
||||
host.CredExpire != 0 ||
|
||||
host.CredHost != "" ||
|
||||
host.PathPrefix != "" ||
|
||||
len(host.Mirrors) != 0 ||
|
||||
host.Priority != 0 ||
|
||||
host.RepoAuth ||
|
||||
len(host.APIOpts) != 0 ||
|
||||
host.BlobChunk != 0 ||
|
||||
host.BlobMax != 0 ||
|
||||
(host.ReqPerSec != 0 && host.ReqPerSec != float64(defaultReqPerSec)) ||
|
||||
(host.ReqConcurrent != 0 && host.ReqConcurrent != int64(defaultConcurrent)) ||
|
||||
!host.credRefresh.IsZero() {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Merge adds fields from a new config host entry.
|
||||
func (host *Host) Merge(newHost Host, log *slog.Logger) error {
|
||||
name := newHost.Name
|
||||
if name == "" {
|
||||
name = host.Name
|
||||
}
|
||||
if log == nil {
|
||||
log = slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{}))
|
||||
}
|
||||
|
||||
// merge the existing and new config host
|
||||
if host.Name == "" {
|
||||
// only set the name if it's not initialized, this shouldn't normally change
|
||||
host.Name = newHost.Name
|
||||
}
|
||||
|
||||
if newHost.CredHelper == "" && (newHost.Pass != "" || host.Token != "") {
|
||||
// unset existing cred helper for user/pass or token
|
||||
host.CredHelper = ""
|
||||
host.CredExpire = 0
|
||||
}
|
||||
if newHost.CredHelper != "" && newHost.User == "" && newHost.Pass == "" && newHost.Token == "" {
|
||||
// unset existing user/pass/token for cred helper
|
||||
host.User = ""
|
||||
host.Pass = ""
|
||||
host.Token = ""
|
||||
}
|
||||
|
||||
if newHost.User != "" {
|
||||
if host.User != "" && host.User != newHost.User {
|
||||
log.Warn("Changing login user for registry",
|
||||
slog.String("orig", host.User),
|
||||
slog.String("new", newHost.User),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.User = newHost.User
|
||||
}
|
||||
|
||||
if newHost.Pass != "" {
|
||||
if host.Pass != "" && host.Pass != newHost.Pass {
|
||||
log.Warn("Changing login password for registry",
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.Pass = newHost.Pass
|
||||
}
|
||||
|
||||
if newHost.Token != "" {
|
||||
if host.Token != "" && host.Token != newHost.Token {
|
||||
log.Warn("Changing login token for registry",
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.Token = newHost.Token
|
||||
}
|
||||
|
||||
if newHost.CredHelper != "" {
|
||||
if host.CredHelper != "" && host.CredHelper != newHost.CredHelper {
|
||||
log.Warn("Changing credential helper for registry",
|
||||
slog.String("host", name),
|
||||
slog.String("orig", host.CredHelper),
|
||||
slog.String("new", newHost.CredHelper))
|
||||
}
|
||||
host.CredHelper = newHost.CredHelper
|
||||
}
|
||||
|
||||
if newHost.CredExpire != 0 {
|
||||
if host.CredExpire != 0 && host.CredExpire != newHost.CredExpire {
|
||||
log.Warn("Changing credential expire for registry",
|
||||
slog.String("host", name),
|
||||
slog.Any("orig", host.CredExpire),
|
||||
slog.Any("new", newHost.CredExpire))
|
||||
}
|
||||
host.CredExpire = newHost.CredExpire
|
||||
}
|
||||
|
||||
if newHost.CredHost != "" {
|
||||
if host.CredHost != "" && host.CredHost != newHost.CredHost {
|
||||
log.Warn("Changing credential host for registry",
|
||||
slog.String("host", name),
|
||||
slog.String("orig", host.CredHost),
|
||||
slog.String("new", newHost.CredHost))
|
||||
}
|
||||
host.CredHost = newHost.CredHost
|
||||
}
|
||||
|
||||
if newHost.TLS != TLSUndefined {
|
||||
if host.TLS != TLSUndefined && host.TLS != newHost.TLS {
|
||||
tlsOrig, _ := host.TLS.MarshalText()
|
||||
tlsNew, _ := newHost.TLS.MarshalText()
|
||||
log.Warn("Changing TLS settings for registry",
|
||||
slog.String("orig", string(tlsOrig)),
|
||||
slog.String("new", string(tlsNew)),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.TLS = newHost.TLS
|
||||
}
|
||||
|
||||
if newHost.RegCert != "" {
|
||||
if host.RegCert != "" && host.RegCert != newHost.RegCert {
|
||||
log.Warn("Changing certificate settings for registry",
|
||||
slog.String("orig", host.RegCert),
|
||||
slog.String("new", newHost.RegCert),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.RegCert = newHost.RegCert
|
||||
}
|
||||
|
||||
if newHost.ClientCert != "" {
|
||||
if host.ClientCert != "" && host.ClientCert != newHost.ClientCert {
|
||||
log.Warn("Changing client certificate settings for registry",
|
||||
slog.String("orig", host.ClientCert),
|
||||
slog.String("new", newHost.ClientCert),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.ClientCert = newHost.ClientCert
|
||||
}
|
||||
|
||||
if newHost.ClientKey != "" {
|
||||
if host.ClientKey != "" && host.ClientKey != newHost.ClientKey {
|
||||
log.Warn("Changing client certificate key settings for registry",
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.ClientKey = newHost.ClientKey
|
||||
}
|
||||
|
||||
if newHost.Hostname != "" {
|
||||
if host.Hostname != "" && host.Hostname != newHost.Hostname {
|
||||
log.Warn("Changing hostname settings for registry",
|
||||
slog.String("orig", host.Hostname),
|
||||
slog.String("new", newHost.Hostname),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.Hostname = newHost.Hostname
|
||||
}
|
||||
|
||||
if newHost.PathPrefix != "" {
|
||||
newHost.PathPrefix = strings.Trim(newHost.PathPrefix, "/") // leading and trailing / are not needed
|
||||
if host.PathPrefix != "" && host.PathPrefix != newHost.PathPrefix {
|
||||
log.Warn("Changing path prefix settings for registry",
|
||||
slog.String("orig", host.PathPrefix),
|
||||
slog.String("new", newHost.PathPrefix),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.PathPrefix = newHost.PathPrefix
|
||||
}
|
||||
|
||||
if len(newHost.Mirrors) > 0 {
|
||||
if len(host.Mirrors) > 0 && !slices.Equal(host.Mirrors, newHost.Mirrors) {
|
||||
log.Warn("Changing mirror settings for registry",
|
||||
slog.Any("orig", host.Mirrors),
|
||||
slog.Any("new", newHost.Mirrors),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.Mirrors = newHost.Mirrors
|
||||
}
|
||||
|
||||
if newHost.Priority != 0 {
|
||||
if host.Priority != 0 && host.Priority != newHost.Priority {
|
||||
log.Warn("Changing priority settings for registry",
|
||||
slog.Uint64("orig", uint64(host.Priority)),
|
||||
slog.Uint64("new", uint64(newHost.Priority)),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.Priority = newHost.Priority
|
||||
}
|
||||
|
||||
if newHost.RepoAuth {
|
||||
host.RepoAuth = newHost.RepoAuth
|
||||
}
|
||||
|
||||
// TODO: eventually delete
|
||||
if newHost.API != "" {
|
||||
log.Warn("API field has been deprecated",
|
||||
slog.String("api", newHost.API),
|
||||
slog.String("host", name))
|
||||
}
|
||||
|
||||
if len(newHost.APIOpts) > 0 {
|
||||
if len(host.APIOpts) > 0 {
|
||||
merged := maps.Clone(host.APIOpts)
|
||||
for k, v := range newHost.APIOpts {
|
||||
if host.APIOpts[k] != "" && host.APIOpts[k] != v {
|
||||
log.Warn("Changing APIOpts setting for registry",
|
||||
slog.String("orig", host.APIOpts[k]),
|
||||
slog.String("new", newHost.APIOpts[k]),
|
||||
slog.String("opt", k),
|
||||
slog.String("host", name))
|
||||
}
|
||||
merged[k] = v
|
||||
}
|
||||
host.APIOpts = merged
|
||||
} else {
|
||||
host.APIOpts = newHost.APIOpts
|
||||
}
|
||||
}
|
||||
|
||||
if newHost.BlobChunk > 0 {
|
||||
if host.BlobChunk != 0 && host.BlobChunk != newHost.BlobChunk {
|
||||
log.Warn("Changing blobChunk settings for registry",
|
||||
slog.Int64("orig", host.BlobChunk),
|
||||
slog.Int64("new", newHost.BlobChunk),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.BlobChunk = newHost.BlobChunk
|
||||
}
|
||||
|
||||
if newHost.BlobMax != 0 {
|
||||
if host.BlobMax != 0 && host.BlobMax != newHost.BlobMax {
|
||||
log.Warn("Changing blobMax settings for registry",
|
||||
slog.Int64("orig", host.BlobMax),
|
||||
slog.Int64("new", newHost.BlobMax),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.BlobMax = newHost.BlobMax
|
||||
}
|
||||
|
||||
if newHost.ReqPerSec != 0 {
|
||||
if host.ReqPerSec != 0 && host.ReqPerSec != newHost.ReqPerSec {
|
||||
log.Warn("Changing reqPerSec settings for registry",
|
||||
slog.Float64("orig", host.ReqPerSec),
|
||||
slog.Float64("new", newHost.ReqPerSec),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.ReqPerSec = newHost.ReqPerSec
|
||||
}
|
||||
|
||||
if newHost.ReqConcurrent > 0 {
|
||||
if host.ReqConcurrent != 0 && host.ReqConcurrent != newHost.ReqConcurrent {
|
||||
log.Warn("Changing reqPerSec settings for registry",
|
||||
slog.Int64("orig", host.ReqConcurrent),
|
||||
slog.Int64("new", newHost.ReqConcurrent),
|
||||
slog.String("host", name))
|
||||
}
|
||||
host.ReqConcurrent = newHost.ReqConcurrent
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseName splits a registry into the scheme, hostname, and repository/path.
|
||||
func parseName(name string) (string, string, string) {
|
||||
scheme := "https"
|
||||
path := ""
|
||||
// Docker Hub is a special case
|
||||
if name == DockerRegistryAuth || name == DockerRegistryDNS || name == DockerRegistry {
|
||||
return scheme, DockerRegistry, ""
|
||||
}
|
||||
// handle http/https prefix
|
||||
i := strings.Index(name, "://")
|
||||
if i > 0 {
|
||||
scheme = name[:i]
|
||||
name = name[i+3:]
|
||||
}
|
||||
// trim any repository path
|
||||
i = strings.Index(name, "/")
|
||||
if i > 0 {
|
||||
path = name[i+1:]
|
||||
name = name[:i]
|
||||
}
|
||||
return scheme, name, path
|
||||
}
|
||||
+1905
File diff suppressed because it is too large
Load Diff
+923
@@ -0,0 +1,923 @@
|
||||
// Package auth is used for HTTP authentication
|
||||
package auth
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/regclient/regclient/internal/regnet"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
)
|
||||
|
||||
type charLU byte
|
||||
|
||||
var charLUs [256]charLU
|
||||
|
||||
var defaultClientID = "regclient"
|
||||
|
||||
// minTokenLife tokens are required to last at least 60 seconds to support older docker clients
|
||||
var minTokenLife = 60
|
||||
|
||||
// tokenBuffer is used to renew a token before it expires to account for time to process requests on the server
|
||||
var tokenBuffer = time.Second * 5
|
||||
|
||||
const (
|
||||
isSpace charLU = 1 << iota
|
||||
isToken
|
||||
)
|
||||
|
||||
func init() {
|
||||
for c := range 256 {
|
||||
charLUs[c] = 0
|
||||
if strings.ContainsRune(" \t\r\n", rune(c)) {
|
||||
charLUs[c] |= isSpace
|
||||
}
|
||||
if (rune('a') <= rune(c) && rune(c) <= rune('z')) || (rune('A') <= rune(c) && rune(c) <= rune('Z') || (rune('0') <= rune(c) && rune(c) <= rune('9')) || strings.ContainsRune("-._~+/", rune(c))) {
|
||||
charLUs[c] |= isToken
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CredsFn is passed to lookup credentials for a given hostname, response is a username and password or empty strings
|
||||
type CredsFn func(host string) Cred
|
||||
|
||||
// Cred is returned by the CredsFn.
|
||||
// If Token is provided and auth method is bearer, it will attempt to use it as a refresh token.
|
||||
// Else if user and password are provided, they are attempted with all auth methods.
|
||||
// Else if neither are provided and auth method is bearer, an anonymous login is attempted.
|
||||
type Cred struct {
|
||||
//#nosec G117 exported struct intentionally holds secrets
|
||||
User, Password string // clear text username and password
|
||||
Token string // refresh token only used for bearer auth
|
||||
}
|
||||
|
||||
// challenge is the extracted contents of the WWW-Authenticate header.
|
||||
type challenge struct {
|
||||
authType string
|
||||
params map[string]string
|
||||
}
|
||||
|
||||
// handler handles a challenge for a host to return an auth header
|
||||
type handler interface {
|
||||
AddScope(scope string) error
|
||||
ProcessChallenge(challenge) error
|
||||
UpdateRequest(*http.Request) error
|
||||
}
|
||||
|
||||
// handlerBuild is used to make a new handler for a specific authType and URL
|
||||
type handlerBuild func(client *http.Client, clientID, host string, credFn CredsFn, slog *slog.Logger) handler
|
||||
|
||||
// Opts configures options for NewAuth
|
||||
type Opts func(*Auth)
|
||||
|
||||
// Auth is used to handle authentication requests.
|
||||
type Auth struct {
|
||||
httpClient *http.Client
|
||||
clientID string
|
||||
credsFn CredsFn
|
||||
hbs map[string]handlerBuild // handler builders based on authType
|
||||
hs map[string]map[string]handler // handlers based on url and authType
|
||||
authTypes []string
|
||||
slog *slog.Logger
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewAuth creates a new Auth
|
||||
func NewAuth(opts ...Opts) *Auth {
|
||||
a := &Auth{
|
||||
httpClient: &http.Client{},
|
||||
clientID: defaultClientID,
|
||||
credsFn: DefaultCredsFn,
|
||||
hbs: map[string]handlerBuild{},
|
||||
hs: map[string]map[string]handler{},
|
||||
authTypes: []string{},
|
||||
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(a)
|
||||
}
|
||||
|
||||
if len(a.authTypes) == 0 {
|
||||
a.addDefaultHandlers()
|
||||
}
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
// WithCreds provides a user/pass lookup for a url
|
||||
func WithCreds(f CredsFn) Opts {
|
||||
return func(a *Auth) {
|
||||
if f != nil {
|
||||
a.credsFn = f
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithHTTPClient uses a specific http client with requests
|
||||
func WithHTTPClient(h *http.Client) Opts {
|
||||
return func(a *Auth) {
|
||||
if h != nil {
|
||||
a.httpClient = h
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithClientID uses a client ID with request headers
|
||||
func WithClientID(clientID string) Opts {
|
||||
return func(a *Auth) {
|
||||
a.clientID = clientID
|
||||
}
|
||||
}
|
||||
|
||||
// WithHandler includes a handler for a specific auth type
|
||||
func WithHandler(authType string, hb handlerBuild) Opts {
|
||||
return func(a *Auth) {
|
||||
lcat := strings.ToLower(authType)
|
||||
a.hbs[lcat] = hb
|
||||
a.authTypes = append(a.authTypes, lcat)
|
||||
}
|
||||
}
|
||||
|
||||
// WithDefaultHandlers includes a Basic and Bearer handler, this is automatically added with "WithHandler" is not called
|
||||
func WithDefaultHandlers() Opts {
|
||||
return func(a *Auth) {
|
||||
a.addDefaultHandlers()
|
||||
}
|
||||
}
|
||||
|
||||
// WithLog injects a Logger
|
||||
func WithLog(slog *slog.Logger) Opts {
|
||||
return func(a *Auth) {
|
||||
a.slog = slog
|
||||
}
|
||||
}
|
||||
|
||||
// AddScope extends an existing auth with additional scopes.
|
||||
// This is used to pre-populate scopes with the Docker convention rather than
|
||||
// depend on the registry to respond with the correct http status and headers.
|
||||
func (a *Auth) AddScope(host, scope string) error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
success := false
|
||||
if a.hs[host] == nil {
|
||||
return errs.ErrNoNewChallenge
|
||||
}
|
||||
for _, at := range a.authTypes {
|
||||
if a.hs[host][at] != nil {
|
||||
err := a.hs[host][at].AddScope(scope)
|
||||
if err == nil {
|
||||
success = true
|
||||
} else if err != errs.ErrNoNewChallenge {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if !success {
|
||||
return errs.ErrNoNewChallenge
|
||||
}
|
||||
a.slog.Debug("Auth scope added",
|
||||
slog.String("host", host),
|
||||
slog.String("scope", scope))
|
||||
return nil
|
||||
}
|
||||
|
||||
// HandleResponse parses the 401 response, extracting the WWW-Authenticate
|
||||
// header and verifying the requirement is different from what was included in
|
||||
// the last request
|
||||
func (a *Auth) HandleResponse(resp *http.Response) error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
// verify response is an access denied
|
||||
if resp.StatusCode != http.StatusUnauthorized {
|
||||
return errs.ErrUnsupported
|
||||
}
|
||||
|
||||
// extract host and auth header
|
||||
host := resp.Request.URL.Host
|
||||
cl, err := ParseAuthHeaders(resp.Header.Values("WWW-Authenticate"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.slog.Debug("Auth request parsed",
|
||||
slog.Any("challenge", cl))
|
||||
if len(cl) < 1 {
|
||||
return errs.ErrEmptyChallenge
|
||||
}
|
||||
goodChallenge := false
|
||||
// loop over the received challenge(s)
|
||||
for _, c := range cl {
|
||||
if _, ok := a.hbs[c.authType]; !ok {
|
||||
a.slog.Warn("Unsupported auth type",
|
||||
slog.String("authtype", c.authType))
|
||||
continue
|
||||
}
|
||||
// setup a handler for the host and auth type
|
||||
if _, ok := a.hs[host]; !ok {
|
||||
a.hs[host] = map[string]handler{}
|
||||
}
|
||||
if _, ok := a.hs[host][c.authType]; !ok {
|
||||
h := a.hbs[c.authType](a.httpClient, a.clientID, host, a.credsFn, a.slog)
|
||||
if h == nil {
|
||||
continue
|
||||
}
|
||||
a.hs[host][c.authType] = h
|
||||
}
|
||||
// process the challenge with that handler
|
||||
err := a.hs[host][c.authType].ProcessChallenge(c)
|
||||
if err == nil {
|
||||
goodChallenge = true
|
||||
} else if err == errs.ErrNoNewChallenge {
|
||||
// handle race condition when another request updates the challenge
|
||||
// detect that by seeing the current auth header is different
|
||||
prevAH := resp.Request.Header.Get("Authorization")
|
||||
err := a.hs[host][c.authType].UpdateRequest(resp.Request)
|
||||
if err == nil && prevAH != resp.Request.Header.Get("Authorization") {
|
||||
goodChallenge = true
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !goodChallenge {
|
||||
return errs.ErrHTTPUnauthorized
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateRequest adds Authorization headers to a request
|
||||
func (a *Auth) UpdateRequest(req *http.Request) error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
host := req.URL.Host
|
||||
if a.hs[host] == nil {
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
for _, at := range a.authTypes {
|
||||
if a.hs[host][at] != nil {
|
||||
err = a.hs[host][at].UpdateRequest(req)
|
||||
if err != nil {
|
||||
a.slog.Debug("Failed to generate auth",
|
||||
slog.String("err", err.Error()),
|
||||
slog.String("host", host),
|
||||
slog.String("authtype", at))
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Auth) addDefaultHandlers() {
|
||||
if _, ok := a.hbs["basic"]; !ok {
|
||||
a.hbs["basic"] = NewBasicHandler
|
||||
a.authTypes = append(a.authTypes, "basic")
|
||||
}
|
||||
if _, ok := a.hbs["bearer"]; !ok {
|
||||
a.hbs["bearer"] = NewBearerHandler
|
||||
a.authTypes = append(a.authTypes, "bearer")
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultCredsFn is used to return no credentials when auth is not configured with a CredsFn
|
||||
// This avoids the need to check for nil pointers
|
||||
func DefaultCredsFn(h string) Cred {
|
||||
return Cred{}
|
||||
}
|
||||
|
||||
// ParseAuthHeaders extracts the scheme and realm from WWW-Authenticate headers
|
||||
func ParseAuthHeaders(ahl []string) ([]challenge, error) {
|
||||
var cl []challenge
|
||||
for _, ah := range ahl {
|
||||
c, err := parseAuthHeader(ah)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse challenge header: %s, %w", ah, err)
|
||||
}
|
||||
cl = append(cl, c...)
|
||||
}
|
||||
return cl, nil
|
||||
}
|
||||
|
||||
// parseAuthHeader parses a single header line for WWW-Authenticate
|
||||
// Example values:
|
||||
// Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push"
|
||||
// Basic realm="GitHub Package Registry"
|
||||
func parseAuthHeader(ah string) ([]challenge, error) {
|
||||
var cl []challenge
|
||||
var c *challenge
|
||||
curElement := []byte{}
|
||||
curKey := ""
|
||||
stateElement := "string"
|
||||
stateSyntax := "start"
|
||||
|
||||
for _, b := range []byte(ah) {
|
||||
switch stateElement {
|
||||
case "string":
|
||||
// string: ignore leading space, enter quote only if first character, handle escapes, handle valid tokens, else end
|
||||
if charLUs[b]&isToken != 0 {
|
||||
// add valid tokens to element
|
||||
curElement = append(curElement, b)
|
||||
} else if charLUs[b]&isSpace != 0 && len(curElement) == 0 {
|
||||
// ignore leading spaces
|
||||
} else if b == '"' && len(curElement) == 0 {
|
||||
stateElement = "quote"
|
||||
} else if b == '\\' {
|
||||
stateElement = "escape_string"
|
||||
} else {
|
||||
stateElement = "end"
|
||||
}
|
||||
case "quote":
|
||||
// quote: handle escapes, handle closing quote (quote_end to read next character), all other tokens are valid
|
||||
if b == '\\' {
|
||||
stateElement = "escape_quote"
|
||||
} else if b == '"' {
|
||||
stateElement = "quote_end"
|
||||
} else {
|
||||
curElement = append(curElement, b)
|
||||
}
|
||||
case "quote_end":
|
||||
// any character after the close quote is the end of the element
|
||||
stateElement = "end"
|
||||
case "escape_string":
|
||||
// escape_string: handle any character and return to string state
|
||||
curElement = append(curElement, b)
|
||||
stateElement = "string"
|
||||
case "escape_quote":
|
||||
// escape_quote: handle any character and return to quote state
|
||||
curElement = append(curElement, b)
|
||||
stateElement = "quote"
|
||||
case "end":
|
||||
// finished parsing element, continue to processing element according to the current state
|
||||
default:
|
||||
return nil, fmt.Errorf("unhandled element case: %w", errs.ErrParsingFailed)
|
||||
}
|
||||
if stateElement != "end" {
|
||||
// continue parsing the element until it ends
|
||||
continue
|
||||
}
|
||||
|
||||
// syntax looks at each string within the overall challenge syntax
|
||||
switch stateSyntax {
|
||||
case "start":
|
||||
// start: (start of auth_type) read auth_type and space (end_auth_type) or auth_type and comma (start)
|
||||
if charLUs[b]&isSpace != 0 && len(curElement) > 0 {
|
||||
stateSyntax = "end_auth_type"
|
||||
} else if b == ',' && len(curElement) > 0 {
|
||||
// state remains at start
|
||||
} else {
|
||||
return nil, fmt.Errorf("start element did not end with a space or comma: %w", errs.ErrParsingFailed)
|
||||
}
|
||||
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
|
||||
cl = append(cl, *c)
|
||||
case "start_or_param":
|
||||
// start_or_param: (after param_value) read auth_type and space (end_auth_type) or param_key and equals (param_value)
|
||||
if charLUs[b]&isSpace != 0 && len(curElement) > 0 {
|
||||
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
|
||||
cl = append(cl, *c)
|
||||
stateSyntax = "end_auth_type"
|
||||
} else if b == '=' && len(curElement) > 0 {
|
||||
curKey = strings.ToLower(string(curElement))
|
||||
stateSyntax = "param_value"
|
||||
} else {
|
||||
return nil, fmt.Errorf("expected auth type or param: %w", errs.ErrParsingFailed)
|
||||
}
|
||||
case "end_auth_type":
|
||||
// end_auth_type: (after reading auth_type) read param_key and equals (param_value) or just a comma (start)
|
||||
if b == '=' && len(curElement) > 0 {
|
||||
curKey = strings.ToLower(string(curElement))
|
||||
stateSyntax = "param_value"
|
||||
} else if b == ',' && len(curElement) == 0 {
|
||||
// ignore white space between end of auth_type and comma
|
||||
stateSyntax = "start"
|
||||
} else {
|
||||
return nil, fmt.Errorf("expected param or comma: %w", errs.ErrParsingFailed)
|
||||
}
|
||||
case "param_value":
|
||||
// param_value: (after param_key) read param_value and comma (start_or_param)
|
||||
if b == ',' {
|
||||
c.params[curKey] = string(curElement)
|
||||
stateSyntax = "start_or_param"
|
||||
curKey = ""
|
||||
} else {
|
||||
return nil, fmt.Errorf("expected param value: %w", errs.ErrParsingFailed)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unhandled syntax case: %w", errs.ErrParsingFailed)
|
||||
}
|
||||
// reset element state
|
||||
stateElement = "string"
|
||||
curElement = []byte{}
|
||||
}
|
||||
// at end of parsing, if the element is not empty, process according to syntax state:
|
||||
if len(curElement) > 0 {
|
||||
// ensure this is not within an unclosed quote or partial escape
|
||||
if stateElement != "string" && stateElement != "quote_end" {
|
||||
return nil, fmt.Errorf("eol element in state %s: %w", stateElement, errs.ErrParsingFailed)
|
||||
}
|
||||
switch stateSyntax {
|
||||
case "start", "start_or_param":
|
||||
// add a new auth type if a string is seen at the start, before any equals
|
||||
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
|
||||
cl = append(cl, *c)
|
||||
case "param_value":
|
||||
// add the last param key=val
|
||||
c.params[curKey] = string(curElement)
|
||||
case "end_auth_type":
|
||||
// missing equals for param
|
||||
return nil, fmt.Errorf("eol at param without value: %w", errs.ErrParsingFailed)
|
||||
}
|
||||
}
|
||||
|
||||
return cl, nil
|
||||
}
|
||||
|
||||
// basicHandler supports Basic auth type requests
|
||||
type basicHandler struct {
|
||||
realm string
|
||||
host string
|
||||
credsFn CredsFn
|
||||
}
|
||||
|
||||
// NewBasicHandler creates a new BasicHandler
|
||||
func NewBasicHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
|
||||
return &basicHandler{
|
||||
realm: "",
|
||||
host: host,
|
||||
credsFn: credsFn,
|
||||
}
|
||||
}
|
||||
|
||||
// AddScope is not valid for BasicHandler
|
||||
func (b *basicHandler) AddScope(scope string) error {
|
||||
return errs.ErrNoNewChallenge
|
||||
}
|
||||
|
||||
// ProcessChallenge for BasicHandler is a noop
|
||||
func (b *basicHandler) ProcessChallenge(c challenge) error {
|
||||
if _, ok := c.params["realm"]; !ok {
|
||||
return errs.ErrInvalidChallenge
|
||||
}
|
||||
if b.realm != c.params["realm"] {
|
||||
b.realm = c.params["realm"]
|
||||
return nil
|
||||
}
|
||||
return errs.ErrNoNewChallenge
|
||||
}
|
||||
|
||||
// UpdateRequest for BasicHandler generates base64 encoded user/pass for a host
|
||||
func (b *basicHandler) UpdateRequest(req *http.Request) error {
|
||||
cred := b.credsFn(b.host)
|
||||
if cred.User == "" || cred.Password == "" {
|
||||
return fmt.Errorf("no credentials available: %w", errs.ErrHTTPUnauthorized)
|
||||
}
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
|
||||
base64.StdEncoding.EncodeToString([]byte(cred.User+":"+cred.Password))))
|
||||
return nil
|
||||
}
|
||||
|
||||
// bearerHandler supports Bearer auth type requests
|
||||
type bearerHandler struct {
|
||||
client *http.Client
|
||||
clientID string
|
||||
realm, service string
|
||||
host string
|
||||
credsFn CredsFn
|
||||
scopes []string
|
||||
tokenURL *url.URL
|
||||
token bearerToken
|
||||
slog *slog.Logger
|
||||
}
|
||||
|
||||
// bearerToken is the json response to the Bearer request
|
||||
type bearerToken struct {
|
||||
Token string `json:"token"`
|
||||
AccessToken string `json:"access_token"` //#nosec G117 exported struct intentionally holds secrets
|
||||
ExpiresIn int `json:"expires_in"`
|
||||
IssuedAt time.Time `json:"issued_at"`
|
||||
RefreshToken string `json:"refresh_token"` //#nosec G117 exported struct intentionally holds secrets
|
||||
Scope string `json:"scope"`
|
||||
}
|
||||
|
||||
// NewBearerHandler creates a new BearerHandler
|
||||
func NewBearerHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
|
||||
return &bearerHandler{
|
||||
client: client,
|
||||
clientID: clientID,
|
||||
host: host,
|
||||
credsFn: credsFn,
|
||||
realm: "",
|
||||
service: "",
|
||||
scopes: []string{},
|
||||
slog: slog,
|
||||
}
|
||||
}
|
||||
|
||||
// AddScope appends a new scope if it doesn't already exist
|
||||
func (b *bearerHandler) AddScope(scope string) error {
|
||||
if b.scopeExists(scope) {
|
||||
if b.token.Token == "" || !b.isExpired() {
|
||||
return errs.ErrNoNewChallenge
|
||||
}
|
||||
return nil
|
||||
}
|
||||
b.addScope(scope)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *bearerHandler) addScope(scope string) {
|
||||
if !b.tryExtendExistingScope(scope) {
|
||||
b.scopes = append(b.scopes, scope)
|
||||
}
|
||||
// delete old token
|
||||
b.token.Token = ""
|
||||
}
|
||||
|
||||
var knownActions = []string{"pull", "push", "delete"}
|
||||
|
||||
// tryExtendExistingScope extends an existing scope if both the new scope and the current scope contain only knownActions.
|
||||
// It returns true if actions are added or are already present. Otherwise, it returns false,
|
||||
// indicating that the new scope should be appended to b.scopes instead.
|
||||
func (b *bearerHandler) tryExtendExistingScope(scope string) bool {
|
||||
repo, actions, ok := parseScope(scope)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
scopePrefix := "repository:" + repo + ":"
|
||||
for i, cur := range b.scopes {
|
||||
if !strings.HasPrefix(cur, scopePrefix) {
|
||||
continue
|
||||
}
|
||||
_, curActions, curOk := parseScope(cur)
|
||||
if !curOk {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, a := range actions {
|
||||
if !slices.Contains(curActions, a) {
|
||||
curActions = append(curActions, a)
|
||||
}
|
||||
}
|
||||
b.scopes[i] = scopePrefix + strings.Join(curActions, ",")
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseScope splits a scope into the repo and slice of actions.
|
||||
// Unknown actions in the scope will set bool to false.
|
||||
func parseScope(scope string) (string, []string, bool) {
|
||||
scopeSplit := strings.SplitN(scope, ":", 3)
|
||||
if scopeSplit[0] != "repository" || len(scopeSplit) < 3 {
|
||||
return "", nil, false
|
||||
}
|
||||
actionSplit := strings.Split(scopeSplit[2], ",")
|
||||
for _, a := range actionSplit {
|
||||
if !slices.Contains(knownActions, a) {
|
||||
return "", nil, false
|
||||
}
|
||||
}
|
||||
return scopeSplit[1], actionSplit, true
|
||||
}
|
||||
|
||||
// ProcessChallenge handles WWW-Authenticate header for bearer tokens
|
||||
// Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push"
|
||||
func (b *bearerHandler) ProcessChallenge(c challenge) error {
|
||||
if _, ok := c.params["realm"]; !ok {
|
||||
return errs.ErrInvalidChallenge
|
||||
}
|
||||
if _, ok := c.params["service"]; !ok {
|
||||
c.params["service"] = ""
|
||||
}
|
||||
if _, ok := c.params["scope"]; !ok {
|
||||
c.params["scope"] = ""
|
||||
}
|
||||
|
||||
existingScope := b.scopeExists(c.params["scope"])
|
||||
|
||||
if b.realm == c.params["realm"] && b.service == c.params["service"] && existingScope && (b.token.Token == "" || !b.isExpired()) {
|
||||
return errs.ErrNoNewChallenge
|
||||
}
|
||||
|
||||
if b.realm == "" {
|
||||
b.realm = c.params["realm"]
|
||||
} else if b.realm != c.params["realm"] {
|
||||
return errs.ErrInvalidChallenge
|
||||
}
|
||||
if b.service == "" {
|
||||
b.service = c.params["service"]
|
||||
} else if b.service != c.params["service"] {
|
||||
return errs.ErrInvalidChallenge
|
||||
}
|
||||
if !existingScope {
|
||||
b.addScope(c.params["scope"])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateRequest for BearerHandler adds a bearer token to the request.
|
||||
func (b *bearerHandler) UpdateRequest(req *http.Request) error {
|
||||
// handle relative realm values
|
||||
if b.tokenURL == nil {
|
||||
u, err := req.URL.Parse(b.realm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.tokenURL = u
|
||||
}
|
||||
// verify tokenURL is allowed for request URL
|
||||
if err := regnet.AllowRedirect(*req.URL, *b.tokenURL); err != nil {
|
||||
return err
|
||||
}
|
||||
// if unexpired token already exists, return it
|
||||
if b.token.Token != "" && !b.isExpired() {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
|
||||
return nil
|
||||
}
|
||||
// attempt to post if a refresh token is available or token auth is being used
|
||||
cred := b.credsFn(b.host)
|
||||
if b.token.RefreshToken != "" || cred.Token != "" {
|
||||
if err := b.tryPost(cred); err == nil {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
|
||||
return nil
|
||||
} else if err != errs.ErrHTTPUnauthorized {
|
||||
return fmt.Errorf("failed to request auth token (post): %w%.0w", err, errs.ErrHTTPUnauthorized)
|
||||
}
|
||||
}
|
||||
// attempt a get (with basic auth if user/pass available)
|
||||
if err := b.tryGet(cred); err == nil {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
|
||||
return nil
|
||||
} else if err != errs.ErrHTTPUnauthorized {
|
||||
return fmt.Errorf("failed to request auth token (get): %w%.0w", err, errs.ErrHTTPUnauthorized)
|
||||
}
|
||||
return errs.ErrHTTPUnauthorized
|
||||
}
|
||||
|
||||
// isExpired returns true when token issue date is either 0, token has expired,
|
||||
// or will expire within buffer time
|
||||
func (b *bearerHandler) isExpired() bool {
|
||||
if b.token.IssuedAt.IsZero() {
|
||||
return true
|
||||
}
|
||||
expireSec := b.token.IssuedAt.Add(time.Duration(b.token.ExpiresIn) * time.Second)
|
||||
expireSec = expireSec.Add(tokenBuffer * -1)
|
||||
return time.Now().After(expireSec)
|
||||
}
|
||||
|
||||
// tryGet requests a new token with a GET request
|
||||
func (b *bearerHandler) tryGet(cred Cred) error {
|
||||
//#nosec G704 inputs follow specification
|
||||
req, err := http.NewRequest("GET", b.tokenURL.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reqParams := req.URL.Query()
|
||||
reqParams.Add("client_id", b.clientID)
|
||||
// Note, an offline_token should not be requested by default due to broken OAuth2 implementations returning an invalid token
|
||||
if b.service != "" {
|
||||
reqParams.Add("service", b.service)
|
||||
}
|
||||
|
||||
for _, s := range b.scopes {
|
||||
reqParams.Add("scope", s)
|
||||
}
|
||||
|
||||
if cred.User != "" && cred.Password != "" {
|
||||
reqParams.Add("account", cred.User)
|
||||
req.SetBasicAuth(cred.User, cred.Password)
|
||||
}
|
||||
|
||||
req.Header.Add("User-Agent", b.clientID)
|
||||
req.URL.RawQuery = reqParams.Encode()
|
||||
|
||||
//#nosec G704 inputs follow specification
|
||||
resp, err := b.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return b.validateResponse(resp)
|
||||
}
|
||||
|
||||
// tryPost requests a new token via a POST request
|
||||
func (b *bearerHandler) tryPost(cred Cred) error {
|
||||
form := url.Values{}
|
||||
if len(b.scopes) > 0 {
|
||||
form.Set("scope", strings.Join(b.scopes, " "))
|
||||
}
|
||||
if b.service != "" {
|
||||
form.Set("service", b.service)
|
||||
}
|
||||
form.Set("client_id", b.clientID)
|
||||
if b.token.RefreshToken != "" {
|
||||
form.Set("grant_type", "refresh_token")
|
||||
form.Set("refresh_token", b.token.RefreshToken)
|
||||
} else if cred.Token != "" {
|
||||
form.Set("grant_type", "refresh_token")
|
||||
form.Set("refresh_token", cred.Token)
|
||||
} else if cred.User != "" && cred.Password != "" {
|
||||
form.Set("grant_type", "password")
|
||||
form.Set("username", cred.User)
|
||||
form.Set("password", cred.Password)
|
||||
}
|
||||
|
||||
//#nosec G704 inputs are user controlled or follow specification
|
||||
req, err := http.NewRequest("POST", b.tokenURL.String(), strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.Header.Add("User-Agent", b.clientID)
|
||||
|
||||
//#nosec G704 inputs are user controlled or follow specification
|
||||
resp, err := b.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return b.validateResponse(resp)
|
||||
}
|
||||
|
||||
// scopeExists check if the scope already exists within the list of scopes
|
||||
func (b *bearerHandler) scopeExists(search string) bool {
|
||||
if search == "" {
|
||||
return true
|
||||
}
|
||||
searchRepo, searchActions, searchOk := parseScope(search)
|
||||
if !searchOk {
|
||||
return slices.Contains(b.scopes, search)
|
||||
}
|
||||
scopePrefix := "repository:" + searchRepo + ":"
|
||||
for _, scope := range b.scopes {
|
||||
if scope == search {
|
||||
return true
|
||||
}
|
||||
if !strings.HasPrefix(scope, scopePrefix) {
|
||||
continue
|
||||
}
|
||||
_, actions, ok := parseScope(scope)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, sa := range searchActions {
|
||||
if !slices.Contains(actions, sa) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// validateResponse extracts the returned token
|
||||
func (b *bearerHandler) validateResponse(resp *http.Response) error {
|
||||
if resp.StatusCode != 200 {
|
||||
return errs.ErrHTTPUnauthorized
|
||||
}
|
||||
|
||||
// decode response and if successful, update token
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
decoded := bearerToken{}
|
||||
if err := decoder.Decode(&decoded); err != nil {
|
||||
return err
|
||||
}
|
||||
b.token = decoded
|
||||
|
||||
if b.token.ExpiresIn < minTokenLife {
|
||||
b.token.ExpiresIn = minTokenLife
|
||||
}
|
||||
|
||||
// If token is already expired, it was sent with a zero value or
|
||||
// there may be a clock skew between the client and auth server.
|
||||
// Also handle cases of remote time in the future.
|
||||
// But if remote time is slightly in the past, leave as is so token
|
||||
// expires here before the server.
|
||||
if b.isExpired() || b.token.IssuedAt.After(time.Now()) {
|
||||
b.token.IssuedAt = time.Now().UTC()
|
||||
}
|
||||
|
||||
// AccessToken and Token should be the same and we use Token elsewhere
|
||||
if b.token.AccessToken != "" {
|
||||
b.token.Token = b.token.AccessToken
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// jwtHubHandler supports JWT auth type requests.
|
||||
type jwtHubHandler struct {
|
||||
client *http.Client
|
||||
clientID string
|
||||
realm string
|
||||
host string
|
||||
credsFn CredsFn
|
||||
jwt string
|
||||
}
|
||||
|
||||
type jwtHubPost struct {
|
||||
User string `json:"username"`
|
||||
Pass string `json:"password"` //#nosec G117 exported struct intentionally holds secrets
|
||||
}
|
||||
type jwtHubResp struct {
|
||||
Detail string `json:"detail"`
|
||||
Token string `json:"token"`
|
||||
RefreshToken string `json:"refresh_token"` //#nosec G117 exported struct intentionally holds secrets
|
||||
}
|
||||
|
||||
// NewJWTHubHandler creates a new JWTHandler for Docker Hub.
|
||||
func NewJWTHubHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
|
||||
// JWT handler is only tested against Hub, and the API is Hub specific
|
||||
if host == "hub.docker.com" {
|
||||
return &jwtHubHandler{
|
||||
client: client,
|
||||
clientID: clientID,
|
||||
host: host,
|
||||
credsFn: credsFn,
|
||||
realm: "https://hub.docker.com/v2/users/login",
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddScope is not valid for JWTHubHandler
|
||||
func (j *jwtHubHandler) AddScope(scope string) error {
|
||||
return errs.ErrNoNewChallenge
|
||||
}
|
||||
|
||||
// ProcessChallenge handles WWW-Authenticate header for JWT auth on Docker Hub
|
||||
func (j *jwtHubHandler) ProcessChallenge(c challenge) error {
|
||||
cred := j.credsFn(j.host)
|
||||
// use token if provided
|
||||
if cred.Token != "" {
|
||||
j.jwt = cred.Token
|
||||
return nil
|
||||
}
|
||||
|
||||
// send a login request to hub
|
||||
bodyBytes, err := json.Marshal(jwtHubPost{
|
||||
User: cred.User,
|
||||
Pass: cred.Password, //#nosec G117 field name follows spec
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", j.realm, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("User-Agent", j.clientID)
|
||||
|
||||
//#nosec G704 inputs are user controlled or follow specification requirements
|
||||
resp, err := j.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != 200 || resp.StatusCode >= 300 {
|
||||
return errs.ErrHTTPUnauthorized
|
||||
}
|
||||
|
||||
var bodyParsed jwtHubResp
|
||||
err = json.Unmarshal(body, &bodyParsed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
j.jwt = bodyParsed.Token
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateRequest for JWTHubHandler adds JWT header
|
||||
func (j *jwtHubHandler) UpdateRequest(req *http.Request) error {
|
||||
if len(j.jwt) > 0 {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("JWT %s", j.jwt))
|
||||
return nil
|
||||
}
|
||||
return errs.ErrHTTPUnauthorized
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrEmptyChallenge indicates an issue with the received challenge in the WWW-Authenticate header
|
||||
//
|
||||
// Deprecated: replace with [errs.ErrEmptyChallenge].
|
||||
//go:fix inline
|
||||
ErrEmptyChallenge = errs.ErrEmptyChallenge
|
||||
// ErrInvalidChallenge indicates an issue with the received challenge in the WWW-Authenticate header
|
||||
//
|
||||
// Deprecated: replace with [errs.ErrInvalidChallenge].
|
||||
//go:fix inline
|
||||
ErrInvalidChallenge = errs.ErrInvalidChallenge
|
||||
// ErrNoNewChallenge indicates a challenge update did not result in any change
|
||||
//
|
||||
// Deprecated: replace with [errs.ErrNoNewChallenge].
|
||||
//go:fix inline
|
||||
ErrNoNewChallenge = errs.ErrNoNewChallenge
|
||||
// ErrNotFound indicates no credentials found for basic auth
|
||||
//
|
||||
// Deprecated: replace with [errs.ErrNotFound].
|
||||
//go:fix inline
|
||||
ErrNotFound = errs.ErrNotFound
|
||||
// ErrNotImplemented returned when method has not been implemented yet
|
||||
//
|
||||
// Deprecated: replace with [errs.ErrNotImplemented].
|
||||
//go:fix inline
|
||||
ErrNotImplemented = errs.ErrNotImplemented
|
||||
// ErrParseFailure indicates the WWW-Authenticate header could not be parsed
|
||||
//
|
||||
// Deprecated: replace with [errs.ErrParseFailure].
|
||||
//go:fix inline
|
||||
ErrParseFailure = errs.ErrParsingFailed
|
||||
// ErrUnauthorized request was not authorized
|
||||
//
|
||||
// Deprecated: replace with [errs.ErrUnauthorized].
|
||||
//go:fix inline
|
||||
ErrUnauthorized = errs.ErrHTTPUnauthorized
|
||||
// ErrUnsupported indicates the request was unsupported
|
||||
//
|
||||
// Deprecated: replace with [errs.ErrUnsupported].
|
||||
//go:fix inline
|
||||
ErrUnsupported = errs.ErrUnsupported
|
||||
)
|
||||
+181
@@ -0,0 +1,181 @@
|
||||
//go:build go1.18
|
||||
|
||||
// Package cache is used to store values with limits.
|
||||
// Items are automatically pruned when too many entries are stored, or values become stale.
|
||||
package cache
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
)
|
||||
|
||||
type Cache[k comparable, v any] struct {
|
||||
mu sync.Mutex
|
||||
minAge time.Duration
|
||||
maxAge time.Duration
|
||||
minCount int
|
||||
maxCount int
|
||||
timer *time.Timer
|
||||
entries map[k]*Entry[v]
|
||||
}
|
||||
|
||||
type Entry[v any] struct {
|
||||
used time.Time
|
||||
value v
|
||||
}
|
||||
|
||||
type sortKeys[k comparable] struct {
|
||||
keys []k
|
||||
lessFn func(a, b k) bool
|
||||
}
|
||||
|
||||
type conf struct {
|
||||
minAge time.Duration
|
||||
maxCount int
|
||||
}
|
||||
|
||||
type cacheOpts func(*conf)
|
||||
|
||||
func WithAge(age time.Duration) cacheOpts {
|
||||
return func(c *conf) {
|
||||
c.minAge = age
|
||||
}
|
||||
}
|
||||
|
||||
func WithCount(count int) cacheOpts {
|
||||
return func(c *conf) {
|
||||
c.maxCount = count
|
||||
}
|
||||
}
|
||||
|
||||
func New[k comparable, v any](opts ...cacheOpts) Cache[k, v] {
|
||||
c := conf{}
|
||||
for _, opt := range opts {
|
||||
opt(&c)
|
||||
}
|
||||
maxAge := c.minAge + (c.minAge / 10)
|
||||
minCount := 0
|
||||
if c.maxCount > 0 {
|
||||
minCount = int(float64(c.maxCount) * 0.9)
|
||||
}
|
||||
return Cache[k, v]{
|
||||
minAge: c.minAge,
|
||||
maxAge: maxAge,
|
||||
minCount: minCount,
|
||||
maxCount: c.maxCount,
|
||||
entries: map[k]*Entry[v]{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache[k, v]) Delete(key k) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
delete(c.entries, key)
|
||||
if len(c.entries) == 0 && c.timer != nil {
|
||||
c.timer.Stop()
|
||||
c.timer = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache[k, v]) Set(key k, val v) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.entries[key] = &Entry[v]{
|
||||
used: time.Now(),
|
||||
value: val,
|
||||
}
|
||||
if len(c.entries) > c.maxCount {
|
||||
c.pruneLocked()
|
||||
} else if c.timer == nil {
|
||||
// prune resets the timer, so this is only needed if the prune wasn't triggered
|
||||
c.timer = time.AfterFunc(c.maxAge, c.prune)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache[k, v]) Get(key k) (v, error) {
|
||||
if c == nil {
|
||||
var val v
|
||||
return val, errs.ErrNotFound
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if e, ok := c.entries[key]; ok {
|
||||
if e.used.Add(c.minAge).Before(time.Now()) {
|
||||
// entry expired
|
||||
go c.prune()
|
||||
} else {
|
||||
c.entries[key].used = time.Now()
|
||||
return e.value, nil
|
||||
}
|
||||
}
|
||||
var val v
|
||||
return val, errs.ErrNotFound
|
||||
}
|
||||
|
||||
func (c *Cache[k, v]) prune() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.pruneLocked()
|
||||
}
|
||||
|
||||
func (c *Cache[k, v]) pruneLocked() {
|
||||
// sort key list by last used date
|
||||
keyList := make([]k, 0, len(c.entries))
|
||||
for key := range c.entries {
|
||||
keyList = append(keyList, key)
|
||||
}
|
||||
sk := sortKeys[k]{
|
||||
keys: keyList,
|
||||
lessFn: func(a, b k) bool {
|
||||
return c.entries[a].used.Before(c.entries[b].used)
|
||||
},
|
||||
}
|
||||
sort.Sort(&sk)
|
||||
// prune entries
|
||||
now := time.Now()
|
||||
cutoff := now.Add(c.minAge * -1)
|
||||
nextTime := now
|
||||
delCount := len(keyList) - c.minCount
|
||||
for i, key := range keyList {
|
||||
if i < delCount || c.entries[key].used.Before(cutoff) {
|
||||
delete(c.entries, key)
|
||||
} else {
|
||||
nextTime = c.entries[key].used
|
||||
break
|
||||
}
|
||||
}
|
||||
// set next timer
|
||||
if len(c.entries) > 0 {
|
||||
dur := nextTime.Sub(now) + c.maxAge
|
||||
if c.timer == nil {
|
||||
// this shouldn't be possible
|
||||
c.timer = time.AfterFunc(dur, c.prune)
|
||||
} else {
|
||||
c.timer.Reset(dur)
|
||||
}
|
||||
} else if c.timer != nil {
|
||||
c.timer.Stop()
|
||||
c.timer = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (sk *sortKeys[k]) Len() int {
|
||||
return len(sk.keys)
|
||||
}
|
||||
|
||||
func (sk *sortKeys[k]) Less(i, j int) bool {
|
||||
return sk.lessFn(sk.keys[i], sk.keys[j])
|
||||
}
|
||||
|
||||
func (sk *sortKeys[k]) Swap(i, j int) {
|
||||
sk.keys[i], sk.keys[j] = sk.keys[j], sk.keys[i]
|
||||
}
|
||||
+188
@@ -0,0 +1,188 @@
|
||||
// Package conffile wraps the read and write of configuration files
|
||||
package conffile
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
perms int
|
||||
fullname string
|
||||
}
|
||||
|
||||
type Opt func(*File)
|
||||
|
||||
// New returns a new File.
|
||||
// The last successful option determines the filename.
|
||||
func New(opts ...Opt) *File {
|
||||
f := File{perms: 0o600}
|
||||
for _, fn := range opts {
|
||||
fn(&f)
|
||||
}
|
||||
if f.fullname == "" {
|
||||
return nil
|
||||
}
|
||||
return &f
|
||||
}
|
||||
|
||||
// WithAppDir determines the filename from the XDG or Windows specification.
|
||||
// By default, this is based in $HOME/.config on Linux and %APPDATA% on Windows.
|
||||
// If the file does not exist, this will set the filename only if "force" is true.
|
||||
func WithAppDir(unixDir, winDir, name string, force bool) Opt {
|
||||
var dir string
|
||||
if winDir == "" {
|
||||
dir = unixDir
|
||||
} else {
|
||||
dir = osString(unixDir, winDir)
|
||||
}
|
||||
return func(f *File) {
|
||||
fullname := filepath.Join(appDir(), dir, name)
|
||||
if force || exists(fullname) {
|
||||
f.fullname = fullname
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithDirName determines the filename from a subdirectory in the user's HOME.
|
||||
//
|
||||
// Deprecated: Replace with [WithHomeDir]
|
||||
//
|
||||
//go:fix inline
|
||||
func WithDirName(dir, name string) Opt {
|
||||
return WithHomeDir(dir, name, true)
|
||||
}
|
||||
|
||||
// WithEnvFile sets the fullname to the environment value if defined.
|
||||
func WithEnvFile(envVar string) Opt {
|
||||
return func(f *File) {
|
||||
val := os.Getenv(envVar)
|
||||
if val != "" {
|
||||
f.fullname = val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithEnvDir sets the fullname to the environment value + filename if the environment variable is defined.
|
||||
func WithEnvDir(envVar, name string) Opt {
|
||||
return func(f *File) {
|
||||
val := os.Getenv(envVar)
|
||||
if val != "" {
|
||||
f.fullname = filepath.Join(val, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithFullname specifies the filename.
|
||||
// This will always set the filename even if the file does not exist.
|
||||
func WithFullname(fullname string) Opt {
|
||||
return func(f *File) {
|
||||
f.fullname = fullname
|
||||
}
|
||||
}
|
||||
|
||||
// WithHomeDir determines the filename from a subdirectory in the user's HOME
|
||||
// e.g. dir=".app", name="config.json", sets the fullname to "$HOME/.app/config.json".
|
||||
// If the file does not exist, this will set the filename only if "force" is true.
|
||||
func WithHomeDir(dir, name string, force bool) Opt {
|
||||
return func(f *File) {
|
||||
filename := filepath.Join(homeDir(), dir, name)
|
||||
if force || exists(filename) {
|
||||
f.fullname = filename
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithPerms specifies the permissions to create a file with (default 0600).
|
||||
func WithPerms(perms int) Opt {
|
||||
return func(f *File) {
|
||||
f.perms = perms
|
||||
}
|
||||
}
|
||||
|
||||
func (f *File) Name() string {
|
||||
return f.fullname
|
||||
}
|
||||
|
||||
func (f *File) Open() (io.ReadCloser, error) {
|
||||
return os.Open(f.fullname)
|
||||
}
|
||||
|
||||
func (f *File) Write(rdr io.Reader) error {
|
||||
// create temp file/open
|
||||
dir := filepath.Dir(f.fullname)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, filepath.Base(f.fullname))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpStat, err := tmp.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmpStat.Name()
|
||||
tmpFullname := filepath.Join(dir, tmpName)
|
||||
defer os.Remove(tmpFullname)
|
||||
|
||||
// copy from rdr to temp file
|
||||
_, err = io.Copy(tmp, rdr)
|
||||
errC := tmp.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write config: %w", err)
|
||||
}
|
||||
if errC != nil {
|
||||
return fmt.Errorf("failed to close config: %w", errC)
|
||||
}
|
||||
|
||||
// adjust file ownership/permissions
|
||||
mode := os.FileMode(0o600)
|
||||
uid := os.Getuid()
|
||||
gid := os.Getgid()
|
||||
// adjust defaults based on existing file if available
|
||||
stat, err := os.Stat(f.fullname)
|
||||
if err == nil {
|
||||
// adjust mode to existing file
|
||||
if stat.Mode().IsRegular() {
|
||||
mode = stat.Mode()
|
||||
}
|
||||
uid, gid, _ = getFileOwner(stat)
|
||||
} else if !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
// update mode and owner of temp file
|
||||
//#nosec G703 tempfile location is user controlled
|
||||
if err := os.Chmod(tmpFullname, mode); err != nil {
|
||||
return err
|
||||
}
|
||||
if uid > 0 && gid > 0 {
|
||||
//#nosec G703 tempfile location is user controlled
|
||||
_ = os.Chown(tmpFullname, uid, gid)
|
||||
}
|
||||
// move temp file to target filename
|
||||
//#nosec G703 tempfile location is user controlled
|
||||
return os.Rename(tmpFullname, f.fullname)
|
||||
}
|
||||
|
||||
func exists(name string) bool {
|
||||
_, err := os.Stat(name)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func homeDir() string {
|
||||
home := os.Getenv(homeEnv)
|
||||
if home == "" {
|
||||
u, err := user.Current()
|
||||
if err == nil {
|
||||
home = u.HomeDir
|
||||
}
|
||||
}
|
||||
return home
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
//go:build !windows
|
||||
|
||||
package conffile
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
const (
|
||||
appDirEnv = "XDG_CONFIG_HOME"
|
||||
homeEnv = "HOME"
|
||||
)
|
||||
|
||||
func appDir() string {
|
||||
appDir := os.Getenv(appDirEnv)
|
||||
if appDir == "" {
|
||||
home := homeDir()
|
||||
appDir = filepath.Join(home, ".config")
|
||||
}
|
||||
return appDir
|
||||
}
|
||||
|
||||
func getFileOwner(stat fs.FileInfo) (int, int, error) {
|
||||
var uid, gid int
|
||||
if sysstat, ok := stat.Sys().(*syscall.Stat_t); ok {
|
||||
uid = int(sysstat.Uid)
|
||||
gid = int(sysstat.Gid)
|
||||
}
|
||||
return uid, gid, nil
|
||||
}
|
||||
|
||||
func osString(unix, _ string) string {
|
||||
return unix
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
//go:build windows
|
||||
|
||||
package conffile
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const (
|
||||
appDirEnv = "APPDATA"
|
||||
homeEnv = "USERPROFILE"
|
||||
)
|
||||
|
||||
func appDir() string {
|
||||
appDir := os.Getenv(appDirEnv)
|
||||
if appDir == "" {
|
||||
home := homeDir()
|
||||
appDir = filepath.Join(home, "AppData")
|
||||
}
|
||||
return appDir
|
||||
}
|
||||
|
||||
func getFileOwner(_ fs.FileInfo) (int, int, error) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
func osString(_, win string) string {
|
||||
return win
|
||||
}
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
// Package httplink parses the Link header from HTTP responses according to RFC5988
|
||||
package httplink
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
)
|
||||
|
||||
type (
|
||||
Links []Link
|
||||
Link struct {
|
||||
URI string
|
||||
Param map[string]string
|
||||
}
|
||||
)
|
||||
|
||||
type charLU byte
|
||||
|
||||
var charLUs [256]charLU
|
||||
|
||||
const (
|
||||
isSpace charLU = 1 << iota
|
||||
isToken
|
||||
isAlphaNum
|
||||
)
|
||||
|
||||
func init() {
|
||||
for c := range 256 {
|
||||
charLUs[c] = 0
|
||||
if strings.ContainsRune(" \t\r\n", rune(c)) {
|
||||
charLUs[c] |= isSpace
|
||||
}
|
||||
if (rune('a') <= rune(c) && rune(c) <= rune('z')) || (rune('A') <= rune(c) && rune(c) <= rune('Z') || (rune('0') <= rune(c) && rune(c) <= rune('9'))) {
|
||||
charLUs[c] |= isAlphaNum | isToken
|
||||
}
|
||||
if strings.ContainsRune("!#$%&'()*+-./:<=>?@[]^_`{|}~", rune(c)) {
|
||||
charLUs[c] |= isToken
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse reads "Link" http headers into an array of Link structs.
|
||||
// Header array should be the output of resp.Header.Values("link").
|
||||
func Parse(headers []string) (Links, error) {
|
||||
links := []Link{}
|
||||
for _, h := range headers {
|
||||
state := "init"
|
||||
var ub, pnb, pvb []byte
|
||||
parms := map[string]string{}
|
||||
endLink := func() {
|
||||
links = append(links, Link{
|
||||
URI: string(ub),
|
||||
Param: parms,
|
||||
})
|
||||
// reset state
|
||||
ub, pnb, pvb = []byte{}, []byte{}, []byte{}
|
||||
parms = map[string]string{}
|
||||
}
|
||||
endParm := func() {
|
||||
if _, ok := parms[string(pnb)]; !ok {
|
||||
parms[string(pnb)] = string(pvb)
|
||||
}
|
||||
// reset parm
|
||||
pnb, pvb = []byte{}, []byte{}
|
||||
}
|
||||
for i, b := range []byte(h) {
|
||||
switch state {
|
||||
case "init":
|
||||
if b == '<' {
|
||||
state = "uriQuoted"
|
||||
} else if charLUs[b]&isToken != 0 {
|
||||
state = "uri"
|
||||
ub = append(ub, b)
|
||||
} else if charLUs[b]&isSpace != 0 || b == ',' {
|
||||
// noop
|
||||
} else {
|
||||
// unknown character
|
||||
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
|
||||
}
|
||||
case "uri":
|
||||
// parse tokens until space or comma
|
||||
if charLUs[b]&isToken != 0 {
|
||||
ub = append(ub, b)
|
||||
} else if charLUs[b]&isSpace != 0 {
|
||||
state = "fieldSep"
|
||||
} else if b == ';' {
|
||||
state = "parmName"
|
||||
} else if b == ',' {
|
||||
state = "init"
|
||||
endLink()
|
||||
} else {
|
||||
// unknown character
|
||||
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
|
||||
}
|
||||
case "uriQuoted":
|
||||
// parse tokens until quote
|
||||
if b == '>' {
|
||||
state = "fieldSep"
|
||||
} else {
|
||||
ub = append(ub, b)
|
||||
}
|
||||
case "fieldSep":
|
||||
if b == ';' {
|
||||
state = "parmName"
|
||||
} else if b == ',' {
|
||||
state = "init"
|
||||
endLink()
|
||||
} else if charLUs[b]&isSpace != 0 {
|
||||
// noop
|
||||
} else {
|
||||
// unknown character
|
||||
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
|
||||
}
|
||||
case "parmName":
|
||||
if len(pnb) > 0 && b == '=' {
|
||||
state = "parmValue"
|
||||
} else if len(pnb) > 0 && b == '*' {
|
||||
state = "parmNameStar"
|
||||
} else if charLUs[b]&isAlphaNum != 0 {
|
||||
pnb = append(pnb, b)
|
||||
} else if len(pnb) == 0 && charLUs[b]&isSpace != 0 {
|
||||
// noop
|
||||
} else {
|
||||
// unknown character
|
||||
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
|
||||
}
|
||||
case "parmNameStar":
|
||||
if b == '=' {
|
||||
state = "parmValue"
|
||||
} else {
|
||||
// unknown character
|
||||
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
|
||||
}
|
||||
case "parmValue":
|
||||
if len(pvb) == 0 {
|
||||
if charLUs[b]&isToken != 0 {
|
||||
pvb = append(pvb, b)
|
||||
} else if b == '"' {
|
||||
state = "parmValueQuoted"
|
||||
} else {
|
||||
// unknown character
|
||||
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
|
||||
}
|
||||
} else {
|
||||
if charLUs[b]&isToken != 0 {
|
||||
pvb = append(pvb, b)
|
||||
} else if charLUs[b]&isSpace != 0 {
|
||||
state = "fieldSep"
|
||||
endParm()
|
||||
} else if b == ';' {
|
||||
state = "parmName"
|
||||
endParm()
|
||||
} else if b == ',' {
|
||||
state = "init"
|
||||
endParm()
|
||||
endLink()
|
||||
} else {
|
||||
// unknown character
|
||||
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
|
||||
}
|
||||
}
|
||||
case "parmValueQuoted":
|
||||
if b == '"' {
|
||||
state = "fieldSep"
|
||||
endParm()
|
||||
} else {
|
||||
pvb = append(pvb, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
// check for valid state at end of header
|
||||
switch state {
|
||||
case "parmValue":
|
||||
endParm()
|
||||
endLink()
|
||||
case "uri", "fieldSep":
|
||||
endLink()
|
||||
case "init":
|
||||
// noop
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected end state %s for header %s: %w", state, h, errs.ErrParsingFailed)
|
||||
}
|
||||
}
|
||||
|
||||
return links, nil
|
||||
}
|
||||
|
||||
// Get returns a link with a specific parm value, e.g. rel="next"
|
||||
func (links Links) Get(parm, val string) (Link, error) {
|
||||
for _, link := range links {
|
||||
if link.Param != nil && link.Param[parm] == val {
|
||||
return link, nil
|
||||
}
|
||||
}
|
||||
return Link{}, errs.ErrNotFound
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
// Package limitread provides a reader that will error if the limit is ever exceeded
|
||||
package limitread
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
)
|
||||
|
||||
type LimitRead struct {
|
||||
Reader io.Reader
|
||||
Limit int64
|
||||
}
|
||||
|
||||
func (lr *LimitRead) Read(p []byte) (int, error) {
|
||||
if lr.Limit < 0 {
|
||||
return 0, fmt.Errorf("read limit exceeded%.0w", errs.ErrSizeLimitExceeded)
|
||||
}
|
||||
if int64(len(p)) > lr.Limit+1 {
|
||||
p = p[0 : lr.Limit+1]
|
||||
}
|
||||
n, err := lr.Reader.Read(p)
|
||||
lr.Limit -= int64(n)
|
||||
if lr.Limit < 0 {
|
||||
return n, fmt.Errorf("read limit exceeded%.0w", errs.ErrSizeLimitExceeded)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
// Package pqueue implements a priority queue.
|
||||
package pqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Queue[T any] struct {
|
||||
mu sync.Mutex
|
||||
max int
|
||||
next func(queued, active []*T) int
|
||||
active []*T
|
||||
queued []*T
|
||||
wait []*chan struct{}
|
||||
}
|
||||
|
||||
// Opts is used to configure a new priority queue.
|
||||
type Opts[T any] struct {
|
||||
Max int // maximum concurrent entries, defaults to 1.
|
||||
Next func(queued, active []*T) int // function to lookup index of next queued entry to release, defaults to oldest entry.
|
||||
}
|
||||
|
||||
// New creates a new priority queue.
|
||||
func New[T any](opts Opts[T]) *Queue[T] {
|
||||
if opts.Max <= 0 {
|
||||
opts.Max = 1
|
||||
}
|
||||
return &Queue[T]{
|
||||
max: opts.Max,
|
||||
next: opts.Next,
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire adds a new entry to the queue and returns once it is ready.
|
||||
// The returned function must be called when the queued job completes to release the next entry.
|
||||
// If there is any error, the returned function will be nil.
|
||||
func (q *Queue[T]) Acquire(ctx context.Context, e T) (func(), error) {
|
||||
if q == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
found, err := q.checkContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if found {
|
||||
return func() {}, nil
|
||||
}
|
||||
q.mu.Lock()
|
||||
if len(q.active)+len(q.queued) < q.max {
|
||||
q.active = append(q.active, &e)
|
||||
q.mu.Unlock()
|
||||
return q.releaseFn(&e), nil
|
||||
}
|
||||
// limit reached, add to queue and wait
|
||||
w := make(chan struct{}, 1)
|
||||
q.queued = append(q.queued, &e)
|
||||
q.wait = append(q.wait, &w)
|
||||
q.mu.Unlock()
|
||||
// wait on both context and queue
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// context abort, remove queued entry
|
||||
q.mu.Lock()
|
||||
if i := slices.Index(q.queued, &e); i >= 0 {
|
||||
q.queued = slices.Delete(q.queued, i, i+1)
|
||||
q.wait = slices.Delete(q.wait, i, i+1)
|
||||
q.mu.Unlock()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
q.mu.Unlock()
|
||||
// queued entry found, assume race condition with context and entry being released, release next entry
|
||||
q.release(&e)
|
||||
return nil, ctx.Err()
|
||||
case <-w:
|
||||
return q.releaseFn(&e), nil
|
||||
}
|
||||
}
|
||||
|
||||
// TryAcquire attempts to add an entry on to the list of active entries.
|
||||
// If the returned function is nil, the queue was not available.
|
||||
// If the returned function is not nil, it must be called when the job is complete to release the next entry.
|
||||
func (q *Queue[T]) TryAcquire(ctx context.Context, e T) (func(), error) {
|
||||
if q == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
found, err := q.checkContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if found {
|
||||
return func() {}, nil
|
||||
}
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if len(q.active)+len(q.queued) < q.max {
|
||||
q.active = append(q.active, &e)
|
||||
return q.releaseFn(&e), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// release next entry or noop.
|
||||
func (q *Queue[T]) release(prev *T) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
// remove prev entry from active list
|
||||
if i := slices.Index(q.active, prev); i >= 0 {
|
||||
q.active = slices.Delete(q.active, i, i+1)
|
||||
}
|
||||
// skip checks when at limit or nothing queued
|
||||
if len(q.queued) == 0 {
|
||||
if len(q.active) == 0 {
|
||||
// free up slices if this was the last active entry
|
||||
q.active = nil
|
||||
q.queued = nil
|
||||
q.wait = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(q.active) >= q.max {
|
||||
return
|
||||
}
|
||||
i := 0
|
||||
if q.next != nil && len(q.queued) > 1 {
|
||||
i = q.next(q.queued, q.active)
|
||||
// validate response
|
||||
i = max(min(i, len(q.queued)-1), 0)
|
||||
}
|
||||
// release queued entry, move to active list, and remove from queued/wait lists
|
||||
close(*q.wait[i])
|
||||
q.active = append(q.active, q.queued[i])
|
||||
q.queued = slices.Delete(q.queued, i, i+1)
|
||||
q.wait = slices.Delete(q.wait, i, i+1)
|
||||
}
|
||||
|
||||
// releaseFn is a convenience wrapper around [release].
|
||||
func (q *Queue[T]) releaseFn(prev *T) func() {
|
||||
return func() {
|
||||
q.release(prev)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: is there a way to make a different context key for each generic type?
|
||||
type ctxType int
|
||||
|
||||
var ctxKey ctxType
|
||||
|
||||
type valMulti[T any] struct {
|
||||
qList []*Queue[T]
|
||||
}
|
||||
|
||||
// AcquireMulti is used to simultaneously lock multiple queues without the risk of deadlock.
|
||||
// The returned context needs to be used on calls to [Acquire] or [TryAcquire] which will immediately succeed since the resource is already acquired.
|
||||
// Attempting to acquire other resources with [Acquire], [TryAcquire], or [AcquireMulti] using the returned context and will fail for being outside of the transaction.
|
||||
// The returned function must be called to release the resources.
|
||||
// The returned function is not thread safe, ensure no other simultaneous calls to [Acquire] or [TryAcquire] using the returned context have finished before it is called.
|
||||
func AcquireMulti[T any](ctx context.Context, e T, qList ...*Queue[T]) (context.Context, func(), error) {
|
||||
// verify context not already holding locks
|
||||
qCtx := ctx.Value(ctxKey)
|
||||
if qCtx != nil {
|
||||
if qCtxVal, ok := qCtx.(*valMulti[T]); !ok || qCtxVal.qList != nil {
|
||||
return ctx, nil, fmt.Errorf("context already used by another AcquireMulti request")
|
||||
}
|
||||
}
|
||||
// delete nil entries
|
||||
for i := len(qList) - 1; i >= 0; i-- {
|
||||
if qList[i] == nil {
|
||||
qList = slices.Delete(qList, i, i+1)
|
||||
}
|
||||
}
|
||||
// empty/nil list is a noop
|
||||
if len(qList) == 0 {
|
||||
return ctx, func() {}, nil
|
||||
}
|
||||
// dedup entries from the list
|
||||
for i := len(qList) - 2; i >= 0; i-- {
|
||||
for j := len(qList) - 1; j > i; j-- {
|
||||
if qList[i] == qList[j] {
|
||||
qList[j] = qList[len(qList)-1]
|
||||
qList = qList[:len(qList)-1]
|
||||
}
|
||||
}
|
||||
}
|
||||
// Loop through queues to acquire, waiting on the first, and attempting the remaining.
|
||||
// If any of the remaining entries cannot be immediately acquired, reset and make it the new queue to wait on.
|
||||
lockI := 0
|
||||
doneList := make([]func(), len(qList))
|
||||
for {
|
||||
acquired := true
|
||||
i := 0
|
||||
done, err := qList[lockI].Acquire(ctx, e)
|
||||
if err != nil {
|
||||
return ctx, nil, err
|
||||
}
|
||||
doneList[lockI] = done
|
||||
for i < len(qList) {
|
||||
if i != lockI {
|
||||
doneList[i], err = qList[i].TryAcquire(ctx, e)
|
||||
if doneList[i] == nil || err != nil {
|
||||
acquired = false
|
||||
break
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
if err == nil && acquired {
|
||||
break
|
||||
}
|
||||
// cleanup on failed attempt
|
||||
if lockI > i {
|
||||
doneList[lockI]()
|
||||
}
|
||||
// track blocking index for a retry
|
||||
lockI = i
|
||||
for i > 0 {
|
||||
i--
|
||||
doneList[i]()
|
||||
}
|
||||
// abort on errors
|
||||
if err != nil {
|
||||
return ctx, nil, err
|
||||
}
|
||||
}
|
||||
// success, update context
|
||||
ctxVal := valMulti[T]{qList: qList}
|
||||
newCtx := context.WithValue(ctx, ctxKey, &ctxVal)
|
||||
cleanup := func() {
|
||||
ctxVal.qList = nil
|
||||
// dequeue in reverse order to minimize chance of another AcquireMulti being freed and immediately blocking on the next queue
|
||||
for i := len(doneList) - 1; i >= 0; i-- {
|
||||
doneList[i]()
|
||||
}
|
||||
}
|
||||
return newCtx, cleanup, nil
|
||||
}
|
||||
|
||||
func (q *Queue[T]) checkContext(ctx context.Context) (bool, error) {
|
||||
qCtx := ctx.Value(ctxKey)
|
||||
if qCtx == nil {
|
||||
return false, nil
|
||||
}
|
||||
qCtxVal, ok := qCtx.(*valMulti[T])
|
||||
if !ok {
|
||||
return false, nil // another type is using the context, treat it as unset
|
||||
}
|
||||
if qCtxVal.qList == nil {
|
||||
return false, nil
|
||||
}
|
||||
if slices.Contains(qCtxVal.qList, q) {
|
||||
// instance already locked
|
||||
return true, nil
|
||||
}
|
||||
return true, fmt.Errorf("cannot acquire new locks during a transaction")
|
||||
}
|
||||
+987
@@ -0,0 +1,987 @@
|
||||
// Package reghttp is used for HTTP requests to a registry
|
||||
package reghttp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
// crypto libraries included for go-digest
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/regclient/regclient/config"
|
||||
"github.com/regclient/regclient/internal/auth"
|
||||
"github.com/regclient/regclient/internal/pqueue"
|
||||
"github.com/regclient/regclient/internal/regnet"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/types"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/warning"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultDelayInit, _ = time.ParseDuration("0.1s")
|
||||
defaultDelayMax, _ = time.ParseDuration("30s")
|
||||
warnRegexp = regexp.MustCompile(`^299\s+-\s+"([^"]+)"`)
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultRetryLimit = 5 // number of times a request will be retried
|
||||
backoffResetCount = 5 // number of successful requests needed to reduce the backoff
|
||||
)
|
||||
|
||||
// Client is an HTTP client wrapper.
|
||||
// It handles features like authentication, retries, backoff delays, TLS settings.
|
||||
type Client struct {
|
||||
httpClient *http.Client // upstream [http.Client], this is wrapped per repository for an auth handler on redirects
|
||||
getConfigHost func(string) *config.Host // call-back to get the [config.Host] for a specific registry
|
||||
host map[string]*clientHost // host specific settings, wrap access with a mutex lock
|
||||
rootCAPool [][]byte // list of root CAs for configuring the http.Client transport
|
||||
rootCADirs []string // list of directories for additional root CAs
|
||||
retryLimit int // number of retries before failing a request, this applies to each host, and each request
|
||||
delayInit time.Duration // how long to initially delay requests on a failure
|
||||
delayMax time.Duration // maximum time to delay a request
|
||||
slog *slog.Logger // logging for tracing and failures
|
||||
userAgent string // user agent to specify in http request headers
|
||||
mu sync.Mutex // mutex to prevent data races
|
||||
}
|
||||
|
||||
type clientHost struct {
|
||||
config *config.Host // config entry
|
||||
httpClient *http.Client // modified http client for registry specific settings
|
||||
userAgent string // user agent to specify in http request headers
|
||||
slog *slog.Logger // logging for tracing and failures
|
||||
auth map[string]*auth.Auth // map of auth handlers by repository
|
||||
backoffCur int // current count of backoffs for this host
|
||||
backoffLast time.Time // time the last request was released, this may be in the future if there is a queue, or zero if no delay is needed
|
||||
backoffReset int // count of successful requests when a backoff is experienced, once [backoffResetCount] is reached, [backoffCur] is reduced by one and this is reset to 0
|
||||
reqFreq time.Duration // how long between submitting requests for this host
|
||||
reqNext time.Time // time to release the next request
|
||||
throttle *pqueue.Queue[reqmeta.Data] // limit concurrent requests to the host
|
||||
mu sync.Mutex // mutex to prevent data races
|
||||
}
|
||||
|
||||
// Req is a request to send to a registry.
|
||||
type Req struct {
|
||||
MetaKind reqmeta.Kind // kind of request for the priority queue
|
||||
Host string // registry name, hostname and mirrors will be looked up from host configuration
|
||||
Method string // http method to call
|
||||
DirectURL *url.URL // url to query, overrides repository, path, and query
|
||||
Repository string // repository to scope the request
|
||||
Path string // path of the request within a repository
|
||||
Query url.Values // url query parameters
|
||||
BodyLen int64 // length of body to send
|
||||
BodyBytes []byte // bytes of the body, overridden by BodyFunc
|
||||
BodyFunc func() (io.ReadCloser, error) // function to return a new body
|
||||
Headers http.Header // headers to send in the request
|
||||
NoPrefix bool // do not include the repository prefix
|
||||
NoMirrors bool // do not send request to a mirror
|
||||
ExpectLen int64 // expected size of the returned body
|
||||
TransactLen int64 // size of an overall transaction for the priority queue
|
||||
IgnoreErr bool // ignore http errors and do not trigger backoffs
|
||||
}
|
||||
|
||||
// Resp is used to handle the result of a request.
|
||||
type Resp struct {
|
||||
ctx context.Context
|
||||
client *Client
|
||||
req *Req
|
||||
resp *http.Response
|
||||
mirror string
|
||||
done bool
|
||||
reader io.Reader
|
||||
readCur, readMax int64
|
||||
retryCount int
|
||||
throttleDone func()
|
||||
}
|
||||
|
||||
// Opts is used to configure client options.
|
||||
type Opts func(*Client)
|
||||
|
||||
// NewClient returns a client for handling requests.
|
||||
func NewClient(opts ...Opts) *Client {
|
||||
c := Client{
|
||||
httpClient: &http.Client{},
|
||||
host: map[string]*clientHost{},
|
||||
retryLimit: DefaultRetryLimit,
|
||||
delayInit: defaultDelayInit,
|
||||
delayMax: defaultDelayMax,
|
||||
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
|
||||
rootCAPool: [][]byte{},
|
||||
rootCADirs: []string{},
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(&c)
|
||||
}
|
||||
return &c
|
||||
}
|
||||
|
||||
// WithCerts adds certificates.
|
||||
func WithCerts(certs [][]byte) Opts {
|
||||
return func(c *Client) {
|
||||
c.rootCAPool = append(c.rootCAPool, certs...)
|
||||
}
|
||||
}
|
||||
|
||||
// WithCertDirs adds directories to check for host specific certs.
|
||||
func WithCertDirs(dirs []string) Opts {
|
||||
return func(c *Client) {
|
||||
c.rootCADirs = append(c.rootCADirs, dirs...)
|
||||
}
|
||||
}
|
||||
|
||||
// WithCertFiles adds certificates by filename.
|
||||
func WithCertFiles(files []string) Opts {
|
||||
return func(c *Client) {
|
||||
for _, f := range files {
|
||||
//#nosec G304 command is run by a user accessing their own files
|
||||
cert, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
c.slog.Warn("Failed to read certificate",
|
||||
slog.String("err", err.Error()),
|
||||
slog.String("file", f))
|
||||
} else {
|
||||
c.rootCAPool = append(c.rootCAPool, cert)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithConfigHostFn adds the callback to request a [config.Host] struct.
|
||||
// The function must normalize the hostname for Docker Hub support.
|
||||
func WithConfigHostFn(gch func(string) *config.Host) Opts {
|
||||
return func(c *Client) {
|
||||
c.getConfigHost = gch
|
||||
}
|
||||
}
|
||||
|
||||
// WithDelay initial time to wait between retries (increased with exponential backoff).
|
||||
func WithDelay(delayInit time.Duration, delayMax time.Duration) Opts {
|
||||
return func(c *Client) {
|
||||
if delayInit > 0 {
|
||||
c.delayInit = delayInit
|
||||
}
|
||||
// delayMax must be at least delayInit, if 0 initialize to 30x delayInit
|
||||
if delayMax > c.delayInit {
|
||||
c.delayMax = delayMax
|
||||
} else if delayMax > 0 {
|
||||
c.delayMax = c.delayInit
|
||||
} else {
|
||||
c.delayMax = c.delayInit * 30
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithHTTPClient uses a specific http client with retryable requests.
|
||||
func WithHTTPClient(hc *http.Client) Opts {
|
||||
return func(c *Client) {
|
||||
c.httpClient = hc
|
||||
}
|
||||
}
|
||||
|
||||
// WithRetryLimit restricts the number of retries (defaults to 5).
|
||||
func WithRetryLimit(rl int) Opts {
|
||||
return func(c *Client) {
|
||||
if rl > 0 {
|
||||
c.retryLimit = rl
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithLog injects a slog Logger configuration.
|
||||
func WithLog(slog *slog.Logger) Opts {
|
||||
return func(c *Client) {
|
||||
c.slog = slog
|
||||
}
|
||||
}
|
||||
|
||||
// WithTransport uses a specific http transport with retryable requests.
|
||||
func WithTransport(t *http.Transport) Opts {
|
||||
return func(c *Client) {
|
||||
c.httpClient = &http.Client{Transport: t}
|
||||
}
|
||||
}
|
||||
|
||||
// WithUserAgent sets a user agent header.
|
||||
func WithUserAgent(ua string) Opts {
|
||||
return func(c *Client) {
|
||||
c.userAgent = ua
|
||||
}
|
||||
}
|
||||
|
||||
// Do runs a request, returning the response result.
|
||||
func (c *Client) Do(ctx context.Context, req *Req) (*Resp, error) {
|
||||
resp := &Resp{
|
||||
ctx: ctx,
|
||||
client: c,
|
||||
req: req,
|
||||
readCur: 0,
|
||||
readMax: req.ExpectLen,
|
||||
}
|
||||
err := resp.next()
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// next sends requests until a mirror responds or all requests fail.
|
||||
func (resp *Resp) next() error {
|
||||
var err error
|
||||
c := resp.client
|
||||
req := resp.req
|
||||
// lookup reqHost entry
|
||||
reqHost := c.getHost(req.Host)
|
||||
// create sorted list of mirrors, based on backoffs, upstream, and priority
|
||||
hosts := make([]*clientHost, 0, 1+len(reqHost.config.Mirrors))
|
||||
if !req.NoMirrors {
|
||||
for _, m := range reqHost.config.Mirrors {
|
||||
hosts = append(hosts, c.getHost(m))
|
||||
}
|
||||
}
|
||||
hosts = append(hosts, reqHost)
|
||||
sort.Slice(hosts, sortHostsCmp(hosts, reqHost.config.Name))
|
||||
// loop over requests to mirrors and retries
|
||||
curHost := 0
|
||||
for {
|
||||
backoff := false
|
||||
dropHost := false
|
||||
retryHost := false
|
||||
if len(hosts) == 0 {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return errs.ErrAllRequestsFailed
|
||||
}
|
||||
if curHost >= len(hosts) {
|
||||
curHost = 0
|
||||
}
|
||||
h := hosts[curHost]
|
||||
resp.mirror = h.config.Name
|
||||
// there is an intentional extra retry in this check to allow for auth requests
|
||||
if resp.retryCount > c.retryLimit {
|
||||
return errs.ErrRetryLimitExceeded
|
||||
}
|
||||
resp.retryCount++
|
||||
|
||||
// check that context isn't canceled/done
|
||||
ctxErr := resp.ctx.Err()
|
||||
if ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
// wait for other concurrent requests to this host
|
||||
throttleDone, throttleErr := h.throttle.Acquire(resp.ctx, reqmeta.Data{
|
||||
Kind: req.MetaKind,
|
||||
Size: req.BodyLen + req.ExpectLen + req.TransactLen,
|
||||
})
|
||||
if throttleErr != nil {
|
||||
return throttleErr
|
||||
}
|
||||
|
||||
// try each host in a closure to handle all the backoff/dropHost from one place
|
||||
loopErr := func() error {
|
||||
var err error
|
||||
if req.Method == "HEAD" && h.config.APIOpts != nil {
|
||||
var disableHead bool
|
||||
disableHead, err = strconv.ParseBool(h.config.APIOpts["disableHead"])
|
||||
if err == nil && disableHead {
|
||||
dropHost = true
|
||||
return fmt.Errorf("head requests disabled for host \"%s\": %w", h.config.Name, errs.ErrUnsupportedAPI)
|
||||
}
|
||||
}
|
||||
|
||||
// build the url
|
||||
var u url.URL
|
||||
if req.DirectURL != nil {
|
||||
u = *req.DirectURL
|
||||
} else {
|
||||
u = url.URL{
|
||||
Host: h.config.Hostname,
|
||||
Scheme: "https",
|
||||
}
|
||||
path := strings.Builder{}
|
||||
path.WriteString("/v2")
|
||||
if h.config.PathPrefix != "" && !req.NoPrefix {
|
||||
path.WriteString("/" + h.config.PathPrefix)
|
||||
}
|
||||
if req.Repository != "" {
|
||||
path.WriteString("/" + req.Repository)
|
||||
}
|
||||
path.WriteString("/" + req.Path)
|
||||
u.Path = path.String()
|
||||
if h.config.TLS == config.TLSDisabled {
|
||||
u.Scheme = "http"
|
||||
}
|
||||
query := url.Values{}
|
||||
if req.Query != nil {
|
||||
query = req.Query
|
||||
}
|
||||
if h.config.Hostname != reqHost.config.Hostname {
|
||||
query.Set("ns", reqHost.config.Hostname)
|
||||
}
|
||||
u.RawQuery = query.Encode()
|
||||
}
|
||||
// close previous response
|
||||
if resp.resp != nil && resp.resp.Body != nil {
|
||||
_ = resp.resp.Body.Close()
|
||||
}
|
||||
// delay for backoff if needed
|
||||
bu := resp.backoffGet()
|
||||
if !bu.IsZero() && bu.After(time.Now()) {
|
||||
sleepTime := time.Until(bu)
|
||||
c.slog.Debug("Sleeping for backoff",
|
||||
slog.String("Host", h.config.Name),
|
||||
slog.Duration("Duration", sleepTime))
|
||||
select {
|
||||
case <-resp.ctx.Done():
|
||||
return errs.ErrCanceled
|
||||
case <-time.After(sleepTime):
|
||||
}
|
||||
}
|
||||
var httpReq *http.Request
|
||||
httpReq, err = http.NewRequestWithContext(resp.ctx, req.Method, u.String(), nil)
|
||||
if err != nil {
|
||||
dropHost = true
|
||||
return err
|
||||
}
|
||||
if req.BodyFunc != nil {
|
||||
body, err := req.BodyFunc()
|
||||
if err != nil {
|
||||
dropHost = true
|
||||
return err
|
||||
}
|
||||
httpReq.Body = body
|
||||
httpReq.GetBody = req.BodyFunc
|
||||
httpReq.ContentLength = req.BodyLen
|
||||
} else if len(req.BodyBytes) > 0 {
|
||||
body := io.NopCloser(bytes.NewReader(req.BodyBytes))
|
||||
httpReq.Body = body
|
||||
httpReq.GetBody = func() (io.ReadCloser, error) { return body, nil }
|
||||
httpReq.ContentLength = req.BodyLen
|
||||
}
|
||||
if len(req.Headers) > 0 {
|
||||
httpReq.Header = req.Headers.Clone()
|
||||
}
|
||||
if c.userAgent != "" && httpReq.Header.Get("User-Agent") == "" {
|
||||
httpReq.Header.Add("User-Agent", c.userAgent)
|
||||
}
|
||||
if resp.readCur > 0 && resp.readMax > 0 {
|
||||
if req.Headers.Get("Range") == "" {
|
||||
httpReq.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", resp.readCur, resp.readMax))
|
||||
} else {
|
||||
// TODO: support Seek within a range request
|
||||
dropHost = true
|
||||
return fmt.Errorf("unable to resume a connection within a range request")
|
||||
}
|
||||
}
|
||||
|
||||
hAuth := h.getAuth(req.Repository)
|
||||
if hAuth != nil {
|
||||
// include docker generated scope to emulate docker clients
|
||||
if req.Repository != "" {
|
||||
scope := "repository:" + req.Repository + ":pull"
|
||||
if req.Method != "HEAD" && req.Method != "GET" {
|
||||
scope = scope + ",push"
|
||||
}
|
||||
_ = hAuth.AddScope(h.config.Hostname, scope)
|
||||
}
|
||||
// add auth headers
|
||||
err = hAuth.UpdateRequest(httpReq)
|
||||
// abort on auth errors, but only after the first request has been attempted
|
||||
if err != nil && resp.resp != nil {
|
||||
if errors.Is(err, errs.ErrHTTPUnauthorized) {
|
||||
dropHost = true
|
||||
} else {
|
||||
backoff = true
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// delay for the rate limit
|
||||
if h.reqFreq > 0 {
|
||||
sleep := time.Duration(0)
|
||||
h.mu.Lock()
|
||||
if time.Now().Before(h.reqNext) {
|
||||
sleep = time.Until(h.reqNext)
|
||||
h.reqNext = h.reqNext.Add(h.reqFreq)
|
||||
} else {
|
||||
h.reqNext = time.Now().Add(h.reqFreq)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if sleep > 0 {
|
||||
time.Sleep(sleep)
|
||||
}
|
||||
}
|
||||
|
||||
// send request
|
||||
hc := h.getHTTPClient(req.Repository)
|
||||
//#nosec G704 inputs are user controlled and sanitized
|
||||
resp.resp, err = hc.Do(httpReq)
|
||||
if err != nil {
|
||||
c.slog.Debug("Request failed",
|
||||
slog.String("URL", u.String()),
|
||||
slog.String("err", err.Error()))
|
||||
backoff = true
|
||||
return err
|
||||
}
|
||||
|
||||
statusCode := resp.resp.StatusCode
|
||||
if statusCode < 200 || statusCode >= 300 {
|
||||
switch statusCode {
|
||||
case http.StatusUnauthorized:
|
||||
// if auth can be done, retry same host without delay, otherwise drop/backoff
|
||||
if hAuth != nil {
|
||||
err = hAuth.HandleResponse(resp.resp)
|
||||
} else {
|
||||
err = fmt.Errorf("authentication handler unavailable")
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, errs.ErrEmptyChallenge) || errors.Is(err, errs.ErrNoNewChallenge) || errors.Is(err, errs.ErrHTTPUnauthorized) {
|
||||
c.slog.Debug("Failed to handle auth request",
|
||||
slog.String("URL", u.String()),
|
||||
slog.String("Err", err.Error()))
|
||||
} else {
|
||||
c.slog.Warn("Failed to handle auth request",
|
||||
slog.String("URL", u.String()),
|
||||
slog.String("Err", err.Error()))
|
||||
}
|
||||
dropHost = true
|
||||
} else {
|
||||
err = fmt.Errorf("authentication required")
|
||||
retryHost = true
|
||||
}
|
||||
return err
|
||||
case http.StatusNotFound:
|
||||
// if not found, drop mirror for this req, but other requests don't need backoff
|
||||
dropHost = true
|
||||
case http.StatusRequestedRangeNotSatisfiable:
|
||||
// if range request error (blob push), drop mirror for this req, but other requests don't need backoff
|
||||
dropHost = true
|
||||
case http.StatusTooManyRequests, http.StatusRequestTimeout, http.StatusGatewayTimeout, http.StatusBadGateway, http.StatusInternalServerError:
|
||||
// server is likely overloaded, backoff but still retry
|
||||
backoff = true
|
||||
default:
|
||||
// all other errors indicate a bigger issue, don't retry and set backoff
|
||||
backoff = true
|
||||
dropHost = true
|
||||
}
|
||||
errHTTP := HTTPError(resp.resp.StatusCode)
|
||||
errBody, _ := io.ReadAll(resp.resp.Body)
|
||||
_ = resp.resp.Body.Close()
|
||||
return fmt.Errorf("request failed: %w: %s", errHTTP, errBody)
|
||||
}
|
||||
|
||||
resp.reader = resp.resp.Body
|
||||
resp.done = false
|
||||
// set variables from headers if found
|
||||
clHeader := resp.resp.Header.Get("Content-Length")
|
||||
if resp.readCur == 0 && clHeader != "" {
|
||||
cl, parseErr := strconv.ParseInt(clHeader, 10, 64)
|
||||
if parseErr != nil {
|
||||
c.slog.Debug("failed to parse content-length header",
|
||||
slog.String("err", parseErr.Error()),
|
||||
slog.String("header", clHeader))
|
||||
} else if resp.readMax > 0 {
|
||||
if resp.readMax != cl {
|
||||
return fmt.Errorf("unexpected content-length, expected %d, received %d", resp.readMax, cl)
|
||||
}
|
||||
} else {
|
||||
resp.readMax = cl
|
||||
}
|
||||
}
|
||||
// verify Content-Range header when range request used, fail if missing
|
||||
if httpReq.Header.Get("Range") != "" && resp.resp.Header.Get("Content-Range") == "" {
|
||||
dropHost = true
|
||||
_ = resp.resp.Body.Close()
|
||||
return fmt.Errorf("range request not supported by server")
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
// return on success
|
||||
if loopErr == nil {
|
||||
resp.throttleDone = throttleDone
|
||||
return nil
|
||||
}
|
||||
// backoff, dropHost, and/or go to next host in the list
|
||||
if backoff {
|
||||
if req.IgnoreErr {
|
||||
// don't set a backoff, immediately drop the host when errors ignored
|
||||
dropHost = true
|
||||
} else {
|
||||
boErr := resp.backoffSet()
|
||||
if boErr != nil {
|
||||
// reached backoff limit
|
||||
dropHost = true
|
||||
}
|
||||
}
|
||||
}
|
||||
throttleDone()
|
||||
// when error does not allow retries, abort with the last known err value
|
||||
if err != nil && errors.Is(loopErr, errs.ErrNotRetryable) {
|
||||
return err
|
||||
}
|
||||
err = loopErr
|
||||
if dropHost {
|
||||
hosts = slices.Delete(hosts, curHost, curHost+1)
|
||||
} else if !retryHost {
|
||||
curHost++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetThrottle returns the current [pqueue.Queue] for a host used to throttle connections.
|
||||
// This can be used to acquire multiple throttles before performing a request across multiple hosts.
|
||||
func (c *Client) GetThrottle(host string) *pqueue.Queue[reqmeta.Data] {
|
||||
ch := c.getHost(host)
|
||||
return ch.throttle
|
||||
}
|
||||
|
||||
// HTTPResponse returns the [http.Response] from the last request.
|
||||
func (resp *Resp) HTTPResponse() *http.Response {
|
||||
return resp.resp
|
||||
}
|
||||
|
||||
// Read provides a retryable read from the body of the response.
|
||||
func (resp *Resp) Read(b []byte) (int, error) {
|
||||
if resp.done || resp.reader == nil {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if resp.resp == nil {
|
||||
return 0, errs.ErrNotFound
|
||||
}
|
||||
// perform the read
|
||||
i, err := resp.reader.Read(b)
|
||||
resp.readCur += int64(i)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
if resp.resp.Request.Method == "HEAD" || resp.readCur >= resp.readMax {
|
||||
resp.backoffReset()
|
||||
resp.done = true
|
||||
} else {
|
||||
// short read, retry?
|
||||
resp.client.slog.Debug("EOF before reading all content, retrying",
|
||||
slog.Int64("curRead", resp.readCur),
|
||||
slog.Int64("contentLen", resp.readMax))
|
||||
// retry
|
||||
respErr := resp.backoffSet()
|
||||
if respErr == nil {
|
||||
respErr = resp.next()
|
||||
}
|
||||
// unrecoverable EOF
|
||||
if respErr != nil {
|
||||
resp.client.slog.Warn("Failed to recover from short read",
|
||||
slog.String("err", respErr.Error()))
|
||||
resp.done = true
|
||||
return i, err
|
||||
}
|
||||
// retry successful, no EOF
|
||||
return i, nil
|
||||
}
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
return i, nil
|
||||
}
|
||||
return i, err
|
||||
}
|
||||
|
||||
// Close frees up resources from the request.
|
||||
func (resp *Resp) Close() error {
|
||||
if resp.throttleDone != nil {
|
||||
resp.throttleDone()
|
||||
resp.throttleDone = nil
|
||||
}
|
||||
if resp.resp == nil {
|
||||
return errs.ErrNotFound
|
||||
}
|
||||
if !resp.done {
|
||||
resp.backoffReset()
|
||||
}
|
||||
resp.done = true
|
||||
return resp.resp.Body.Close()
|
||||
}
|
||||
|
||||
// Seek provides a limited ability seek within the request response.
|
||||
func (resp *Resp) Seek(offset int64, whence int) (int64, error) {
|
||||
newOffset := resp.readCur
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
newOffset = offset
|
||||
case io.SeekCurrent:
|
||||
newOffset += offset
|
||||
case io.SeekEnd:
|
||||
if resp.readMax <= 0 {
|
||||
return resp.readCur, fmt.Errorf("seek from end is not supported")
|
||||
} else if resp.readMax+offset < 0 {
|
||||
return resp.readCur, fmt.Errorf("seek past beginning of the file is not supported")
|
||||
}
|
||||
newOffset = resp.readMax + offset
|
||||
default:
|
||||
return resp.readCur, fmt.Errorf("unknown value of whence: %d", whence)
|
||||
}
|
||||
if newOffset != resp.readCur {
|
||||
resp.readCur = newOffset
|
||||
// rerun the request to restart
|
||||
resp.retryCount-- // do not count a seek as a retry
|
||||
err := resp.next()
|
||||
if err != nil {
|
||||
return resp.readCur, err
|
||||
}
|
||||
}
|
||||
return resp.readCur, nil
|
||||
}
|
||||
|
||||
func (resp *Resp) backoffGet() time.Time {
|
||||
c := resp.client
|
||||
ch := c.getHost(resp.mirror)
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
if ch.backoffCur > 0 {
|
||||
delay := c.delayInit << ch.backoffCur
|
||||
delay = min(delay, c.delayMax)
|
||||
next := ch.backoffLast.Add(delay)
|
||||
now := time.Now()
|
||||
if now.After(next) {
|
||||
next = now
|
||||
}
|
||||
ch.backoffLast = next
|
||||
return next
|
||||
}
|
||||
// reset a stale "retry-after" time
|
||||
if !ch.backoffLast.IsZero() && ch.backoffLast.Before(time.Now()) {
|
||||
ch.backoffLast = time.Time{}
|
||||
}
|
||||
return ch.backoffLast
|
||||
}
|
||||
|
||||
func (resp *Resp) backoffSet() error {
|
||||
c := resp.client
|
||||
ch := c.getHost(resp.mirror)
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
// check rate limit header and use that directly if possible
|
||||
if resp.resp != nil && resp.resp.Header.Get("Retry-After") != "" {
|
||||
ras := resp.resp.Header.Get("Retry-After")
|
||||
ra, _ := time.ParseDuration(ras + "s")
|
||||
if ra > 0 {
|
||||
next := time.Now().Add(ra)
|
||||
if ch.backoffLast.Before(next) {
|
||||
ch.backoffLast = next
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// Else track the number of backoffs and fail when the limit is exceeded.
|
||||
// New requests always get at least one try, but fail fast if the server has been throwing errors.
|
||||
ch.backoffCur++
|
||||
if ch.backoffLast.IsZero() {
|
||||
ch.backoffLast = time.Now()
|
||||
}
|
||||
if ch.backoffCur >= c.retryLimit {
|
||||
return fmt.Errorf("%w: backoffs %d", errs.ErrBackoffLimit, ch.backoffCur)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (resp *Resp) backoffReset() {
|
||||
c := resp.client
|
||||
ch := c.getHost(resp.mirror)
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
if ch.backoffCur > 0 {
|
||||
ch.backoffReset++
|
||||
// If enough successful requests are seen, lower the backoffCur count.
|
||||
// This requires multiple successful requests of a flaky server, but quickly drops when above the retry limit.
|
||||
if ch.backoffReset > backoffResetCount || ch.backoffCur > c.retryLimit {
|
||||
ch.backoffReset = 0
|
||||
ch.backoffCur--
|
||||
if ch.backoffCur == 0 {
|
||||
// reset the last time to the zero value
|
||||
ch.backoffLast = time.Time{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getHost looks up or creates a clientHost for a given registry.
|
||||
func (c *Client) getHost(host string) *clientHost {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if h, ok := c.host[host]; ok {
|
||||
return h
|
||||
}
|
||||
var conf *config.Host
|
||||
if c.getConfigHost != nil {
|
||||
conf = c.getConfigHost(host)
|
||||
} else {
|
||||
conf = config.HostNewName(host)
|
||||
}
|
||||
if conf.Name != host {
|
||||
if h, ok := c.host[conf.Name]; ok {
|
||||
return h
|
||||
}
|
||||
}
|
||||
h := &clientHost{
|
||||
config: conf,
|
||||
userAgent: c.userAgent,
|
||||
slog: c.slog,
|
||||
auth: map[string]*auth.Auth{},
|
||||
}
|
||||
if h.config.ReqPerSec > 0 {
|
||||
h.reqFreq = time.Duration(float64(time.Second) / h.config.ReqPerSec)
|
||||
}
|
||||
if h.config.ReqConcurrent > 0 {
|
||||
h.throttle = pqueue.New(pqueue.Opts[reqmeta.Data]{Max: int(h.config.ReqConcurrent), Next: reqmeta.DataNext})
|
||||
}
|
||||
// copy the http client and configure registry specific settings
|
||||
hc := *c.httpClient
|
||||
h.httpClient = &hc
|
||||
if h.httpClient.Transport == nil {
|
||||
h.httpClient.Transport = http.DefaultTransport.(*http.Transport).Clone()
|
||||
}
|
||||
// configure transport for insecure requests and root certs
|
||||
if h.config.TLS == config.TLSInsecure || len(c.rootCAPool) > 0 || len(c.rootCADirs) > 0 || h.config.RegCert != "" || (h.config.ClientCert != "" && h.config.ClientKey != "") {
|
||||
t, ok := h.httpClient.Transport.(*http.Transport)
|
||||
if ok {
|
||||
var tlsc *tls.Config
|
||||
if t.TLSClientConfig != nil {
|
||||
tlsc = t.TLSClientConfig.Clone()
|
||||
} else {
|
||||
//#nosec G402 the default TLS 1.2 minimum version is allowed to support older registries
|
||||
tlsc = &tls.Config{}
|
||||
}
|
||||
if h.config.TLS == config.TLSInsecure {
|
||||
tlsc.InsecureSkipVerify = true
|
||||
} else {
|
||||
rootPool, err := makeRootPool(c.rootCAPool, c.rootCADirs, h.config.Hostname, h.config.RegCert)
|
||||
if err != nil {
|
||||
c.slog.Warn("failed to setup CA pool",
|
||||
slog.String("err", err.Error()))
|
||||
} else {
|
||||
tlsc.RootCAs = rootPool
|
||||
}
|
||||
}
|
||||
if h.config.ClientCert != "" && h.config.ClientKey != "" {
|
||||
cert, err := tls.X509KeyPair([]byte(h.config.ClientCert), []byte(h.config.ClientKey))
|
||||
if err != nil {
|
||||
c.slog.Warn("failed to configure client certs",
|
||||
slog.String("err", err.Error()))
|
||||
} else {
|
||||
tlsc.Certificates = []tls.Certificate{cert}
|
||||
}
|
||||
}
|
||||
t.TLSClientConfig = tlsc
|
||||
h.httpClient.Transport = t
|
||||
}
|
||||
}
|
||||
// wrap the transport for logging and to handle warning headers
|
||||
h.httpClient.Transport = &wrapTransport{c: c, orig: h.httpClient.Transport}
|
||||
|
||||
c.host[conf.Name] = h
|
||||
if conf.Name != host {
|
||||
// save another reference for faster lookups
|
||||
c.host[host] = h
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// getHTTPClient returns a client specific to the repo being queried.
|
||||
// Repository specific authentication needs a dedicated CheckRedirect handler.
|
||||
func (ch *clientHost) getHTTPClient(repo string) *http.Client {
|
||||
hc := *ch.httpClient
|
||||
hc.CheckRedirect = ch.checkRedirect(repo, hc.CheckRedirect)
|
||||
return &hc
|
||||
}
|
||||
|
||||
// checkRedirect wraps http.CheckRedirect to inject auth headers to specific hosts in the redirect chain
|
||||
func (ch *clientHost) checkRedirect(repo string, orig func(req *http.Request, via []*http.Request) error) func(req *http.Request, via []*http.Request) error {
|
||||
return func(req *http.Request, via []*http.Request) error {
|
||||
// fail on too many redirects
|
||||
if len(via) >= 10 {
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
// verify redirect is allowed
|
||||
last := via[len(via)-1]
|
||||
if err := regnet.AllowRedirect(*last.URL, *req.URL); err != nil {
|
||||
return err
|
||||
}
|
||||
// add auth headers if appropriate for the target host
|
||||
hAuth := ch.getAuth(repo)
|
||||
err := hAuth.UpdateRequest(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// wrap original redirect check
|
||||
if orig != nil {
|
||||
return orig(req, via)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// getAuth returns an auth, which may be repository specific.
|
||||
func (ch *clientHost) getAuth(repo string) *auth.Auth {
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
if !ch.config.RepoAuth {
|
||||
repo = "" // without RepoAuth, unset the provided repo
|
||||
}
|
||||
if _, ok := ch.auth[repo]; !ok {
|
||||
ch.auth[repo] = auth.NewAuth(
|
||||
auth.WithLog(ch.slog),
|
||||
auth.WithHTTPClient(ch.httpClient),
|
||||
auth.WithCreds(ch.AuthCreds()),
|
||||
auth.WithClientID(ch.userAgent),
|
||||
)
|
||||
}
|
||||
return ch.auth[repo]
|
||||
}
|
||||
|
||||
func (ch *clientHost) AuthCreds() func(h string) auth.Cred {
|
||||
if ch == nil || ch.config == nil {
|
||||
return auth.DefaultCredsFn
|
||||
}
|
||||
return func(h string) auth.Cred {
|
||||
// only return credentials to challenges from the registry server, not to any redirects
|
||||
if h == ch.config.Hostname {
|
||||
hCred := ch.config.GetCred()
|
||||
return auth.Cred{User: hCred.User, Password: hCred.Password, Token: hCred.Token}
|
||||
} else {
|
||||
return auth.Cred{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type wrapTransport struct {
|
||||
c *Client
|
||||
orig http.RoundTripper
|
||||
}
|
||||
|
||||
func (wt *wrapTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
resp, err := wt.orig.RoundTrip(req)
|
||||
// copy headers to censor auth field
|
||||
reqHead := req.Header.Clone()
|
||||
if reqHead.Get("Authorization") != "" {
|
||||
reqHead.Set("Authorization", "[censored]")
|
||||
}
|
||||
if err != nil {
|
||||
wt.c.slog.Debug("reg http request",
|
||||
slog.String("req-method", req.Method),
|
||||
slog.String("req-url", req.URL.String()),
|
||||
slog.Any("req-headers", reqHead),
|
||||
slog.String("err", err.Error()))
|
||||
} else {
|
||||
// extract any warnings
|
||||
for _, wh := range resp.Header.Values("Warning") {
|
||||
if match := warnRegexp.FindStringSubmatch(wh); len(match) == 2 {
|
||||
// TODO(bmitch): pass other fields (registry hostname) with structured logging
|
||||
warning.Handle(req.Context(), wt.c.slog, match[1])
|
||||
}
|
||||
}
|
||||
wt.c.slog.Log(req.Context(), types.LevelTrace, "reg http request",
|
||||
slog.String("req-method", req.Method),
|
||||
slog.String("req-url", req.URL.String()),
|
||||
slog.Any("req-headers", reqHead),
|
||||
slog.String("resp-status", resp.Status),
|
||||
slog.Any("resp-headers", resp.Header))
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// HTTPError returns an error based on the status code.
|
||||
func HTTPError(statusCode int) error {
|
||||
switch statusCode {
|
||||
case 401:
|
||||
return fmt.Errorf("%w [http %d]", errs.ErrHTTPUnauthorized, statusCode)
|
||||
case 403:
|
||||
return fmt.Errorf("%w [http %d]", errs.ErrHTTPUnauthorized, statusCode)
|
||||
case 404:
|
||||
return fmt.Errorf("%w [http %d]", errs.ErrNotFound, statusCode)
|
||||
case 429:
|
||||
return fmt.Errorf("%w [http %d]", errs.ErrHTTPRateLimit, statusCode)
|
||||
default:
|
||||
return fmt.Errorf("%w: %s [http %d]", errs.ErrHTTPStatus, http.StatusText(statusCode), statusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func makeRootPool(rootCAPool [][]byte, rootCADirs []string, hostname string, hostcert string) (*x509.CertPool, error) {
|
||||
pool, err := x509.SystemCertPool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, ca := range rootCAPool {
|
||||
if ok := pool.AppendCertsFromPEM(ca); !ok {
|
||||
return nil, fmt.Errorf("failed to load ca: %s", ca)
|
||||
}
|
||||
}
|
||||
for _, dir := range rootCADirs {
|
||||
hostDir := filepath.Join(dir, hostname)
|
||||
files, err := os.ReadDir(hostDir)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("failed to read directory %s: %w", hostDir, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, f := range files {
|
||||
if f.IsDir() {
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(f.Name(), ".crt") {
|
||||
f := filepath.Join(hostDir, f.Name())
|
||||
//#nosec G304 file from a known directory and extension read by the user running the command on their own host
|
||||
cert, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read %s: %w", f, err)
|
||||
}
|
||||
if ok := pool.AppendCertsFromPEM(cert); !ok {
|
||||
return nil, fmt.Errorf("failed to import cert from %s", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if hostcert != "" {
|
||||
if ok := pool.AppendCertsFromPEM([]byte(hostcert)); !ok {
|
||||
// try to parse the certificate and generate a useful error
|
||||
block, _ := pem.Decode([]byte(hostcert))
|
||||
if block == nil {
|
||||
err = fmt.Errorf("pem.Decode is nil")
|
||||
} else {
|
||||
_, err = x509.ParseCertificate(block.Bytes)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to load host specific ca (registry: %s): %w: %s", hostname, err, hostcert)
|
||||
}
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
// sortHostCmp to sort host list of mirrors.
|
||||
func sortHostsCmp(hosts []*clientHost, upstream string) func(i, j int) bool {
|
||||
now := time.Now()
|
||||
// sort by backoff first, then priority decending, then upstream name last
|
||||
return func(i, j int) bool {
|
||||
if now.Before(hosts[i].backoffLast) || now.Before(hosts[j].backoffLast) {
|
||||
return hosts[i].backoffLast.Before(hosts[j].backoffLast)
|
||||
}
|
||||
if hosts[i].config.Priority != hosts[j].config.Priority {
|
||||
return hosts[i].config.Priority < hosts[j].config.Priority
|
||||
}
|
||||
return hosts[i].config.Name != upstream
|
||||
}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
// Package regnet contains networking helper functions for interacting with registries.
|
||||
package regnet
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
)
|
||||
|
||||
func AllowRedirect(src, dest url.URL) error {
|
||||
if src.Scheme == "https" && dest.Scheme != "https" {
|
||||
return fmt.Errorf("redirect from an https to non-https server is not allowed (%s)%.0w", dest.String(), errs.ErrHTTPRedirectRefused)
|
||||
}
|
||||
if !IsLocal(src.Host) && IsLocal(dest.Host) {
|
||||
return fmt.Errorf("redirect to a local domain is not allowed (%s)%.0w", dest.String(), errs.ErrHTTPRedirectRefused)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func IsLocal(hostPort string) bool {
|
||||
// strip trailing port
|
||||
host, _, err := net.SplitHostPort(hostPort)
|
||||
if err != nil {
|
||||
host = hostPort
|
||||
}
|
||||
// parse IP
|
||||
ip := net.ParseIP(host)
|
||||
if ip != nil {
|
||||
return isIPLocal(ip)
|
||||
}
|
||||
// else resolve the hostname and then check each IP
|
||||
ips, err := net.LookupIP(host)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if ip != nil && isIPLocal(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isIPLocal(ip net.IP) bool {
|
||||
return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
// Package reproducible is used for reproducibility.
|
||||
// See <https://reproducible-builds.org/docs/> for more information of the issues being solved.
|
||||
package reproducible
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package reproducible
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
const EpocEnv = "SOURCE_DATE_EPOC"
|
||||
|
||||
var errInvalidEpoc = errors.New("invalid epoc var")
|
||||
|
||||
// TimeNow returns the current time or SOURCE_DATE_EPOC if that is set.
|
||||
func TimeNow() time.Time {
|
||||
now, err := TimeEpocEnv()
|
||||
if err == nil {
|
||||
return now
|
||||
}
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
// TimeEpocEnv returns the time parsed by SOURCE_DATE_EPOC.
|
||||
// This should be used to override any timestamps that should be reproducible.
|
||||
func TimeEpocEnv() (time.Time, error) {
|
||||
sec := os.Getenv(EpocEnv)
|
||||
if sec == "" {
|
||||
return time.Time{}, errInvalidEpoc
|
||||
}
|
||||
secI, err := strconv.ParseInt(sec, 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return time.Unix(secI, 0).UTC(), nil
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
// Package reqmeta provides metadata on requests for prioritizing with a pqueue.
|
||||
package reqmeta
|
||||
|
||||
type Data struct {
|
||||
Kind Kind
|
||||
Size int64
|
||||
}
|
||||
|
||||
type Kind int
|
||||
|
||||
const (
|
||||
Unknown Kind = iota
|
||||
Head
|
||||
Manifest
|
||||
Query
|
||||
Blob
|
||||
)
|
||||
|
||||
const (
|
||||
smallLimit = 4194304 // 4MiB
|
||||
largePct = 0.9 // anything above 90% of largest queued entry size is large
|
||||
)
|
||||
|
||||
func DataNext(queued, active []*Data) int {
|
||||
if len(queued) == 0 {
|
||||
return -1
|
||||
}
|
||||
// After removing one small entry, split remaining requests 50/50 between large and old (truncated int division always rounds down).
|
||||
// If len active = 2, this function returns the 3rd entry (+1), minus 1 for the small, divide by 2 to split with old = goal of 1.
|
||||
largeGoal := len(active) / 2
|
||||
largeI := 0
|
||||
var largeSize int64
|
||||
if largeGoal > 0 {
|
||||
// find the largest queued blob requests
|
||||
for i, cur := range queued {
|
||||
if cur.Kind == Blob && cur.Size > largeSize {
|
||||
largeI = i
|
||||
largeSize = cur.Size
|
||||
}
|
||||
}
|
||||
}
|
||||
largeCutoff := int64(float64(largeSize) * 0.9)
|
||||
// count active requests by type
|
||||
small := 0
|
||||
large := 0
|
||||
old := 0
|
||||
for _, cur := range active {
|
||||
if cur.Kind != Blob && cur.Size <= smallLimit {
|
||||
small++
|
||||
} else if cur.Kind == Blob && largeSize > 0 && cur.Size >= largeCutoff {
|
||||
large++
|
||||
} else {
|
||||
old++
|
||||
}
|
||||
}
|
||||
// if there is at least one active, and none are small, return the best small entry if available.
|
||||
if len(active) > 0 && small == 0 {
|
||||
var sizeI int64
|
||||
bestI := -1
|
||||
kindI := Unknown
|
||||
for i, cur := range queued {
|
||||
// the small search skips blobs and large requests
|
||||
if cur.Kind == Blob || cur.Size > smallLimit {
|
||||
continue
|
||||
}
|
||||
// the best small entry is the:
|
||||
// - first one found if no other matches
|
||||
// - one with a better Kind (Head > Manifest > Query)
|
||||
// - one with the same kind but smaller request
|
||||
if bestI < 0 ||
|
||||
(cur.Kind != Unknown && (kindI == Unknown || cur.Kind < kindI)) ||
|
||||
(cur.Kind == kindI && cur.Size > 0 && (cur.Size < sizeI || sizeI <= 0)) {
|
||||
bestI = i
|
||||
kindI = cur.Kind
|
||||
sizeI = cur.Size
|
||||
}
|
||||
}
|
||||
if bestI >= 0 {
|
||||
return bestI
|
||||
}
|
||||
}
|
||||
// Prefer the biggest of these blobs to minimize the size of the last running blob.
|
||||
if largeGoal > 0 && large < largeGoal && largeSize > 0 {
|
||||
return largeI
|
||||
}
|
||||
// enough small and large, or none available, so return the oldest queued entry to avoid starvation.
|
||||
return 0
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
//go:build !wasm
|
||||
|
||||
// Package sloghandle provides a transition handler for migrating from logrus to slog.
|
||||
package sloghandle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/regclient/regclient/types"
|
||||
)
|
||||
|
||||
func Logrus(logger *logrus.Logger) *logrusHandler {
|
||||
return &logrusHandler{
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
type logrusHandler struct {
|
||||
logger *logrus.Logger
|
||||
attrs []slog.Attr
|
||||
groups []string
|
||||
}
|
||||
|
||||
func (h *logrusHandler) Enabled(_ context.Context, level slog.Level) bool {
|
||||
ll := h.logger.GetLevel()
|
||||
if curLevel, ok := logrusToSlog[ll]; ok {
|
||||
return level >= curLevel
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *logrusHandler) Handle(ctx context.Context, r slog.Record) error {
|
||||
log := logrus.NewEntry(h.logger).WithContext(ctx)
|
||||
if !r.Time.IsZero() {
|
||||
log = log.WithTime(r.Time)
|
||||
}
|
||||
fields := logrus.Fields{}
|
||||
for _, a := range h.attrs {
|
||||
if a.Key != "" {
|
||||
fields[a.Key] = a.Value
|
||||
}
|
||||
}
|
||||
r.Attrs(func(a slog.Attr) bool {
|
||||
if a.Key != "" {
|
||||
fields[a.Key] = a.Value
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(fields) > 0 {
|
||||
log = log.WithFields(fields)
|
||||
}
|
||||
log.Log(slogToLogrus(r.Level), r.Message)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *logrusHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||
ret := h.clone()
|
||||
prefix := ""
|
||||
if len(h.groups) > 0 {
|
||||
prefix = strings.Join(h.groups, ":") + ":"
|
||||
}
|
||||
for _, a := range attrs {
|
||||
if a.Key == "" {
|
||||
continue
|
||||
}
|
||||
ret.attrs = append(ret.attrs, slog.Attr{
|
||||
Key: prefix + a.Key,
|
||||
Value: a.Value,
|
||||
})
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func (h *logrusHandler) WithGroup(name string) slog.Handler {
|
||||
if name == "" {
|
||||
return h
|
||||
}
|
||||
ret := h.clone()
|
||||
ret.groups = append(ret.groups, name)
|
||||
return ret
|
||||
}
|
||||
|
||||
func (h *logrusHandler) clone() *logrusHandler {
|
||||
attrs := make([]slog.Attr, len(h.attrs))
|
||||
copy(attrs, h.attrs)
|
||||
groups := make([]string, len(h.groups))
|
||||
copy(groups, h.groups)
|
||||
return &logrusHandler{
|
||||
logger: h.logger,
|
||||
attrs: attrs,
|
||||
groups: groups,
|
||||
}
|
||||
}
|
||||
|
||||
var logrusToSlog = map[logrus.Level]slog.Level{
|
||||
logrus.TraceLevel: types.LevelTrace,
|
||||
logrus.DebugLevel: slog.LevelDebug,
|
||||
logrus.InfoLevel: slog.LevelInfo,
|
||||
logrus.WarnLevel: slog.LevelWarn,
|
||||
logrus.ErrorLevel: slog.LevelError,
|
||||
logrus.FatalLevel: slog.LevelError + 4,
|
||||
logrus.PanicLevel: slog.LevelError + 8,
|
||||
}
|
||||
|
||||
func slogToLogrus(level slog.Level) logrus.Level {
|
||||
if level <= types.LevelTrace {
|
||||
return logrus.TraceLevel
|
||||
} else if level <= slog.LevelDebug {
|
||||
return logrus.DebugLevel
|
||||
} else if level <= slog.LevelInfo {
|
||||
return logrus.InfoLevel
|
||||
} else if level <= slog.LevelWarn {
|
||||
return logrus.WarnLevel
|
||||
} else if level <= slog.LevelError {
|
||||
return logrus.ErrorLevel
|
||||
} else if level <= slog.LevelError+4 {
|
||||
return logrus.FatalLevel
|
||||
} else {
|
||||
return logrus.PanicLevel
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
// Package strparse is used to parse strings
|
||||
package strparse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
)
|
||||
|
||||
// SplitCSKV splits a comma separated key=value list into a map
|
||||
func SplitCSKV(s string) (map[string]string, error) {
|
||||
state := "key"
|
||||
key := ""
|
||||
val := ""
|
||||
result := map[string]string{}
|
||||
procKV := func() {
|
||||
if key != "" {
|
||||
result[key] = val
|
||||
}
|
||||
state = "key"
|
||||
key = ""
|
||||
val = ""
|
||||
}
|
||||
for _, c := range s {
|
||||
switch state {
|
||||
case "key":
|
||||
switch c {
|
||||
case '"':
|
||||
state = "keyQuote"
|
||||
case '\\':
|
||||
state = "keyEscape"
|
||||
case '=':
|
||||
state = "val"
|
||||
case ',':
|
||||
procKV()
|
||||
default:
|
||||
key = key + string(c)
|
||||
}
|
||||
case "keyQuote":
|
||||
switch c {
|
||||
case '"':
|
||||
state = "key"
|
||||
case '\\':
|
||||
state = "keyEscapeQuote"
|
||||
default:
|
||||
key = key + string(c)
|
||||
}
|
||||
case "keyEscape":
|
||||
key = key + string(c)
|
||||
state = "key"
|
||||
case "keyEscapeQuote":
|
||||
key = key + string(c)
|
||||
state = "keyQuote"
|
||||
case "val":
|
||||
switch c {
|
||||
case '"':
|
||||
state = "valQuote"
|
||||
case ',':
|
||||
procKV()
|
||||
case '\\':
|
||||
state = "valEscape"
|
||||
default:
|
||||
val = val + string(c)
|
||||
}
|
||||
case "valQuote":
|
||||
switch c {
|
||||
case '"':
|
||||
state = "val"
|
||||
case '\\':
|
||||
state = "valEscapeQuote"
|
||||
default:
|
||||
val = val + string(c)
|
||||
}
|
||||
case "valEscape":
|
||||
val = val + string(c)
|
||||
state = "val"
|
||||
case "valEscapeQuote":
|
||||
val = val + string(c)
|
||||
state = "valQuote"
|
||||
default:
|
||||
return nil, fmt.Errorf("unhandled state: %s", state)
|
||||
}
|
||||
}
|
||||
switch state {
|
||||
case "val", "key":
|
||||
procKV()
|
||||
default:
|
||||
return nil, fmt.Errorf("string parsing failed, end state: %s%.0w", state, errs.ErrParsingFailed)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
// Package timejson extends time methods with marshal/unmarshal for json
|
||||
package timejson
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
var errInvalid = errors.New("invalid duration")
|
||||
|
||||
// Duration is an alias to time.Duration
|
||||
// Implementation taken from https://stackoverflow.com/questions/48050945/how-to-unmarshal-json-into-durations
|
||||
type Duration time.Duration
|
||||
|
||||
// MarshalJSON converts a duration to json
|
||||
func (d Duration) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(time.Duration(d).String())
|
||||
}
|
||||
|
||||
// UnmarshalJSON converts json to a duration
|
||||
func (d *Duration) UnmarshalJSON(b []byte) error {
|
||||
var v any
|
||||
if err := json.Unmarshal(b, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
switch value := v.(type) {
|
||||
case float64:
|
||||
*d = Duration(time.Duration(value))
|
||||
return nil
|
||||
case string:
|
||||
timeDur, err := time.ParseDuration(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*d = Duration(timeDur)
|
||||
return nil
|
||||
default:
|
||||
return errInvalid
|
||||
}
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
// Package units is taken from https://github.com/docker/go-units
|
||||
package units
|
||||
|
||||
// Copyright 2015 Docker, Inc.
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var (
|
||||
decimapAbbrs = []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
|
||||
binaryAbbrs = []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"}
|
||||
)
|
||||
|
||||
func getSizeAndUnit(size float64, base float64, unitList []string) (float64, string) {
|
||||
i := 0
|
||||
unitsLimit := len(unitList) - 1
|
||||
for size >= base && i < unitsLimit {
|
||||
size = size / base
|
||||
i++
|
||||
}
|
||||
return size, unitList[i]
|
||||
}
|
||||
|
||||
// CustomSize returns a human-readable approximation of a size using custom format.
|
||||
func CustomSize(format string, size float64, base float64, unitList []string) string {
|
||||
size, unit := getSizeAndUnit(size, base, unitList)
|
||||
return fmt.Sprintf(format, size, unit)
|
||||
}
|
||||
|
||||
// HumanSizeWithPrecision allows the size to be in any precision.
|
||||
func HumanSizeWithPrecision(size float64, width, precision int) string {
|
||||
size, unit := getSizeAndUnit(size, 1000.0, decimapAbbrs)
|
||||
return fmt.Sprintf("%*.*f%s", width, precision, size, unit)
|
||||
}
|
||||
|
||||
// HumanSize returns a human-readable approximation of a size
|
||||
// with a width of 5 (eg. "2.746MB", "796.0KB").
|
||||
func HumanSize(size float64) string {
|
||||
return HumanSizeWithPrecision(size, 5, 3)
|
||||
}
|
||||
|
||||
// BytesSize returns a human-readable size in bytes, kibibytes,
|
||||
// mebibytes, gibibytes, or tebibytes (eg. "44.2kiB", "17.6MiB").
|
||||
func BytesSize(size float64) string {
|
||||
return CustomSize("%5.3f%s", size, 1024.0, binaryAbbrs)
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
// Package version returns details on the Go and Git repo used in the build
|
||||
package version
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"text/tabwriter"
|
||||
)
|
||||
|
||||
const (
|
||||
stateClean = "clean"
|
||||
stateDirty = "dirty"
|
||||
unknown = "unknown"
|
||||
biVCSDate = "vcs.time"
|
||||
biVCSCommit = "vcs.revision"
|
||||
biVCSModified = "vcs.modified"
|
||||
)
|
||||
|
||||
func (i Info) MarshalPretty() ([]byte, error) {
|
||||
buf := &bytes.Buffer{}
|
||||
tw := tabwriter.NewWriter(buf, 0, 0, 1, ' ', 0)
|
||||
fmt.Fprintf(tw, "VCSTag:\t%s\n", i.VCSTag)
|
||||
fmt.Fprintf(tw, "VCSRef:\t%s\n", i.VCSRef)
|
||||
fmt.Fprintf(tw, "VCSCommit:\t%s\n", i.VCSCommit)
|
||||
fmt.Fprintf(tw, "VCSState:\t%s\n", i.VCSState)
|
||||
fmt.Fprintf(tw, "VCSDate:\t%s\n", i.VCSDate)
|
||||
fmt.Fprintf(tw, "Platform:\t%s\n", i.Platform)
|
||||
fmt.Fprintf(tw, "GoVer:\t%s\n", i.GoVer)
|
||||
fmt.Fprintf(tw, "GoCompiler:\t%s\n", i.GoCompiler)
|
||||
err := tw.Flush()
|
||||
return buf.Bytes(), err
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
//go:build go1.18
|
||||
|
||||
package version
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
)
|
||||
|
||||
var vcsTag = ""
|
||||
|
||||
type Info struct {
|
||||
GoVer string `json:"goVersion"` // go version
|
||||
GoCompiler string `json:"goCompiler"` // go compiler
|
||||
Platform string `json:"platform"` // os/arch
|
||||
VCSCommit string `json:"vcsCommit"` // commit sha
|
||||
VCSDate string `json:"vcsDate"` // commit date in RFC3339 format
|
||||
VCSRef string `json:"vcsRef"` // commit sha + dirty if state is not clean
|
||||
VCSState string `json:"vcsState"` // clean or dirty
|
||||
VCSTag string `json:"vcsTag"` // tag is not available from Go
|
||||
Debug *debug.BuildInfo `json:"debug,omitempty"` // build info debugging data
|
||||
}
|
||||
|
||||
func GetInfo() Info {
|
||||
i := Info{
|
||||
GoVer: unknown,
|
||||
Platform: unknown,
|
||||
VCSCommit: unknown,
|
||||
VCSDate: unknown,
|
||||
VCSRef: unknown,
|
||||
VCSState: unknown,
|
||||
VCSTag: vcsTag,
|
||||
}
|
||||
|
||||
i.GoVer = runtime.Version()
|
||||
i.GoCompiler = runtime.Compiler
|
||||
i.Platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH)
|
||||
|
||||
if bi, ok := debug.ReadBuildInfo(); ok && bi != nil {
|
||||
i.Debug = bi
|
||||
if i.VCSTag == "" {
|
||||
i.VCSTag = bi.Main.Version
|
||||
}
|
||||
date := biSetting(bi, biVCSDate)
|
||||
if t, err := time.Parse(time.RFC3339, date); err == nil {
|
||||
i.VCSDate = t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
i.VCSCommit = biSetting(bi, biVCSCommit)
|
||||
i.VCSRef = i.VCSCommit
|
||||
modified := biSetting(bi, biVCSModified)
|
||||
if modified == "true" {
|
||||
i.VCSState = stateDirty
|
||||
i.VCSRef += "-" + stateDirty
|
||||
} else if modified == "false" {
|
||||
i.VCSState = stateClean
|
||||
}
|
||||
}
|
||||
|
||||
return i
|
||||
}
|
||||
|
||||
func biSetting(bi *debug.BuildInfo, key string) string {
|
||||
if bi == nil {
|
||||
return unknown
|
||||
}
|
||||
for _, setting := range bi.Settings {
|
||||
if setting.Key == key {
|
||||
return setting.Value
|
||||
}
|
||||
}
|
||||
return unknown
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
//go:build !go1.18
|
||||
|
||||
package version
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
type Info struct {
|
||||
GoVer string `json:"goVersion"` // go version
|
||||
GoCompiler string `json:"goCompiler"` // go compiler
|
||||
Platform string `json:"platform"` // os/arch
|
||||
VCSCommit string `json:"vcsCommit"` // commit sha
|
||||
VCSDate string `json:"vcsDate"` // commit date in RFC3339 format
|
||||
VCSRef string `json:"vcsRef"` // commit sha + dirty if state is not clean
|
||||
VCSState string `json:"vcsState"` // clean or dirty
|
||||
VCSTag string `json:"vcsTag"` // tag
|
||||
}
|
||||
|
||||
func GetInfo() Info {
|
||||
i := Info{
|
||||
GoVer: unknown,
|
||||
Platform: unknown,
|
||||
VCSCommit: unknown,
|
||||
VCSDate: unknown,
|
||||
VCSRef: unknown,
|
||||
VCSState: unknown,
|
||||
VCSTag: "",
|
||||
}
|
||||
|
||||
i.GoVer = runtime.Version()
|
||||
i.GoCompiler = runtime.Compiler
|
||||
i.Platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH)
|
||||
|
||||
return i
|
||||
}
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
package regclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types/descriptor"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/manifest"
|
||||
"github.com/regclient/regclient/types/platform"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/warning"
|
||||
)
|
||||
|
||||
type manifestOpt struct {
|
||||
d descriptor.Descriptor
|
||||
platform *platform.Platform
|
||||
schemeOpts []scheme.ManifestOpts
|
||||
requireDigest bool
|
||||
}
|
||||
|
||||
// ManifestOpts define options for the Manifest* commands.
|
||||
type ManifestOpts func(*manifestOpt)
|
||||
|
||||
// WithManifest passes a manifest to ManifestDelete.
|
||||
func WithManifest(m manifest.Manifest) ManifestOpts {
|
||||
return func(opts *manifestOpt) {
|
||||
opts.schemeOpts = append(opts.schemeOpts, scheme.WithManifest(m))
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifestCheckReferrers checks for referrers field on ManifestDelete.
|
||||
// This will update the client managed referrer listing.
|
||||
func WithManifestCheckReferrers() ManifestOpts {
|
||||
return func(opts *manifestOpt) {
|
||||
opts.schemeOpts = append(opts.schemeOpts, scheme.WithManifestCheckReferrers())
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifestChild for ManifestPut indicates the manifest is not the top level manifest being copied.
|
||||
// This is used by the ocidir scheme to determine what entries to include in the index.json.
|
||||
func WithManifestChild() ManifestOpts {
|
||||
return func(opts *manifestOpt) {
|
||||
opts.schemeOpts = append(opts.schemeOpts, scheme.WithManifestChild())
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifestDesc includes the descriptor for ManifestGet.
|
||||
// This is used to automatically extract a Data field if available.
|
||||
func WithManifestDesc(d descriptor.Descriptor) ManifestOpts {
|
||||
return func(opts *manifestOpt) {
|
||||
opts.d = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifestPlatform resolves the platform specific manifest on Get and Head requests.
|
||||
// This causes an additional GET query to a registry when an Index or Manifest List is encountered.
|
||||
// This option is ignored if the retrieved manifest is not an Index or Manifest List.
|
||||
func WithManifestPlatform(p platform.Platform) ManifestOpts {
|
||||
return func(opts *manifestOpt) {
|
||||
opts.platform = &p
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifestRequireDigest falls back from a HEAD to a GET request when digest headers aren't received.
|
||||
func WithManifestRequireDigest() ManifestOpts {
|
||||
return func(opts *manifestOpt) {
|
||||
opts.requireDigest = true
|
||||
}
|
||||
}
|
||||
|
||||
// ManifestDelete removes a manifest, including all tags pointing to that registry.
|
||||
// The reference must include the digest to delete (see TagDelete for deleting a tag).
|
||||
// All tags pointing to the manifest will be deleted.
|
||||
func (rc *RegClient) ManifestDelete(ctx context.Context, r ref.Ref, opts ...ManifestOpts) error {
|
||||
if !r.IsSet() {
|
||||
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
|
||||
for _, fn := range opts {
|
||||
fn(&opt)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return schemeAPI.ManifestDelete(ctx, r, opt.schemeOpts...)
|
||||
}
|
||||
|
||||
// ManifestGet retrieves a manifest.
|
||||
func (rc *RegClient) ManifestGet(ctx context.Context, r ref.Ref, opts ...ManifestOpts) (manifest.Manifest, error) {
|
||||
if !r.IsSet() {
|
||||
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
|
||||
for _, fn := range opts {
|
||||
fn(&opt)
|
||||
}
|
||||
if opt.d.Digest != "" {
|
||||
r = r.AddDigest(opt.d.Digest.String())
|
||||
data, err := opt.d.GetData()
|
||||
if err == nil {
|
||||
return manifest.New(
|
||||
manifest.WithDesc(opt.d),
|
||||
manifest.WithRaw(data),
|
||||
manifest.WithRef(r),
|
||||
)
|
||||
}
|
||||
}
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := schemeAPI.ManifestGet(ctx, r)
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
if opt.platform != nil && !m.IsList() {
|
||||
rc.slog.Debug("ignoring platform option, image is not an index",
|
||||
slog.String("platform", opt.platform.String()),
|
||||
slog.String("ref", r.CommonName()))
|
||||
}
|
||||
// this will loop to handle a nested index
|
||||
for opt.platform != nil && m.IsList() {
|
||||
d, err := manifest.GetPlatformDesc(m, opt.platform)
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
r = r.SetDigest(d.Digest.String())
|
||||
m, err = schemeAPI.ManifestGet(ctx, r)
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
// ManifestHead queries for the existence of a manifest and returns metadata (digest, media-type, size).
|
||||
func (rc *RegClient) ManifestHead(ctx context.Context, r ref.Ref, opts ...ManifestOpts) (manifest.Manifest, error) {
|
||||
if !r.IsSet() {
|
||||
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
|
||||
for _, fn := range opts {
|
||||
fn(&opt)
|
||||
}
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := schemeAPI.ManifestHead(ctx, r)
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
if opt.platform != nil && !m.IsList() {
|
||||
rc.slog.Debug("ignoring platform option, image is not an index",
|
||||
slog.String("platform", opt.platform.String()),
|
||||
slog.String("ref", r.CommonName()))
|
||||
}
|
||||
// this will loop to handle a nested index
|
||||
for opt.platform != nil && m.IsList() {
|
||||
if !m.IsSet() {
|
||||
m, err = schemeAPI.ManifestGet(ctx, r)
|
||||
}
|
||||
d, err := manifest.GetPlatformDesc(m, opt.platform)
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
r = r.SetDigest(d.Digest.String())
|
||||
m, err = schemeAPI.ManifestHead(ctx, r)
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
}
|
||||
if opt.requireDigest && m.GetDescriptor().Digest.String() == "" {
|
||||
m, err = schemeAPI.ManifestGet(ctx, r)
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
// ManifestPut pushes a manifest.
|
||||
// Any descriptors referenced by the manifest typically need to be pushed first.
|
||||
func (rc *RegClient) ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...ManifestOpts) error {
|
||||
if !r.IsSetRepo() {
|
||||
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
|
||||
for _, fn := range opts {
|
||||
fn(&opt)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return schemeAPI.ManifestPut(ctx, r, m, opt.schemeOpts...)
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package regclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/regclient/regclient/types/ping"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
// Ping verifies access to a registry or equivalent.
|
||||
func (rc *RegClient) Ping(ctx context.Context, r ref.Ref) (ping.Result, error) {
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return ping.Result{}, err
|
||||
}
|
||||
|
||||
return schemeAPI.Ping(ctx, r)
|
||||
}
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
// Package archive is used to read and write tar files
|
||||
package archive
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// CompressType identifies the detected compression type
|
||||
type CompressType int
|
||||
|
||||
const (
|
||||
CompressNone CompressType = iota // uncompressed or unable to detect compression
|
||||
CompressBzip2 // bzip2
|
||||
CompressGzip // gzip
|
||||
CompressXz // xz
|
||||
CompressZstd // zstd
|
||||
)
|
||||
|
||||
// compressHeaders are used to detect the compression type
|
||||
var compressHeaders = map[CompressType][]byte{
|
||||
CompressBzip2: []byte("\x42\x5A\x68"),
|
||||
CompressGzip: []byte("\x1F\x8B\x08"),
|
||||
CompressXz: []byte("\xFD\x37\x7A\x58\x5A\x00"),
|
||||
CompressZstd: []byte("\x28\xB5\x2F\xFD"),
|
||||
}
|
||||
|
||||
func Compress(r io.Reader, oComp CompressType) (io.ReadCloser, error) {
|
||||
switch oComp {
|
||||
// note, bzip2 compression is not supported
|
||||
case CompressGzip:
|
||||
return writeToRead(r, newGzipWriter)
|
||||
case CompressXz:
|
||||
return writeToRead(r, xz.NewWriter)
|
||||
case CompressZstd:
|
||||
return writeToRead(r, newZstdWriter)
|
||||
case CompressNone:
|
||||
return io.NopCloser(r), nil
|
||||
default:
|
||||
return nil, ErrUnknownType
|
||||
}
|
||||
}
|
||||
|
||||
// newGzipWriter generates a writer and an always nil error.
|
||||
func newGzipWriter(w io.Writer) (io.WriteCloser, error) {
|
||||
return gzip.NewWriter(w), nil
|
||||
}
|
||||
|
||||
// newZstdWriter generates a writer with the default options.
|
||||
func newZstdWriter(w io.Writer) (io.WriteCloser, error) {
|
||||
return zstd.NewWriter(w)
|
||||
}
|
||||
|
||||
// writeToRead uses a pipe + goroutine + copy to switch from a writer to a reader.
|
||||
func writeToRead[wc io.WriteCloser](src io.Reader, newWriterFn func(io.Writer) (wc, error)) (io.ReadCloser, error) {
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
// buffer output to avoid lots of small reads
|
||||
bw := bufio.NewWriterSize(pw, 2<<16)
|
||||
dest, err := newWriterFn(bw)
|
||||
if err != nil {
|
||||
_ = pw.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
if _, err := io.Copy(dest, src); err != nil {
|
||||
_ = pw.CloseWithError(err)
|
||||
}
|
||||
if err := dest.Close(); err != nil {
|
||||
_ = pw.CloseWithError(err)
|
||||
}
|
||||
if err := bw.Flush(); err != nil {
|
||||
_ = pw.CloseWithError(err)
|
||||
}
|
||||
_ = pw.Close()
|
||||
}()
|
||||
return pr, nil
|
||||
}
|
||||
|
||||
// Decompress extracts gzip and bzip streams
|
||||
func Decompress(r io.Reader) (io.Reader, error) {
|
||||
// create bufio to peak on first few bytes
|
||||
br := bufio.NewReader(r)
|
||||
head, err := br.Peek(10)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return br, fmt.Errorf("failed to detect compression: %w", err)
|
||||
}
|
||||
|
||||
// compare peaked data against known compression types
|
||||
switch DetectCompression(head) {
|
||||
case CompressBzip2:
|
||||
return bzip2.NewReader(br), nil
|
||||
case CompressGzip:
|
||||
return gzip.NewReader(br)
|
||||
case CompressXz:
|
||||
return xz.NewReader(br)
|
||||
case CompressZstd:
|
||||
return zstd.NewReader(br)
|
||||
default:
|
||||
return br, nil
|
||||
}
|
||||
}
|
||||
|
||||
// DetectCompression identifies the compression type based on the first few bytes
|
||||
func DetectCompression(head []byte) CompressType {
|
||||
for c, b := range compressHeaders {
|
||||
if bytes.HasPrefix(head, b) {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return CompressNone
|
||||
}
|
||||
|
||||
func (ct CompressType) String() string {
|
||||
mt, err := ct.MarshalText()
|
||||
if err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
return string(mt)
|
||||
}
|
||||
|
||||
func (ct CompressType) MarshalText() ([]byte, error) {
|
||||
switch ct {
|
||||
case CompressNone:
|
||||
return []byte("none"), nil
|
||||
case CompressBzip2:
|
||||
return []byte("bzip2"), nil
|
||||
case CompressGzip:
|
||||
return []byte("gzip"), nil
|
||||
case CompressXz:
|
||||
return []byte("xz"), nil
|
||||
case CompressZstd:
|
||||
return []byte("zstd"), nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown compression type")
|
||||
}
|
||||
|
||||
func (ct *CompressType) UnmarshalText(text []byte) error {
|
||||
switch string(text) {
|
||||
case "none":
|
||||
*ct = CompressNone
|
||||
case "bzip2":
|
||||
*ct = CompressBzip2
|
||||
case "gzip":
|
||||
*ct = CompressGzip
|
||||
case "xz":
|
||||
*ct = CompressXz
|
||||
case "zstd":
|
||||
*ct = CompressZstd
|
||||
default:
|
||||
return fmt.Errorf("unknown compression type %s", string(text))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
package archive
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
// ErrNotImplemented used for routines that need to be developed still
|
||||
ErrNotImplemented = errors.New("this archive routine is not implemented yet")
|
||||
// ErrUnknownType used for unknown compression types
|
||||
ErrUnknownType = errors.New("unknown compression type")
|
||||
// ErrXzUnsupported because there isn't a Go package for this and I'm
|
||||
// avoiding dependencies on external binaries
|
||||
ErrXzUnsupported = errors.New("xz compression is currently unsupported")
|
||||
)
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TarOpts configures options for Create/Extract tar
|
||||
type TarOpts func(*tarOpts)
|
||||
|
||||
// TODO: add support for compressed files with bzip
|
||||
type tarOpts struct {
|
||||
// allowRelative bool // allow relative paths outside of target folder
|
||||
compress string
|
||||
}
|
||||
|
||||
// TarCompressGzip option to use gzip compression on tar files
|
||||
func TarCompressGzip(to *tarOpts) {
|
||||
to.compress = "gzip"
|
||||
}
|
||||
|
||||
// TarUncompressed option to tar (noop)
|
||||
func TarUncompressed(to *tarOpts) {
|
||||
}
|
||||
|
||||
// TODO: add option for full path or to adjust the relative path
|
||||
|
||||
// Tar creation
|
||||
func Tar(ctx context.Context, path string, w io.Writer, opts ...TarOpts) error {
|
||||
to := tarOpts{}
|
||||
for _, opt := range opts {
|
||||
opt(&to)
|
||||
}
|
||||
|
||||
twOut := w
|
||||
if to.compress == "gzip" {
|
||||
gw := gzip.NewWriter(w)
|
||||
defer gw.Close()
|
||||
twOut = gw
|
||||
}
|
||||
|
||||
tw := tar.NewWriter(twOut)
|
||||
defer tw.Close()
|
||||
|
||||
// walk the path performing a recursive tar
|
||||
err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error {
|
||||
// return any errors filepath encounters accessing the file
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: handle symlinks, security attributes, hard links
|
||||
// TODO: add options for file owner and timestamps
|
||||
// TODO: add options to override time, or disable access/change stamps
|
||||
|
||||
// adjust for relative path
|
||||
relPath, err := filepath.Rel(path, file)
|
||||
if err != nil || relPath == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
header, err := tar.FileInfoHeader(fi, relPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header.Format = tar.FormatPAX
|
||||
header.Name = filepath.ToSlash(relPath)
|
||||
header.AccessTime = time.Time{}
|
||||
header.ChangeTime = time.Time{}
|
||||
header.ModTime = header.ModTime.Truncate(time.Second)
|
||||
|
||||
if err = tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// open file and copy contents into tar writer
|
||||
if header.Typeflag == tar.TypeReg && header.Size > 0 {
|
||||
//#nosec G122 G304 filename is limited to provided path directory
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = io.Copy(tw, f); err != nil {
|
||||
return err
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close file: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// Extract Tar
|
||||
func Extract(ctx context.Context, path string, r io.Reader, opts ...TarOpts) error {
|
||||
to := tarOpts{}
|
||||
for _, opt := range opts {
|
||||
opt(&to)
|
||||
}
|
||||
|
||||
// verify path exists
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return fmt.Errorf("extract path must be a directory: \"%s\"", path)
|
||||
}
|
||||
|
||||
// decompress
|
||||
rd, err := Decompress(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rt := tar.NewReader(rd)
|
||||
for {
|
||||
hdr, err := rt.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// join a cleaned version of the filename with the path
|
||||
fn := filepath.Join(path, filepath.Clean("/"+hdr.Name))
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if hdr.Mode < 0 || hdr.Mode > math.MaxUint32 {
|
||||
return fmt.Errorf("integer conversion overflow/underflow (file mode = %d)", hdr.Mode)
|
||||
}
|
||||
err = os.MkdirAll(fn, fs.FileMode(hdr.Mode))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
// TODO: configure file mode, creation timestamp, etc
|
||||
//#nosec G304 filename is limited to provided path directory
|
||||
fh, err := os.Create(fn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := io.CopyN(fh, rt, hdr.Size)
|
||||
errC := fh.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if errC != nil {
|
||||
return fmt.Errorf("failed to close file: %w", errC)
|
||||
}
|
||||
if n != hdr.Size {
|
||||
return fmt.Errorf("size mismatch extracting \"%s\", expected %d, extracted %d", hdr.Name, hdr.Size, n)
|
||||
}
|
||||
// TODO: handle other tar types (symlinks, etc)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package regclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/platform"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/referrer"
|
||||
"github.com/regclient/regclient/types/warning"
|
||||
)
|
||||
|
||||
// ReferrerList retrieves a list of referrers to a manifest.
|
||||
// The descriptor list should contain manifests that each have a subject field matching the requested ref.
|
||||
func (rc *RegClient) ReferrerList(ctx context.Context, rSubject ref.Ref, opts ...scheme.ReferrerOpts) (referrer.ReferrerList, error) {
|
||||
if !rSubject.IsSet() {
|
||||
return referrer.ReferrerList{}, fmt.Errorf("ref is not set: %s%.0w", rSubject.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
// set the digest on the subject reference
|
||||
config := scheme.ReferrerConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&config)
|
||||
}
|
||||
if rSubject.Digest == "" || config.Platform != "" {
|
||||
mo := []ManifestOpts{WithManifestRequireDigest()}
|
||||
if config.Platform != "" {
|
||||
p, err := platform.Parse(config.Platform)
|
||||
if err != nil {
|
||||
return referrer.ReferrerList{}, fmt.Errorf("failed to lookup referrer platform: %w", err)
|
||||
}
|
||||
mo = append(mo, WithManifestPlatform(p))
|
||||
}
|
||||
m, err := rc.ManifestHead(ctx, rSubject, mo...)
|
||||
if err != nil {
|
||||
return referrer.ReferrerList{}, fmt.Errorf("failed to get digest for subject: %w", err)
|
||||
}
|
||||
rSubject = rSubject.SetDigest(m.GetDescriptor().Digest.String())
|
||||
}
|
||||
// lookup the scheme for the appropriate ref
|
||||
var r ref.Ref
|
||||
if config.SrcRepo.IsSet() {
|
||||
r = config.SrcRepo
|
||||
} else {
|
||||
r = rSubject
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return referrer.ReferrerList{}, err
|
||||
}
|
||||
return schemeAPI.ReferrerList(ctx, rSubject, opts...)
|
||||
}
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
// Package regclient is used to access OCI registries.
|
||||
package regclient
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/regclient/regclient/config"
|
||||
"github.com/regclient/regclient/internal/version"
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/scheme/ocidir"
|
||||
"github.com/regclient/regclient/scheme/reg"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultUserAgent sets the header on http requests.
|
||||
DefaultUserAgent = "regclient/regclient"
|
||||
// DockerCertDir default location for docker certs.
|
||||
DockerCertDir = "/etc/docker/certs.d"
|
||||
// DockerRegistry is the well known name of Docker Hub, "docker.io".
|
||||
DockerRegistry = config.DockerRegistry
|
||||
// DockerRegistryAuth is the name of Docker Hub seen in docker's config.json.
|
||||
DockerRegistryAuth = config.DockerRegistryAuth
|
||||
// DockerRegistryDNS is the actual registry DNS name for Docker Hub.
|
||||
DockerRegistryDNS = config.DockerRegistryDNS
|
||||
)
|
||||
|
||||
// RegClient is used to access OCI distribution-spec registries.
|
||||
type RegClient struct {
|
||||
hosts map[string]*config.Host
|
||||
hostDefault *config.Host
|
||||
regOpts []reg.Opts
|
||||
schemes map[string]scheme.API
|
||||
slog *slog.Logger
|
||||
userAgent string
|
||||
}
|
||||
|
||||
// Opt functions are used by [New] to create a [*RegClient].
|
||||
type Opt func(*RegClient)
|
||||
|
||||
// New returns a registry client.
|
||||
func New(opts ...Opt) *RegClient {
|
||||
rc := RegClient{
|
||||
hosts: map[string]*config.Host{},
|
||||
userAgent: DefaultUserAgent,
|
||||
regOpts: []reg.Opts{},
|
||||
schemes: map[string]scheme.API{},
|
||||
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
|
||||
}
|
||||
|
||||
info := version.GetInfo()
|
||||
if info.VCSTag != "" {
|
||||
rc.userAgent = fmt.Sprintf("%s (%s)", rc.userAgent, info.VCSTag)
|
||||
} else {
|
||||
rc.userAgent = fmt.Sprintf("%s (%s)", rc.userAgent, info.VCSRef)
|
||||
}
|
||||
|
||||
// inject Docker Hub settings
|
||||
_ = rc.hostSet(*config.HostNewName(config.DockerRegistryAuth))
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(&rc)
|
||||
}
|
||||
|
||||
// configure regOpts
|
||||
hostList := []*config.Host{}
|
||||
for _, h := range rc.hosts {
|
||||
hostList = append(hostList, h)
|
||||
}
|
||||
rc.regOpts = append(
|
||||
rc.regOpts,
|
||||
reg.WithConfigHosts(hostList),
|
||||
reg.WithConfigHostDefault(rc.hostDefault),
|
||||
reg.WithSlog(rc.slog),
|
||||
reg.WithUserAgent(rc.userAgent),
|
||||
)
|
||||
|
||||
// setup scheme's
|
||||
rc.schemes["reg"] = reg.New(rc.regOpts...)
|
||||
rc.schemes["ocidir"] = ocidir.New(
|
||||
ocidir.WithSlog(rc.slog),
|
||||
)
|
||||
|
||||
rc.slog.Debug("regclient initialized",
|
||||
slog.String("VCSRef", info.VCSRef),
|
||||
slog.String("VCSTag", info.VCSTag))
|
||||
|
||||
return &rc
|
||||
}
|
||||
|
||||
// WithBlobLimit sets the max size for chunked blob uploads which get stored in memory.
|
||||
//
|
||||
// Deprecated: replace with WithRegOpts(reg.WithBlobLimit(limit)), see [WithRegOpts] and [reg.WithBlobLimit].
|
||||
//
|
||||
//go:fix inline
|
||||
func WithBlobLimit(limit int64) Opt {
|
||||
return WithRegOpts(reg.WithBlobLimit(limit))
|
||||
}
|
||||
|
||||
// WithBlobSize overrides default blob sizes.
|
||||
//
|
||||
// Deprecated: replace with WithRegOpts(reg.WithBlobSize(chunk, max)), see [WithRegOpts] and [reg.WithBlobSize].
|
||||
//
|
||||
//go:fix inline
|
||||
func WithBlobSize(chunk, max int64) Opt {
|
||||
return WithRegOpts(reg.WithBlobSize(chunk, max))
|
||||
}
|
||||
|
||||
// WithCertDir adds a path of certificates to trust similar to Docker's /etc/docker/certs.d.
|
||||
//
|
||||
// Deprecated: replace with WithRegOpts(reg.WithCertDirs(path)), see [WithRegOpts] and [reg.WithCertDirs].
|
||||
//
|
||||
//go:fix inline
|
||||
func WithCertDir(path ...string) Opt {
|
||||
return WithRegOpts(reg.WithCertDirs(path))
|
||||
}
|
||||
|
||||
// WithConfigHost adds a list of config host settings.
|
||||
func WithConfigHost(configHost ...config.Host) Opt {
|
||||
return func(rc *RegClient) {
|
||||
rc.hostLoad("host", configHost)
|
||||
}
|
||||
}
|
||||
|
||||
// WithConfigHostDefault adds default settings for new hosts.
|
||||
func WithConfigHostDefault(configHost config.Host) Opt {
|
||||
return func(rc *RegClient) {
|
||||
rc.hostDefault = &configHost
|
||||
}
|
||||
}
|
||||
|
||||
// WithConfigHosts adds a list of config host settings.
|
||||
//
|
||||
// Deprecated: replace with [WithConfigHost].
|
||||
//
|
||||
//go:fix inline
|
||||
func WithConfigHosts(configHosts []config.Host) Opt {
|
||||
return WithConfigHost(configHosts...)
|
||||
}
|
||||
|
||||
// WithDockerCerts adds certificates trusted by docker in /etc/docker/certs.d.
|
||||
func WithDockerCerts() Opt {
|
||||
return WithRegOpts(reg.WithCertDirs([]string{DockerCertDir}))
|
||||
}
|
||||
|
||||
// WithDockerCreds adds configuration from users docker config with registry logins.
|
||||
// This changes the default value from the config file, and should be added after the config file is loaded.
|
||||
func WithDockerCreds() Opt {
|
||||
return func(rc *RegClient) {
|
||||
configHosts, err := config.DockerLoad()
|
||||
if err != nil {
|
||||
rc.slog.Warn("Failed to load docker creds",
|
||||
slog.String("err", err.Error()))
|
||||
return
|
||||
}
|
||||
rc.hostLoad("docker", configHosts)
|
||||
}
|
||||
}
|
||||
|
||||
// WithDockerCredsFile adds configuration from a named docker config file with registry logins.
|
||||
// This changes the default value from the config file, and should be added after the config file is loaded.
|
||||
func WithDockerCredsFile(fname string) Opt {
|
||||
return func(rc *RegClient) {
|
||||
configHosts, err := config.DockerLoadFile(fname)
|
||||
if err != nil {
|
||||
rc.slog.Warn("Failed to load docker creds",
|
||||
slog.String("err", err.Error()))
|
||||
return
|
||||
}
|
||||
rc.hostLoad("docker-file", configHosts)
|
||||
}
|
||||
}
|
||||
|
||||
// WithRegOpts passes through opts to the reg scheme.
|
||||
func WithRegOpts(opts ...reg.Opts) Opt {
|
||||
return func(rc *RegClient) {
|
||||
if len(opts) == 0 {
|
||||
return
|
||||
}
|
||||
rc.regOpts = append(rc.regOpts, opts...)
|
||||
}
|
||||
}
|
||||
|
||||
// WithRetryDelay specifies the time permitted for retry delays.
|
||||
//
|
||||
// Deprecated: replace with WithRegOpts(reg.WithDelay(delayInit, delayMax)), see [WithRegOpts] and [reg.WithDelay].
|
||||
//
|
||||
//go:fix inline
|
||||
func WithRetryDelay(delayInit, delayMax time.Duration) Opt {
|
||||
return WithRegOpts(reg.WithDelay(delayInit, delayMax))
|
||||
}
|
||||
|
||||
// WithRetryLimit specifies the number of retries for non-fatal errors.
|
||||
//
|
||||
// Deprecated: replace with WithRegOpts(reg.WithRetryLimit(retryLimit)), see [WithRegOpts] and [reg.WithRetryLimit].
|
||||
//
|
||||
//go:fix inline
|
||||
func WithRetryLimit(retryLimit int) Opt {
|
||||
return WithRegOpts(reg.WithRetryLimit(retryLimit))
|
||||
}
|
||||
|
||||
// WithSlog configures the slog Logger.
|
||||
func WithSlog(slog *slog.Logger) Opt {
|
||||
return func(rc *RegClient) {
|
||||
rc.slog = slog
|
||||
}
|
||||
}
|
||||
|
||||
// WithUserAgent specifies the User-Agent http header.
|
||||
func WithUserAgent(ua string) Opt {
|
||||
return func(rc *RegClient) {
|
||||
rc.userAgent = ua
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RegClient) hostLoad(src string, hosts []config.Host) {
|
||||
for _, configHost := range hosts {
|
||||
if configHost.Name == "" {
|
||||
if configHost.Pass != "" {
|
||||
configHost.Pass = "***"
|
||||
}
|
||||
if configHost.Token != "" {
|
||||
configHost.Token = "***"
|
||||
}
|
||||
rc.slog.Warn("Ignoring registry config without a name",
|
||||
slog.Any("entry", configHost))
|
||||
continue
|
||||
}
|
||||
if configHost.Name == DockerRegistry || configHost.Name == DockerRegistryDNS || configHost.Name == DockerRegistryAuth {
|
||||
configHost.Name = DockerRegistry
|
||||
if configHost.Hostname == "" || configHost.Hostname == DockerRegistry || configHost.Hostname == DockerRegistryAuth {
|
||||
configHost.Hostname = DockerRegistryDNS
|
||||
}
|
||||
}
|
||||
tls, _ := configHost.TLS.MarshalText()
|
||||
rc.slog.Debug("Loading config",
|
||||
slog.Int64("blobChunk", configHost.BlobChunk),
|
||||
slog.Int64("blobMax", configHost.BlobMax),
|
||||
slog.String("helper", configHost.CredHelper),
|
||||
slog.String("hostname", configHost.Hostname),
|
||||
slog.Any("mirrors", configHost.Mirrors),
|
||||
slog.String("name", configHost.Name),
|
||||
slog.String("pathPrefix", configHost.PathPrefix),
|
||||
slog.Bool("repoAuth", configHost.RepoAuth),
|
||||
slog.String("source", src),
|
||||
slog.String("tls", string(tls)),
|
||||
slog.String("user", configHost.User))
|
||||
err := rc.hostSet(configHost)
|
||||
if err != nil {
|
||||
rc.slog.Warn("Failed to update host config",
|
||||
slog.String("host", configHost.Name),
|
||||
slog.String("user", configHost.User),
|
||||
slog.String("error", err.Error()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RegClient) hostSet(newHost config.Host) error {
|
||||
name := newHost.Name
|
||||
var err error
|
||||
if _, ok := rc.hosts[name]; !ok {
|
||||
// merge newHost with default host settings
|
||||
rc.hosts[name] = config.HostNewDefName(rc.hostDefault, name)
|
||||
err = rc.hosts[name].Merge(newHost, nil)
|
||||
} else {
|
||||
// merge newHost with existing settings
|
||||
err = rc.hosts[name].Merge(newHost, rc.slog)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
//go:build !wasm
|
||||
|
||||
package regclient
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/regclient/regclient/internal/sloghandle"
|
||||
)
|
||||
|
||||
// WithLog configuring logging with a logrus Logger.
|
||||
// Note that regclient has switched to log/slog for logging and my eventually deprecate logrus support.
|
||||
func WithLog(log *logrus.Logger) Opt {
|
||||
return func(rc *RegClient) {
|
||||
rc.slog = slog.New(sloghandle.Logrus(log))
|
||||
}
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
# Release v0.11.5
|
||||
|
||||
Security:
|
||||
|
||||
- Prevent https to non-https downgrades and localhost redirects. ([PR 1093][pr-1093])
|
||||
- Forbid sending auth on redirects. ([PR 1095][pr-1095])
|
||||
|
||||
Features:
|
||||
|
||||
- Add regbot `manifest.descriptor` to the sandbox. ([PR 1091][pr-1091])
|
||||
|
||||
Contributors:
|
||||
|
||||
- @GimmyDatBeeR
|
||||
- @sudo-bmitch
|
||||
|
||||
[pr-1091]: https://github.com/regclient/regclient/pull/1091
|
||||
[pr-1093]: https://github.com/regclient/regclient/pull/1093
|
||||
[pr-1095]: https://github.com/regclient/regclient/pull/1095
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package regclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/repo"
|
||||
)
|
||||
|
||||
type repoLister interface {
|
||||
RepoList(ctx context.Context, hostname string, opts ...scheme.RepoOpts) (*repo.RepoList, error)
|
||||
}
|
||||
|
||||
// RepoList returns a list of repositories on a registry.
|
||||
// Note the underlying "_catalog" API is not supported on many cloud registries.
|
||||
func (rc *RegClient) RepoList(ctx context.Context, hostname string, opts ...scheme.RepoOpts) (*repo.RepoList, error) {
|
||||
i := strings.Index(hostname, "/")
|
||||
if i > 0 {
|
||||
return nil, fmt.Errorf("invalid hostname: %s%.0w", hostname, errs.ErrParsingFailed)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet("reg")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rl, ok := schemeAPI.(repoLister)
|
||||
if !ok {
|
||||
return nil, errs.ErrNotImplemented
|
||||
}
|
||||
return rl.RepoList(ctx, hostname, opts...)
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package regclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
func (rc *RegClient) schemeGet(scheme string) (scheme.API, error) {
|
||||
s, ok := rc.schemes[scheme]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: unknown scheme \"%s\"", errs.ErrNotImplemented, scheme)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Close is used to free resources associated with a reference.
|
||||
// With ocidir, this may trigger a garbage collection process.
|
||||
func (rc *RegClient) Close(ctx context.Context, r ref.Ref) error {
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// verify Closer api is defined, noop if missing
|
||||
sc, ok := schemeAPI.(scheme.Closer)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return sc.Close(ctx, r)
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
package ocidir
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/types/blob"
|
||||
"github.com/regclient/regclient/types/descriptor"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
// BlobDelete removes a blob from the repository.
|
||||
// This method does not verify that blobs are unused.
|
||||
// Calling the [OCIDir.Close] method to trigger the garbage collection is preferred.
|
||||
func (o *OCIDir) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error {
|
||||
err := d.Digest.Validate()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to validate digest %s: %w", d.Digest.String(), err)
|
||||
}
|
||||
file := path.Join(r.Path, "blobs", d.Digest.Algorithm().String(), d.Digest.Encoded())
|
||||
return os.Remove(file)
|
||||
}
|
||||
|
||||
// BlobGet retrieves a blob, returning a reader
|
||||
func (o *OCIDir) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
|
||||
err := d.Digest.Validate()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to validate digest %s: %w", d.Digest.String(), err)
|
||||
}
|
||||
file := path.Join(r.Path, "blobs", d.Digest.Algorithm().String(), d.Digest.Encoded())
|
||||
//#nosec G304 users should validate references they attempt to open
|
||||
fd, err := os.Open(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.Size <= 0 {
|
||||
fi, err := fd.Stat()
|
||||
if err != nil {
|
||||
_ = fd.Close()
|
||||
return nil, err
|
||||
}
|
||||
d.Size = fi.Size()
|
||||
}
|
||||
br := blob.NewReader(
|
||||
blob.WithRef(r),
|
||||
blob.WithReader(fd),
|
||||
blob.WithDesc(d),
|
||||
)
|
||||
o.slog.Debug("retrieved blob",
|
||||
slog.String("ref", r.CommonName()),
|
||||
slog.String("file", file))
|
||||
return br, nil
|
||||
}
|
||||
|
||||
// BlobHead verifies the existence of a blob, the reader contains the headers but no body to read
|
||||
func (o *OCIDir) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
|
||||
err := d.Digest.Validate()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to validate digest %s: %w", d.Digest.String(), err)
|
||||
}
|
||||
file := path.Join(r.Path, "blobs", d.Digest.Algorithm().String(), d.Digest.Encoded())
|
||||
//#nosec G304 users should validate references they attempt to open
|
||||
fd, err := os.Open(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer fd.Close()
|
||||
if d.Size <= 0 {
|
||||
fi, err := fd.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.Size = fi.Size()
|
||||
}
|
||||
br := blob.NewReader(
|
||||
blob.WithRef(r),
|
||||
blob.WithDesc(d),
|
||||
)
|
||||
return br, nil
|
||||
}
|
||||
|
||||
// BlobMount attempts to perform a server side copy of the blob
|
||||
func (o *OCIDir) BlobMount(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor) error {
|
||||
return errs.ErrUnsupported
|
||||
}
|
||||
|
||||
// BlobPut sends a blob to the repository, returns the digest and size when successful
|
||||
func (o *OCIDir) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error) {
|
||||
t := o.throttleGet(r, false)
|
||||
done, err := t.Acquire(ctx, reqmeta.Data{Kind: reqmeta.Blob, Size: d.Size})
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
defer done()
|
||||
|
||||
err = o.initIndex(r, false)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
digester := d.DigestAlgo().Digester()
|
||||
rdr = io.TeeReader(rdr, digester.Hash())
|
||||
// write the blob to a tmp file
|
||||
dir := path.Join(r.Path, "blobs", d.DigestAlgo().String())
|
||||
tmpPattern := "*.tmp"
|
||||
//#nosec G301 defer to user umask settings
|
||||
err = os.MkdirAll(dir, 0o777)
|
||||
if err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return d, fmt.Errorf("failed creating %s: %w", dir, err)
|
||||
}
|
||||
tmpFile, err := os.CreateTemp(dir, tmpPattern)
|
||||
if err != nil {
|
||||
return d, fmt.Errorf("failed creating blob tmp file: %w", err)
|
||||
}
|
||||
fi, err := tmpFile.Stat()
|
||||
if err != nil {
|
||||
return d, fmt.Errorf("failed to stat blob tmpfile: %w", err)
|
||||
}
|
||||
tmpName := fi.Name()
|
||||
i, err := io.Copy(tmpFile, rdr)
|
||||
errC := tmpFile.Close()
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
if errC != nil {
|
||||
return d, errC
|
||||
}
|
||||
// validate result matches descriptor, or update descriptor if it wasn't defined
|
||||
if d.Digest.Validate() != nil {
|
||||
d.Digest = digester.Digest()
|
||||
} else if d.Digest != digester.Digest() {
|
||||
return d, fmt.Errorf("unexpected digest, expected %s, computed %s", d.Digest, digester.Digest())
|
||||
}
|
||||
if d.Size <= 0 {
|
||||
d.Size = i
|
||||
} else if i != d.Size {
|
||||
return d, fmt.Errorf("unexpected blob length, expected %d, received %d", d.Size, i)
|
||||
}
|
||||
file := path.Join(r.Path, "blobs", d.Digest.Algorithm().String(), d.Digest.Encoded())
|
||||
//#nosec G703 inputs are user controlled
|
||||
err = os.Rename(path.Join(dir, tmpName), file)
|
||||
if err != nil {
|
||||
return d, fmt.Errorf("failed to write blob (rename tmp file %s to %s): %w", path.Join(dir, tmpName), file, err)
|
||||
}
|
||||
o.slog.Debug("pushed blob",
|
||||
slog.String("ref", r.CommonName()),
|
||||
slog.String("file", file))
|
||||
|
||||
o.mu.Lock()
|
||||
o.refMod(r)
|
||||
o.mu.Unlock()
|
||||
return d, nil
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
package ocidir
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/regclient/regclient/types/manifest"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
// Close triggers a garbage collection if the underlying path has been modified
|
||||
func (o *OCIDir) Close(ctx context.Context, r ref.Ref) error {
|
||||
if !o.gc {
|
||||
return nil
|
||||
}
|
||||
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if gc, ok := o.modRefs[r.Path]; !ok || !gc.mod || gc.locks > 0 {
|
||||
// unmodified or locked, skip gc
|
||||
return nil
|
||||
}
|
||||
|
||||
// perform GC
|
||||
o.slog.Debug("running GC",
|
||||
slog.String("ref", r.CommonName()))
|
||||
dl := map[string]bool{}
|
||||
// recurse through index, manifests, and blob lists, generating a digest list
|
||||
index, err := o.readIndex(r, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
im, err := manifest.New(manifest.WithOrig(index))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = o.closeProcManifest(ctx, r, im, &dl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// go through filesystem digest list, removing entries not seen in recursive pass
|
||||
blobsPath := path.Join(r.Path, "blobs")
|
||||
blobDirs, err := os.ReadDir(blobsPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, blobDir := range blobDirs {
|
||||
if !blobDir.IsDir() {
|
||||
// should this warn or delete unexpected files in the blobs folder?
|
||||
continue
|
||||
}
|
||||
digestFiles, err := os.ReadDir(path.Join(blobsPath, blobDir.Name()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, digestFile := range digestFiles {
|
||||
digest := fmt.Sprintf("%s:%s", blobDir.Name(), digestFile.Name())
|
||||
if !dl[digest] {
|
||||
o.slog.Debug("ocidir garbage collect",
|
||||
slog.String("digest", digest))
|
||||
// delete
|
||||
err = os.Remove(path.Join(blobsPath, blobDir.Name(), digestFile.Name()))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete %s: %w", path.Join(blobsPath, blobDir.Name(), digestFile.Name()), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
delete(o.modRefs, r.Path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *OCIDir) closeProcManifest(ctx context.Context, r ref.Ref, m manifest.Manifest, dl *map[string]bool) error {
|
||||
if mi, ok := m.(manifest.Indexer); ok {
|
||||
// go through manifest list, updating dl, and recursively processing nested manifests
|
||||
ml, err := mi.GetManifestList()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, cur := range ml {
|
||||
cr := r.SetDigest(cur.Digest.String())
|
||||
(*dl)[cr.Digest] = true
|
||||
cm, err := o.manifestGet(ctx, cr)
|
||||
if err != nil {
|
||||
// ignore errors in case a manifest has been deleted or sparse copy
|
||||
o.slog.Debug("could not retrieve manifest",
|
||||
slog.String("ref", cr.CommonName()),
|
||||
slog.String("err", err.Error()))
|
||||
continue
|
||||
}
|
||||
err = o.closeProcManifest(ctx, cr, cm, dl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if mi, ok := m.(manifest.Imager); ok {
|
||||
// get config from manifest if it exists
|
||||
cd, err := mi.GetConfig()
|
||||
if err == nil {
|
||||
(*dl)[cd.Digest.String()] = true
|
||||
}
|
||||
// finally add all layers to digest list
|
||||
layers, err := mi.GetLayers()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, layer := range layers {
|
||||
(*dl)[layer.Digest.String()] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+306
@@ -0,0 +1,306 @@
|
||||
package ocidir
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
|
||||
// crypto libraries included for go-digest
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/manifest"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
// ManifestDelete removes a manifest, including all tags that point to that manifest
|
||||
func (o *OCIDir) ManifestDelete(ctx context.Context, r ref.Ref, opts ...scheme.ManifestOpts) error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
|
||||
if r.Digest == "" {
|
||||
return fmt.Errorf("digest required to delete manifest, reference %s%.0w", r.CommonName(), errs.ErrMissingDigest)
|
||||
}
|
||||
|
||||
mc := scheme.ManifestConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&mc)
|
||||
}
|
||||
|
||||
// always check for refers with ocidir
|
||||
if mc.Manifest == nil {
|
||||
m, err := o.manifestGet(ctx, r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to pull manifest for refers: %w", err)
|
||||
}
|
||||
mc.Manifest = m
|
||||
}
|
||||
if mc.Manifest != nil {
|
||||
if ms, ok := mc.Manifest.(manifest.Subjecter); ok {
|
||||
sDesc, err := ms.GetSubject()
|
||||
if err == nil && sDesc != nil && sDesc.Digest != "" {
|
||||
// attempt to delete the referrer, but ignore if the referrer entry wasn't found
|
||||
err = o.referrerDelete(ctx, r, mc.Manifest)
|
||||
if err != nil && !errors.Is(err, errs.ErrNotFound) && !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// get index
|
||||
changed := false
|
||||
index, err := o.readIndex(r, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read index: %w", err)
|
||||
}
|
||||
for i := len(index.Manifests) - 1; i >= 0; i-- {
|
||||
// remove matching entry from index
|
||||
if r.Digest != "" && index.Manifests[i].Digest.String() == r.Digest {
|
||||
changed = true
|
||||
index.Manifests = slices.Delete(index.Manifests, i, i+1)
|
||||
}
|
||||
}
|
||||
// push manifest back out
|
||||
if changed {
|
||||
err = o.writeIndex(r, index, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write index: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// delete from filesystem like a registry would do
|
||||
d := digest.Digest(r.Digest)
|
||||
file := path.Join(r.Path, "blobs", d.Algorithm().String(), d.Encoded())
|
||||
err = os.Remove(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete manifest: %w", err)
|
||||
}
|
||||
o.refMod(r)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ManifestGet retrieves a manifest from a repository
|
||||
func (o *OCIDir) ManifestGet(ctx context.Context, r ref.Ref) (manifest.Manifest, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.manifestGet(ctx, r)
|
||||
}
|
||||
|
||||
func (o *OCIDir) manifestGet(_ context.Context, r ref.Ref) (manifest.Manifest, error) {
|
||||
index, err := o.readIndex(r, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read oci index: %w", err)
|
||||
}
|
||||
if r.Digest == "" && r.Tag == "" {
|
||||
r = r.SetTag("latest")
|
||||
}
|
||||
desc, err := indexGet(index, r)
|
||||
if err != nil {
|
||||
if r.Digest != "" {
|
||||
desc.Digest = digest.Digest(r.Digest)
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if desc.Digest == "" {
|
||||
return nil, errs.ErrNotFound
|
||||
}
|
||||
if err = desc.Digest.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid digest in index: %s: %w", string(desc.Digest), err)
|
||||
}
|
||||
file := path.Join(r.Path, "blobs", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
|
||||
//#nosec G304 users should validate references they attempt to open
|
||||
fd, err := os.Open(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open manifest: %w", err)
|
||||
}
|
||||
defer fd.Close()
|
||||
mb, err := io.ReadAll(fd)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read manifest: %w", err)
|
||||
}
|
||||
if desc.Size == 0 {
|
||||
desc.Size = int64(len(mb))
|
||||
}
|
||||
o.slog.Debug("retrieved manifest",
|
||||
slog.String("ref", r.CommonName()),
|
||||
slog.String("file", file))
|
||||
return manifest.New(
|
||||
manifest.WithRef(r),
|
||||
manifest.WithDesc(desc),
|
||||
manifest.WithRaw(mb),
|
||||
)
|
||||
}
|
||||
|
||||
// ManifestHead gets metadata about the manifest (existence, digest, mediatype, size)
|
||||
func (o *OCIDir) ManifestHead(ctx context.Context, r ref.Ref) (manifest.Manifest, error) {
|
||||
index, err := o.readIndex(r, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read oci index: %w", err)
|
||||
}
|
||||
if r.Digest == "" && r.Tag == "" {
|
||||
r = r.SetTag("latest")
|
||||
}
|
||||
desc, err := indexGet(index, r)
|
||||
if err != nil {
|
||||
if r.Digest != "" {
|
||||
desc.Digest = digest.Digest(r.Digest)
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if desc.Digest == "" {
|
||||
return nil, errs.ErrNotFound
|
||||
}
|
||||
if err = desc.Digest.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid digest in index: %s: %w", string(desc.Digest), err)
|
||||
}
|
||||
// verify underlying file exists
|
||||
file := path.Join(r.Path, "blobs", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
|
||||
fi, err := os.Stat(file)
|
||||
if err != nil || fi.IsDir() {
|
||||
return nil, errs.ErrNotFound
|
||||
}
|
||||
// if missing, set media type on desc
|
||||
if desc.MediaType == "" {
|
||||
//#nosec G304 users should validate references they attempt to open
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mt := struct {
|
||||
MediaType string `json:"mediaType,omitempty"`
|
||||
SchemaVersion int `json:"schemaVersion,omitempty"`
|
||||
Signatures []any `json:"signatures,omitempty"`
|
||||
}{}
|
||||
err = json.Unmarshal(raw, &mt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if mt.MediaType != "" {
|
||||
desc.MediaType = mt.MediaType
|
||||
desc.Size = int64(len(raw))
|
||||
} else if mt.SchemaVersion == 1 && len(mt.Signatures) > 0 {
|
||||
desc.MediaType = mediatype.Docker1ManifestSigned
|
||||
} else if mt.SchemaVersion == 1 {
|
||||
desc.MediaType = mediatype.Docker1Manifest
|
||||
desc.Size = int64(len(raw))
|
||||
}
|
||||
}
|
||||
return manifest.New(
|
||||
manifest.WithRef(r),
|
||||
manifest.WithDesc(desc),
|
||||
)
|
||||
}
|
||||
|
||||
// ManifestPut sends a manifest to the repository
|
||||
func (o *OCIDir) ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...scheme.ManifestOpts) error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.manifestPut(ctx, r, m, opts...)
|
||||
}
|
||||
|
||||
func (o *OCIDir) manifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...scheme.ManifestOpts) error {
|
||||
config := scheme.ManifestConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&config)
|
||||
}
|
||||
if !config.Child && r.Digest == "" && r.Tag == "" {
|
||||
r = r.SetTag("latest")
|
||||
}
|
||||
err := o.initIndex(r, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
desc := m.GetDescriptor()
|
||||
if err = desc.Digest.Validate(); err != nil {
|
||||
return fmt.Errorf("invalid digest for manifest: %s: %w", string(desc.Digest), err)
|
||||
}
|
||||
b, err := m.RawBody()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not serialize manifest: %w", err)
|
||||
}
|
||||
if r.Digest != "" && desc.Digest.String() != r.Digest {
|
||||
// Digest algorithm may have changed, try recreating the manifest with the provided ref.
|
||||
// This will fail if the ref digest does not match the manifest.
|
||||
m, err = manifest.New(manifest.WithRef(r), manifest.WithRaw(b))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rebuilding manifest with ref \"%s\": %w", r.CommonName(), err)
|
||||
}
|
||||
}
|
||||
desc.Annotations = map[string]string{}
|
||||
if r.Tag != "" {
|
||||
desc.Annotations[types.AnnotationRefName] = r.Tag
|
||||
}
|
||||
// TODO: if the manifest contains a subject, option to add descriptor details and include entry in the index.json, config.child=false
|
||||
// create manifest CAS file
|
||||
dir := path.Join(r.Path, "blobs", desc.Digest.Algorithm().String())
|
||||
//#nosec G301 defer to user umask settings
|
||||
err = os.MkdirAll(dir, 0o777)
|
||||
if err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return fmt.Errorf("failed creating %s: %w", dir, err)
|
||||
}
|
||||
// write to a tmp file, rename after validating
|
||||
tmpFile, err := os.CreateTemp(dir, desc.Digest.Encoded()+".*.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create manifest tmpfile: %w", err)
|
||||
}
|
||||
fi, err := tmpFile.Stat()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stat manifest tmpfile: %w", err)
|
||||
}
|
||||
tmpName := fi.Name()
|
||||
_, err = tmpFile.Write(b)
|
||||
errC := tmpFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write manifest tmpfile: %w", err)
|
||||
}
|
||||
if errC != nil {
|
||||
return fmt.Errorf("failed to close manifest tmpfile: %w", errC)
|
||||
}
|
||||
file := path.Join(dir, desc.Digest.Encoded())
|
||||
//#nosec G703 inputs are user controlled
|
||||
err = os.Rename(path.Join(dir, tmpName), file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write manifest (rename tmpfile): %w", err)
|
||||
}
|
||||
|
||||
// verify/update index
|
||||
err = o.updateIndex(r, desc, config.Child, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.refMod(r)
|
||||
o.slog.Debug("pushed manifest",
|
||||
slog.String("ref", r.CommonName()),
|
||||
slog.String("file", file))
|
||||
|
||||
// update referrers if defined on this manifest
|
||||
if ms, ok := m.(manifest.Subjecter); ok {
|
||||
mDesc, err := ms.GetSubject()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mDesc != nil && mDesc.Digest != "" {
|
||||
err = o.referrerPut(ctx, r, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+419
@@ -0,0 +1,419 @@
|
||||
// Package ocidir implements the OCI Image Layout scheme with a directory (not packed in a tar)
|
||||
package ocidir
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/regclient/regclient/internal/pqueue"
|
||||
"github.com/regclient/regclient/internal/reproducible"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/types"
|
||||
"github.com/regclient/regclient/types/descriptor"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
v1 "github.com/regclient/regclient/types/oci/v1"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
const (
|
||||
imageLayoutFile = "oci-layout"
|
||||
defThrottle = 3
|
||||
)
|
||||
|
||||
// OCIDir is used for accessing OCI Image Layouts defined as a directory
|
||||
type OCIDir struct {
|
||||
slog *slog.Logger
|
||||
gc bool
|
||||
modRefs map[string]*ociGC
|
||||
throttle map[string]*pqueue.Queue[reqmeta.Data]
|
||||
throttleDef int
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
type ociGC struct {
|
||||
mod bool
|
||||
locks int
|
||||
}
|
||||
|
||||
type ociConf struct {
|
||||
gc bool
|
||||
slog *slog.Logger
|
||||
throttle int
|
||||
}
|
||||
|
||||
// Opts are used for passing options to ocidir
|
||||
type Opts func(*ociConf)
|
||||
|
||||
// New creates a new OCIDir with options
|
||||
func New(opts ...Opts) *OCIDir {
|
||||
conf := ociConf{
|
||||
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
|
||||
gc: true,
|
||||
throttle: defThrottle,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(&conf)
|
||||
}
|
||||
return &OCIDir{
|
||||
slog: conf.slog,
|
||||
gc: conf.gc,
|
||||
modRefs: map[string]*ociGC{},
|
||||
throttle: map[string]*pqueue.Queue[reqmeta.Data]{},
|
||||
throttleDef: conf.throttle,
|
||||
}
|
||||
}
|
||||
|
||||
// WithGC configures the garbage collection setting
|
||||
// This defaults to enabled
|
||||
func WithGC(gc bool) Opts {
|
||||
return func(c *ociConf) {
|
||||
c.gc = gc
|
||||
}
|
||||
}
|
||||
|
||||
// WithSlog provides a slog logger.
|
||||
// By default logging is disabled.
|
||||
func WithSlog(slog *slog.Logger) Opts {
|
||||
return func(c *ociConf) {
|
||||
c.slog = slog
|
||||
}
|
||||
}
|
||||
|
||||
// WithThrottle provides a number of concurrent write actions (blob/manifest put)
|
||||
func WithThrottle(count int) Opts {
|
||||
return func(c *ociConf) {
|
||||
c.throttle = count
|
||||
}
|
||||
}
|
||||
|
||||
// GCLock is used to prevent GC on a ref
|
||||
func (o *OCIDir) GCLock(r ref.Ref) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if gc, ok := o.modRefs[r.Path]; ok && gc != nil {
|
||||
gc.locks++
|
||||
} else {
|
||||
o.modRefs[r.Path] = &ociGC{locks: 1}
|
||||
}
|
||||
}
|
||||
|
||||
// GCUnlock removes a hold on GC of a ref, this must be done before the ref is closed
|
||||
func (o *OCIDir) GCUnlock(r ref.Ref) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if gc, ok := o.modRefs[r.Path]; ok && gc != nil && gc.locks > 0 {
|
||||
gc.locks--
|
||||
}
|
||||
}
|
||||
|
||||
// Throttle is used to limit concurrency
|
||||
func (o *OCIDir) Throttle(r ref.Ref, put bool) []*pqueue.Queue[reqmeta.Data] {
|
||||
tList := []*pqueue.Queue[reqmeta.Data]{}
|
||||
// throttle only applies to put requests
|
||||
if !put || o.throttleDef <= 0 {
|
||||
return tList
|
||||
}
|
||||
return []*pqueue.Queue[reqmeta.Data]{o.throttleGet(r, false)}
|
||||
}
|
||||
|
||||
func (o *OCIDir) throttleGet(r ref.Ref, locked bool) *pqueue.Queue[reqmeta.Data] {
|
||||
if !locked {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
}
|
||||
if t, ok := o.throttle[r.Path]; ok {
|
||||
return t
|
||||
}
|
||||
// init a new throttle
|
||||
o.throttle[r.Path] = pqueue.New(pqueue.Opts[reqmeta.Data]{Max: o.throttleDef})
|
||||
return o.throttle[r.Path]
|
||||
}
|
||||
|
||||
func (o *OCIDir) initIndex(r ref.Ref, locked bool) error {
|
||||
if !locked {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
}
|
||||
layoutFile := path.Join(r.Path, imageLayoutFile)
|
||||
_, err := os.Stat(layoutFile)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
//#nosec G301 defer to user umask settings
|
||||
err = os.MkdirAll(r.Path, 0o777)
|
||||
if err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return fmt.Errorf("failed creating %s: %w", r.Path, err)
|
||||
}
|
||||
// create/replace oci-layout file
|
||||
layout := v1.ImageLayout{
|
||||
Version: "1.0.0",
|
||||
}
|
||||
lb, err := json.Marshal(layout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot marshal layout: %w", err)
|
||||
}
|
||||
//#nosec G304 users should validate references they attempt to open
|
||||
lfh, err := os.Create(layoutFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create %s: %w", imageLayoutFile, err)
|
||||
}
|
||||
defer lfh.Close()
|
||||
_, err = lfh.Write(lb)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot write %s: %w", imageLayoutFile, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *OCIDir) readIndex(r ref.Ref, locked bool) (v1.Index, error) {
|
||||
if !locked {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
}
|
||||
// validate dir
|
||||
index := v1.Index{}
|
||||
err := o.valid(r.Path, true)
|
||||
if err != nil {
|
||||
return index, err
|
||||
}
|
||||
indexFile := path.Join(r.Path, "index.json")
|
||||
//#nosec G304 users should validate references they attempt to open
|
||||
fh, err := os.Open(indexFile)
|
||||
if err != nil {
|
||||
return index, fmt.Errorf("%s cannot be open: %w", indexFile, err)
|
||||
}
|
||||
defer fh.Close()
|
||||
ib, err := io.ReadAll(fh)
|
||||
if err != nil {
|
||||
return index, fmt.Errorf("%s cannot be read: %w", indexFile, err)
|
||||
}
|
||||
err = json.Unmarshal(ib, &index)
|
||||
if err != nil {
|
||||
return index, fmt.Errorf("%s cannot be parsed: %w", indexFile, err)
|
||||
}
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func (o *OCIDir) updateIndex(r ref.Ref, d descriptor.Descriptor, child bool, locked bool) error {
|
||||
if !locked {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
}
|
||||
indexChanged := false
|
||||
index, err := o.readIndex(r, true)
|
||||
if err != nil {
|
||||
index = indexCreate()
|
||||
indexChanged = true
|
||||
}
|
||||
if !child {
|
||||
err := indexSet(&index, r, d)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update index: %w", err)
|
||||
}
|
||||
indexChanged = true
|
||||
}
|
||||
if indexChanged {
|
||||
err = o.writeIndex(r, index, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write index: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *OCIDir) writeIndex(r ref.Ref, i v1.Index, locked bool) error {
|
||||
if !locked {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
}
|
||||
//#nosec G301 defer to user umask settings
|
||||
err := os.MkdirAll(r.Path, 0o777)
|
||||
if err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return fmt.Errorf("failed creating %s: %w", r.Path, err)
|
||||
}
|
||||
// create/replace oci-layout file
|
||||
layout := v1.ImageLayout{
|
||||
Version: "1.0.0",
|
||||
}
|
||||
lb, err := json.Marshal(layout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot marshal layout: %w", err)
|
||||
}
|
||||
lfh, err := os.Create(path.Join(r.Path, imageLayoutFile))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create %s: %w", imageLayoutFile, err)
|
||||
}
|
||||
defer lfh.Close()
|
||||
_, err = lfh.Write(lb)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot write %s: %w", imageLayoutFile, err)
|
||||
}
|
||||
// create/replace index.json file
|
||||
tmpFile, err := os.CreateTemp(r.Path, "index.json.*.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create index tmpfile: %w", err)
|
||||
}
|
||||
fi, err := tmpFile.Stat()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stat index tmpfile: %w", err)
|
||||
}
|
||||
tmpName := fi.Name()
|
||||
b, err := json.Marshal(i)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot marshal index: %w", err)
|
||||
}
|
||||
_, err = tmpFile.Write(b)
|
||||
errC := tmpFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot write index: %w", err)
|
||||
}
|
||||
if errC != nil {
|
||||
return fmt.Errorf("cannot close index: %w", errC)
|
||||
}
|
||||
indexFile := path.Join(r.Path, "index.json")
|
||||
//#nosec G703 inputs are user controlled
|
||||
err = os.Rename(path.Join(r.Path, tmpName), indexFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot rename tmpfile to index: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// func valid (dir) (error) // check for `oci-layout` file and `index.json` for read
|
||||
func (o *OCIDir) valid(dir string, locked bool) error {
|
||||
if !locked {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
}
|
||||
layout := v1.ImageLayout{}
|
||||
reqVer := "1.0.0"
|
||||
//#nosec G304 users should validate references they attempt to open
|
||||
fh, err := os.Open(path.Join(dir, imageLayoutFile))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s cannot be open: %w", imageLayoutFile, err)
|
||||
}
|
||||
defer fh.Close()
|
||||
lb, err := io.ReadAll(fh)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s cannot be read: %w", imageLayoutFile, err)
|
||||
}
|
||||
err = json.Unmarshal(lb, &layout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s cannot be parsed: %w", imageLayoutFile, err)
|
||||
}
|
||||
if layout.Version != reqVer {
|
||||
return fmt.Errorf("unsupported oci layout version, expected %s, received %s", reqVer, layout.Version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *OCIDir) refMod(r ref.Ref) {
|
||||
if gc, ok := o.modRefs[r.Path]; ok && gc != nil {
|
||||
gc.mod = true
|
||||
} else {
|
||||
o.modRefs[r.Path] = &ociGC{mod: true}
|
||||
}
|
||||
}
|
||||
|
||||
func indexCreate() v1.Index {
|
||||
i := v1.Index{
|
||||
Versioned: v1.IndexSchemaVersion,
|
||||
MediaType: mediatype.OCI1ManifestList,
|
||||
Manifests: []descriptor.Descriptor{},
|
||||
Annotations: map[string]string{},
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func indexGet(index v1.Index, r ref.Ref) (descriptor.Descriptor, error) {
|
||||
if r.Digest == "" && r.Tag == "" {
|
||||
r = r.SetTag("latest")
|
||||
}
|
||||
if r.Digest != "" {
|
||||
for _, im := range index.Manifests {
|
||||
if im.Digest.String() == r.Digest {
|
||||
return im, nil
|
||||
}
|
||||
}
|
||||
} else if r.Tag != "" {
|
||||
for _, im := range index.Manifests {
|
||||
if name, ok := im.Annotations[types.AnnotationRefName]; ok && name == r.Tag {
|
||||
return im, nil
|
||||
}
|
||||
}
|
||||
// fall back to support full image name in annotation
|
||||
for _, im := range index.Manifests {
|
||||
if name, ok := im.Annotations[types.AnnotationRefName]; ok && strings.HasSuffix(name, ":"+r.Tag) {
|
||||
return im, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return descriptor.Descriptor{}, errs.ErrNotFound
|
||||
}
|
||||
|
||||
func indexSet(index *v1.Index, r ref.Ref, d descriptor.Descriptor) error {
|
||||
if index == nil {
|
||||
return fmt.Errorf("index is nil")
|
||||
}
|
||||
if d.Annotations == nil {
|
||||
d.Annotations = map[string]string{}
|
||||
}
|
||||
if r.Tag != "" {
|
||||
d.Annotations[types.AnnotationRefName] = r.Tag
|
||||
}
|
||||
if _, ok := d.Annotations[types.AnnotationCreated]; !ok {
|
||||
d.Annotations[types.AnnotationCreated] = reproducible.TimeNow().Format(time.RFC3339)
|
||||
}
|
||||
if index.Manifests == nil {
|
||||
index.Manifests = []descriptor.Descriptor{}
|
||||
}
|
||||
pos := -1
|
||||
// search for existing
|
||||
for i := range index.Manifests {
|
||||
var name string
|
||||
if index.Manifests[i].Annotations != nil {
|
||||
name = index.Manifests[i].Annotations[types.AnnotationRefName]
|
||||
}
|
||||
if r.Tag == "" && name == "" && index.Manifests[i].Digest == d.Digest {
|
||||
// untagged entry already exists, no need to replace it
|
||||
pos = i
|
||||
break
|
||||
} else if r.Tag != "" && name == r.Tag {
|
||||
if index.Manifests[i].Digest != d.Digest {
|
||||
// replace the tag with a new digest/descriptor
|
||||
index.Manifests[i] = d
|
||||
}
|
||||
pos = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if pos >= 0 {
|
||||
// existing entry was replaced, remove any dup entries
|
||||
for i := len(index.Manifests) - 1; i > pos; i-- {
|
||||
var name string
|
||||
if index.Manifests[i].Annotations != nil {
|
||||
name = index.Manifests[i].Annotations[types.AnnotationRefName]
|
||||
}
|
||||
// prune untagged entries with a matching digest if we didn't push a tag
|
||||
// or entries with a matching tag
|
||||
if (r.Tag == "" && name == "" && index.Manifests[i].Digest == d.Digest) || (r.Tag != "" && name == r.Tag) {
|
||||
index.Manifests = slices.Delete(index.Manifests, i, i+1)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// existing entry to replace was not found, add the descriptor
|
||||
index.Manifests = append(index.Manifests, d)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
//go:build !wasm
|
||||
|
||||
package ocidir
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/regclient/regclient/internal/sloghandle"
|
||||
)
|
||||
|
||||
// WithLog provides a logrus logger.
|
||||
// By default logging is disabled.
|
||||
func WithLog(log *logrus.Logger) Opts {
|
||||
return func(c *ociConf) {
|
||||
c.slog = slog.New(sloghandle.Logrus(log))
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package ocidir
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/regclient/regclient/types/ping"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
// Ping for an ocidir verifies access to read the path.
|
||||
func (o *OCIDir) Ping(ctx context.Context, r ref.Ref) (ping.Result, error) {
|
||||
ret := ping.Result{}
|
||||
fd, err := os.Open(r.Path)
|
||||
if err != nil {
|
||||
return ret, err
|
||||
}
|
||||
defer fd.Close()
|
||||
fi, err := fd.Stat()
|
||||
if err != nil {
|
||||
return ret, err
|
||||
}
|
||||
ret.Stat = fi
|
||||
if !fi.IsDir() {
|
||||
return ret, fmt.Errorf("failed to access %s: not a directory", r.Path)
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
package ocidir
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/manifest"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
v1 "github.com/regclient/regclient/types/oci/v1"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/referrer"
|
||||
)
|
||||
|
||||
// ReferrerList returns a list of referrers to a given reference.
|
||||
// The reference must include the digest. Use [regclient.ReferrerList] to resolve the platform or tag.
|
||||
func (o *OCIDir) ReferrerList(ctx context.Context, r ref.Ref, opts ...scheme.ReferrerOpts) (referrer.ReferrerList, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.referrerList(ctx, r, opts...)
|
||||
}
|
||||
|
||||
func (o *OCIDir) referrerList(ctx context.Context, rSubject ref.Ref, opts ...scheme.ReferrerOpts) (referrer.ReferrerList, error) {
|
||||
config := scheme.ReferrerConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&config)
|
||||
}
|
||||
var r ref.Ref
|
||||
if config.SrcRepo.IsSet() {
|
||||
r = config.SrcRepo.SetDigest(rSubject.Digest)
|
||||
} else {
|
||||
r = rSubject.SetDigest(rSubject.Digest)
|
||||
}
|
||||
rl := referrer.New(rSubject)
|
||||
if rSubject.Digest == "" {
|
||||
return rl, fmt.Errorf("digest required to query referrers %s", rSubject.CommonName())
|
||||
}
|
||||
|
||||
// TODO: Add fast path with private field in index.json indicating referrers are tracked using an annotation.
|
||||
// This can be implemented with backward compatible support by also pushing individual manifests or the fallback tag.
|
||||
|
||||
// TODO: Upstream OCI specification of how to track subject/referrers in an OCI Layout is still pending.
|
||||
|
||||
// optional slow query of all manifests for ORAS support
|
||||
if config.SlowSearch {
|
||||
idx, err := o.readIndex(r, true)
|
||||
if err != nil {
|
||||
return rl, err
|
||||
}
|
||||
for _, desc := range idx.Manifests {
|
||||
// pull the manifest
|
||||
rCur := r.SetDigest(desc.Digest.String())
|
||||
m, err := o.manifestGet(ctx, rCur)
|
||||
if err != nil {
|
||||
// skip all errors
|
||||
continue
|
||||
}
|
||||
// check for the subject
|
||||
mSubj, ok := m.(manifest.Subjecter)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
subj, err := mSubj.GetSubject()
|
||||
if err != nil || subj == nil || subj.Digest.String() != rSubject.Digest {
|
||||
continue
|
||||
}
|
||||
// extract/track tag
|
||||
if name, ok := desc.Annotations[types.AnnotationRefName]; ok && !strings.ContainsAny(name, ":/") {
|
||||
rl.Tags = append(rl.Tags, name)
|
||||
}
|
||||
// if matching, add to rl
|
||||
err = rl.Add(m)
|
||||
if err != nil {
|
||||
return rl, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// try to request the fallback tag
|
||||
rlFallback, err := o.referrerListFallback(ctx, r)
|
||||
if err == nil {
|
||||
err = rl.Merge(rlFallback)
|
||||
if err != nil {
|
||||
return rl, fmt.Errorf("failed to merge referrers from fallback tag: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// update referrer list
|
||||
rl.Subject = rSubject
|
||||
if config.SrcRepo.IsSet() {
|
||||
rl.Source = config.SrcRepo
|
||||
}
|
||||
rl = scheme.ReferrerFilter(config, rl)
|
||||
|
||||
return rl, nil
|
||||
}
|
||||
|
||||
func (o *OCIDir) referrerListFallback(ctx context.Context, r ref.Ref) (referrer.ReferrerList, error) {
|
||||
rl := referrer.New(r)
|
||||
rlTag, err := referrer.FallbackTag(r)
|
||||
if err != nil {
|
||||
return rl, err
|
||||
}
|
||||
m, err := o.manifestGet(ctx, rlTag)
|
||||
if err != nil {
|
||||
if errors.Is(err, errs.ErrNotFound) {
|
||||
// empty list, initialize a new manifest
|
||||
rl.Manifest, err = manifest.New(manifest.WithOrig(v1.Index{
|
||||
Versioned: v1.IndexSchemaVersion,
|
||||
MediaType: mediatype.OCI1ManifestList,
|
||||
}))
|
||||
if err != nil {
|
||||
return rl, err
|
||||
}
|
||||
return rl, nil
|
||||
}
|
||||
return rl, err
|
||||
}
|
||||
ociML, ok := m.GetOrig().(v1.Index)
|
||||
if !ok {
|
||||
return rl, fmt.Errorf("manifest is not an OCI index: %s", rlTag.CommonName())
|
||||
}
|
||||
// update referrer list
|
||||
rl.Manifest = m
|
||||
rl.Descriptors = ociML.Manifests
|
||||
rl.Annotations = ociML.Annotations
|
||||
rl.Tags = append(rl.Tags, rlTag.Tag)
|
||||
return rl, nil
|
||||
}
|
||||
|
||||
// referrerDelete deletes a referrer associated with a manifest
|
||||
func (o *OCIDir) referrerDelete(ctx context.Context, r ref.Ref, m manifest.Manifest) error {
|
||||
// get refers field
|
||||
mSubject, ok := m.(manifest.Subjecter)
|
||||
if !ok {
|
||||
return fmt.Errorf("manifest does not support subject: %w", errs.ErrUnsupportedMediaType)
|
||||
}
|
||||
subject, err := mSubject.GetSubject()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// validate/set subject descriptor
|
||||
if subject == nil || subject.Digest == "" {
|
||||
return fmt.Errorf("subject is not set%.0w", errs.ErrNotFound)
|
||||
}
|
||||
|
||||
// get descriptor for subject
|
||||
rSubject := r.SetDigest(subject.Digest.String())
|
||||
|
||||
// pull existing referrer list
|
||||
rl, err := o.referrerList(ctx, rSubject)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = rl.Delete(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// push updated referrer list by tag
|
||||
rlTag, err := referrer.FallbackTag(rSubject)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rl.IsEmpty() {
|
||||
err = o.tagDelete(ctx, rlTag)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
// if delete is not supported, fall back to pushing empty list
|
||||
}
|
||||
return o.manifestPut(ctx, rlTag, rl.Manifest)
|
||||
}
|
||||
|
||||
// referrerPut pushes a new referrer associated with a given reference
|
||||
func (o *OCIDir) referrerPut(ctx context.Context, r ref.Ref, m manifest.Manifest) error {
|
||||
// get subject field
|
||||
mSubject, ok := m.(manifest.Subjecter)
|
||||
if !ok {
|
||||
return fmt.Errorf("manifest does not support subject: %w", errs.ErrUnsupportedMediaType)
|
||||
}
|
||||
subject, err := mSubject.GetSubject()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// validate/set subject descriptor
|
||||
if subject == nil || subject.Digest == "" {
|
||||
return fmt.Errorf("subject is not set%.0w", errs.ErrNotFound)
|
||||
}
|
||||
|
||||
// get descriptor for subject
|
||||
rSubject := r.SetDigest(subject.Digest.String())
|
||||
|
||||
// pull existing referrer list
|
||||
rl, err := o.referrerList(ctx, rSubject)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = rl.Add(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: add support for pushing a fast path generated response
|
||||
|
||||
// TODO: allow fallback tag creation to be skipped, potentially via a config in the manifestPut
|
||||
|
||||
// push updated referrer list by tag
|
||||
rlTag, err := referrer.FallbackTag(rSubject)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return o.manifestPut(ctx, rlTag, rl.Manifest)
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
package ocidir
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/tag"
|
||||
)
|
||||
|
||||
// TagDelete removes a tag from the repository
|
||||
func (o *OCIDir) TagDelete(ctx context.Context, r ref.Ref) error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.tagDelete(ctx, r)
|
||||
}
|
||||
|
||||
func (o *OCIDir) tagDelete(_ context.Context, r ref.Ref) error {
|
||||
if r.Tag == "" {
|
||||
return errs.ErrMissingTag
|
||||
}
|
||||
// get index
|
||||
index, err := o.readIndex(r, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read index: %w", err)
|
||||
}
|
||||
changed := false
|
||||
for i, desc := range index.Manifests {
|
||||
if t, ok := desc.Annotations[types.AnnotationRefName]; ok && t == r.Tag {
|
||||
// remove matching entry from index
|
||||
index.Manifests = slices.Delete(index.Manifests, i, i+1)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return fmt.Errorf("failed deleting %s: %w", r.CommonName(), errs.ErrNotFound)
|
||||
}
|
||||
// push manifest back out
|
||||
err = o.writeIndex(r, index, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write index: %w", err)
|
||||
}
|
||||
o.refMod(r)
|
||||
return nil
|
||||
}
|
||||
|
||||
// TagList returns a list of tags from the repository
|
||||
func (o *OCIDir) TagList(ctx context.Context, r ref.Ref, opts ...scheme.TagOpts) (*tag.List, error) {
|
||||
// get index
|
||||
index, err := o.readIndex(r, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tl := []string{}
|
||||
for _, desc := range index.Manifests {
|
||||
if t, ok := desc.Annotations[types.AnnotationRefName]; ok {
|
||||
if i := strings.LastIndex(t, ":"); i >= 0 {
|
||||
t = t[i+1:]
|
||||
}
|
||||
if !slices.Contains(tl, t) {
|
||||
tl = append(tl, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(tl)
|
||||
ib, err := json.Marshal(index)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// return listing from index
|
||||
t, err := tag.New(
|
||||
tag.WithRaw(ib),
|
||||
tag.WithRef(r),
|
||||
tag.WithMT(mediatype.OCI1ManifestList),
|
||||
tag.WithLayoutIndex(index),
|
||||
tag.WithTags(tl),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
+680
@@ -0,0 +1,680 @@
|
||||
package reg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
// crypto libraries included for go-digest
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/internal/reghttp"
|
||||
"github.com/regclient/regclient/internal/regnet"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/types/blob"
|
||||
"github.com/regclient/regclient/types/descriptor"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/warning"
|
||||
)
|
||||
|
||||
var zeroDig = digest.SHA256.FromBytes([]byte{})
|
||||
|
||||
// BlobDelete removes a blob from the repository
|
||||
func (reg *Reg) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error {
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
Method: "DELETE",
|
||||
Repository: r.Repository,
|
||||
Path: "blobs/" + d.Digest.String(),
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete blob, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), err)
|
||||
}
|
||||
if resp.HTTPResponse().StatusCode != 202 {
|
||||
return fmt.Errorf("failed to delete blob, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BlobGet retrieves a blob from the repository, returning a blob reader
|
||||
func (reg *Reg) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
|
||||
// build/send request
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Blob,
|
||||
Host: r.Registry,
|
||||
Method: "GET",
|
||||
Repository: r.Repository,
|
||||
Path: "blobs/" + d.Digest.String(),
|
||||
ExpectLen: d.Size,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil && len(d.URLs) > 0 {
|
||||
for _, curURL := range d.URLs {
|
||||
// fallback for external blobs
|
||||
var u *url.URL
|
||||
u, err = url.Parse(curURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse external url \"%s\": %w", curURL, err)
|
||||
}
|
||||
// refuse requests to local URLs unless registry is also local
|
||||
// Note: the AllowRedirect check is not used because blobs could be hosted on an http CDN and data is content addressable
|
||||
if regnet.IsLocal(u.Host) && !regnet.IsLocal(req.Host) {
|
||||
return nil, fmt.Errorf("refusing to redirect blob request to a local URL: %s%.0w", curURL, errs.ErrHTTPRedirectRefused)
|
||||
}
|
||||
req = ®http.Req{
|
||||
MetaKind: reqmeta.Blob,
|
||||
Host: r.Registry,
|
||||
Method: "GET",
|
||||
Repository: r.Repository,
|
||||
DirectURL: u,
|
||||
NoMirrors: true,
|
||||
ExpectLen: d.Size,
|
||||
}
|
||||
resp, err = reg.reghttp.Do(ctx, req)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get blob, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), err)
|
||||
}
|
||||
if resp.HTTPResponse().StatusCode != 200 {
|
||||
return nil, fmt.Errorf("failed to get blob, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
b := blob.NewReader(
|
||||
blob.WithRef(r),
|
||||
blob.WithReader(resp),
|
||||
blob.WithDesc(d),
|
||||
blob.WithResp(resp.HTTPResponse()),
|
||||
)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// BlobHead is used to verify if a blob exists and is accessible
|
||||
func (reg *Reg) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
|
||||
// build/send request
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Head,
|
||||
Host: r.Registry,
|
||||
Method: "HEAD",
|
||||
Repository: r.Repository,
|
||||
Path: "blobs/" + d.Digest.String(),
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil && len(d.URLs) > 0 {
|
||||
for _, curURL := range d.URLs {
|
||||
// fallback for external blobs
|
||||
var u *url.URL
|
||||
u, err = url.Parse(curURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse external url \"%s\": %w", curURL, err)
|
||||
}
|
||||
req = ®http.Req{
|
||||
MetaKind: reqmeta.Head,
|
||||
Host: r.Registry,
|
||||
Method: "HEAD",
|
||||
Repository: r.Repository,
|
||||
DirectURL: u,
|
||||
NoMirrors: true,
|
||||
}
|
||||
resp, err = reg.reghttp.Do(ctx, req)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to request blob head, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 200 {
|
||||
return nil, fmt.Errorf("failed to request blob head, digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
b := blob.NewReader(
|
||||
blob.WithRef(r),
|
||||
blob.WithDesc(d),
|
||||
blob.WithResp(resp.HTTPResponse()),
|
||||
)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// BlobMount attempts to perform a server side copy/mount of the blob between repositories
|
||||
func (reg *Reg) BlobMount(ctx context.Context, rSrc ref.Ref, rTgt ref.Ref, d descriptor.Descriptor) error {
|
||||
putURL, _, err := reg.blobMount(ctx, rTgt, d, rSrc)
|
||||
// if mount fails and returns an upload location, cancel that upload
|
||||
if err != nil {
|
||||
_ = reg.blobUploadCancel(ctx, rTgt, putURL)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// BlobPut uploads a blob to a repository.
|
||||
// Descriptor is optional, leave size and digest to zero value if unknown.
|
||||
// Reader must also be an [io.Seeker] to support chunked upload fallback.
|
||||
//
|
||||
// This will attempt an anonymous blob mount first which some registries may support.
|
||||
// It will then try doing a full put of the blob without chunking (most widely supported).
|
||||
// If the full put fails, it will fall back to a chunked upload (useful for flaky networks).
|
||||
func (reg *Reg) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error) {
|
||||
var putURL *url.URL
|
||||
var err error
|
||||
validDesc := (d.Size > 0 && d.Digest.Validate() == nil) || (d.Size == 0 && d.Digest == zeroDig)
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
|
||||
// attempt an anonymous blob mount
|
||||
if validDesc {
|
||||
putURL, _, err = reg.blobMount(ctx, r, d, ref.Ref{})
|
||||
if err == nil {
|
||||
return d, nil
|
||||
}
|
||||
if err != errs.ErrMountReturnedLocation {
|
||||
putURL = nil
|
||||
}
|
||||
}
|
||||
// fallback to requesting upload URL
|
||||
if putURL == nil {
|
||||
putURL, err = reg.blobGetUploadURL(ctx, r, d)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
}
|
||||
// send upload as one-chunk
|
||||
tryPut := validDesc
|
||||
if tryPut {
|
||||
host := reg.hostGet(r.Registry)
|
||||
maxPut := host.BlobMax
|
||||
if maxPut == 0 {
|
||||
maxPut = reg.blobMaxPut
|
||||
}
|
||||
if maxPut > 0 && d.Size > maxPut {
|
||||
tryPut = false
|
||||
}
|
||||
}
|
||||
if tryPut {
|
||||
err = reg.blobPutUploadFull(ctx, r, d, putURL, rdr)
|
||||
if err == nil {
|
||||
return d, nil
|
||||
}
|
||||
// on failure, attempt to seek back to start to perform a chunked upload
|
||||
rdrSeek, ok := rdr.(io.ReadSeeker)
|
||||
if !ok {
|
||||
_ = reg.blobUploadCancel(ctx, r, putURL)
|
||||
return d, err
|
||||
}
|
||||
offset, errR := rdrSeek.Seek(0, io.SeekStart)
|
||||
if errR != nil || offset != 0 {
|
||||
_ = reg.blobUploadCancel(ctx, r, putURL)
|
||||
return d, err
|
||||
}
|
||||
}
|
||||
// send a chunked upload if full upload not possible or too large
|
||||
d, err = reg.blobPutUploadChunked(ctx, r, d, putURL, rdr)
|
||||
if err != nil {
|
||||
_ = reg.blobUploadCancel(ctx, r, putURL)
|
||||
}
|
||||
return d, err
|
||||
}
|
||||
|
||||
func (reg *Reg) blobGetUploadURL(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (*url.URL, error) {
|
||||
q := url.Values{}
|
||||
if d.DigestAlgo() != digest.Canonical {
|
||||
// TODO(bmitch): EXPERIMENTAL parameter, registry support and OCI spec change needed
|
||||
q.Add(paramBlobDigestAlgo, d.DigestAlgo().String())
|
||||
}
|
||||
// request an upload location
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Blob,
|
||||
Host: r.Registry,
|
||||
NoMirrors: true,
|
||||
Method: "POST",
|
||||
Repository: r.Repository,
|
||||
Path: "blobs/uploads/",
|
||||
Query: q,
|
||||
TransactLen: d.Size,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send blob post, ref %s: %w", r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 202 {
|
||||
return nil, fmt.Errorf("failed to send blob post, ref %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
// if min size header received, check/adjust host settings
|
||||
minSizeStr := resp.HTTPResponse().Header.Get(blobChunkMinHeader)
|
||||
if minSizeStr != "" {
|
||||
minSize, err := strconv.ParseInt(minSizeStr, 10, 64)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Failed to parse chunk size header",
|
||||
slog.String("size", minSizeStr),
|
||||
slog.String("err", err.Error()))
|
||||
} else {
|
||||
host := reg.hostGet(r.Registry)
|
||||
reg.muHost.Lock()
|
||||
if (host.BlobChunk > 0 && minSize > host.BlobChunk) || (host.BlobChunk <= 0 && minSize > reg.blobChunkSize) {
|
||||
host.BlobChunk = min(minSize, reg.blobChunkLimit)
|
||||
reg.slog.Debug("Registry requested min chunk size",
|
||||
slog.Int64("size", host.BlobChunk),
|
||||
slog.String("host", host.Name))
|
||||
}
|
||||
reg.muHost.Unlock()
|
||||
}
|
||||
}
|
||||
// Extract the location into a new putURL based on whether it's relative, fqdn with a scheme, or without a scheme.
|
||||
location := resp.HTTPResponse().Header.Get("Location")
|
||||
if location == "" {
|
||||
return nil, fmt.Errorf("failed to send blob post, ref %s: %w", r.CommonName(), errs.ErrMissingLocation)
|
||||
}
|
||||
reg.slog.Debug("Upload location received",
|
||||
slog.String("location", location))
|
||||
|
||||
// put url may be relative to the above post URL, so parse in that context
|
||||
postURL := resp.HTTPResponse().Request.URL
|
||||
putURL, err := postURL.Parse(location)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Location url failed to parse",
|
||||
slog.String("location", location),
|
||||
slog.String("err", err.Error()))
|
||||
return nil, fmt.Errorf("blob upload url invalid, ref %s: %w", r.CommonName(), err)
|
||||
}
|
||||
return putURL, nil
|
||||
}
|
||||
|
||||
func (reg *Reg) blobMount(ctx context.Context, rTgt ref.Ref, d descriptor.Descriptor, rSrc ref.Ref) (*url.URL, string, error) {
|
||||
// build/send request
|
||||
query := url.Values{}
|
||||
query.Set("mount", d.Digest.String())
|
||||
ignoreErr := true // ignore errors from anonymous blob mount attempts
|
||||
if rSrc.Registry == rTgt.Registry && rSrc.Repository != "" {
|
||||
query.Set("from", rSrc.Repository)
|
||||
ignoreErr = false
|
||||
}
|
||||
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Blob,
|
||||
Host: rTgt.Registry,
|
||||
NoMirrors: true,
|
||||
Method: "POST",
|
||||
Repository: rTgt.Repository,
|
||||
Path: "blobs/uploads/",
|
||||
Query: query,
|
||||
IgnoreErr: ignoreErr,
|
||||
TransactLen: d.Size,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to mount blob, digest %s, ref %s: %w", d.Digest.String(), rTgt.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
// if min size header received, check/adjust host settings
|
||||
minSizeStr := resp.HTTPResponse().Header.Get(blobChunkMinHeader)
|
||||
if minSizeStr != "" {
|
||||
minSize, err := strconv.ParseInt(minSizeStr, 10, 64)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Failed to parse chunk size header",
|
||||
slog.String("size", minSizeStr),
|
||||
slog.String("err", err.Error()))
|
||||
} else {
|
||||
host := reg.hostGet(rTgt.Registry)
|
||||
reg.muHost.Lock()
|
||||
if (host.BlobChunk > 0 && minSize > host.BlobChunk) || (host.BlobChunk <= 0 && minSize > reg.blobChunkSize) {
|
||||
host.BlobChunk = min(minSize, reg.blobChunkLimit)
|
||||
reg.slog.Debug("Registry requested min chunk size",
|
||||
slog.Int64("size", host.BlobChunk),
|
||||
slog.String("host", host.Name))
|
||||
}
|
||||
reg.muHost.Unlock()
|
||||
}
|
||||
}
|
||||
// 201 indicates the blob mount succeeded
|
||||
if resp.HTTPResponse().StatusCode == 201 {
|
||||
return nil, "", nil
|
||||
}
|
||||
// 202 indicates blob mount failed but server ready to receive an upload at location
|
||||
location := resp.HTTPResponse().Header.Get("Location")
|
||||
uuid := resp.HTTPResponse().Header.Get("Docker-Upload-UUID")
|
||||
if resp.HTTPResponse().StatusCode == 202 && location != "" {
|
||||
postURL := resp.HTTPResponse().Request.URL
|
||||
putURL, err := postURL.Parse(location)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Mount location header failed to parse",
|
||||
slog.String("digest", d.Digest.String()),
|
||||
slog.String("target", rTgt.CommonName()),
|
||||
slog.String("location", location),
|
||||
slog.String("err", err.Error()))
|
||||
} else {
|
||||
return putURL, uuid, errs.ErrMountReturnedLocation
|
||||
}
|
||||
}
|
||||
// all other responses unhandled
|
||||
return nil, "", fmt.Errorf("failed to mount blob, digest %s, ref %s: %w", d.Digest.String(), rTgt.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
func (reg *Reg) blobPutUploadFull(ctx context.Context, r ref.Ref, d descriptor.Descriptor, putURL *url.URL, rdr io.Reader) error {
|
||||
// append digest to request to use the monolithic upload option
|
||||
if putURL.RawQuery != "" {
|
||||
putURL.RawQuery = putURL.RawQuery + "&digest=" + url.QueryEscape(d.Digest.String())
|
||||
} else {
|
||||
putURL.RawQuery = "digest=" + url.QueryEscape(d.Digest.String())
|
||||
}
|
||||
|
||||
// make a reader function for the blob
|
||||
readOnce := false
|
||||
bodyFunc := func() (io.ReadCloser, error) {
|
||||
// handle attempt to reuse blob reader (e.g. on a connection retry or fallback)
|
||||
if readOnce {
|
||||
rdrSeek, ok := rdr.(io.ReadSeeker)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("blob source is not a seeker%.0w", errs.ErrNotRetryable)
|
||||
}
|
||||
_, err := rdrSeek.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seek on blob source failed: %w%.0w", err, errs.ErrNotRetryable)
|
||||
}
|
||||
}
|
||||
readOnce = true
|
||||
return io.NopCloser(rdr), nil
|
||||
}
|
||||
// special case for the empty blob
|
||||
if d.Size == 0 && d.Digest == zeroDig {
|
||||
bodyFunc = nil
|
||||
}
|
||||
|
||||
// build/send request
|
||||
header := http.Header{
|
||||
"Content-Type": {"application/octet-stream"},
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Blob,
|
||||
Host: r.Registry,
|
||||
Method: "PUT",
|
||||
Repository: r.Repository,
|
||||
DirectURL: putURL,
|
||||
BodyFunc: bodyFunc,
|
||||
BodyLen: d.Size,
|
||||
Headers: header,
|
||||
NoMirrors: true,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send blob (put), digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
// 201 follows distribution-spec, 204 is listed as possible in the Docker registry spec
|
||||
if resp.HTTPResponse().StatusCode != 201 && resp.HTTPResponse().StatusCode != 204 {
|
||||
return fmt.Errorf("failed to send blob (put), digest %s, ref %s: %w", d.Digest.String(), r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (reg *Reg) blobPutUploadChunked(ctx context.Context, r ref.Ref, d descriptor.Descriptor, putURL *url.URL, rdr io.Reader) (descriptor.Descriptor, error) {
|
||||
host := reg.hostGet(r.Registry)
|
||||
bufSize := host.BlobChunk
|
||||
if bufSize <= 0 {
|
||||
bufSize = reg.blobChunkSize
|
||||
}
|
||||
bufBytes := make([]byte, 0, bufSize)
|
||||
bufRdr := bytes.NewReader(bufBytes)
|
||||
bufStart := int64(0)
|
||||
bufChange := false
|
||||
|
||||
// setup buffer and digest pipe
|
||||
digester := d.DigestAlgo().Digester()
|
||||
digestRdr := io.TeeReader(rdr, digester.Hash())
|
||||
finalChunk := false
|
||||
chunkStart := int64(0)
|
||||
chunkSize := 0
|
||||
bodyFunc := func() (io.ReadCloser, error) {
|
||||
// reset to the start on every new read
|
||||
_, err := bufRdr.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.NopCloser(bufRdr), nil
|
||||
}
|
||||
chunkURL := *putURL
|
||||
retryLimit := 10 // TODO: pull limit from reghttp
|
||||
retryCur := 0
|
||||
var err error
|
||||
|
||||
for !finalChunk || chunkStart < bufStart+int64(len(bufBytes)) {
|
||||
bufChange = false
|
||||
for chunkStart >= bufStart+int64(len(bufBytes)) && !finalChunk {
|
||||
bufStart += int64(len(bufBytes))
|
||||
// reset length if previous read was short
|
||||
if cap(bufBytes) != len(bufBytes) {
|
||||
bufBytes = bufBytes[:cap(bufBytes)]
|
||||
bufChange = true
|
||||
}
|
||||
// read a chunk into an input buffer, computing the digest
|
||||
chunkSize, err = io.ReadFull(digestRdr, bufBytes)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
finalChunk = true
|
||||
} else if err != nil {
|
||||
return d, fmt.Errorf("failed to send blob chunk, ref %s: %w", r.CommonName(), err)
|
||||
}
|
||||
// update length on partial read
|
||||
if chunkSize != len(bufBytes) {
|
||||
bufBytes = bufBytes[:chunkSize]
|
||||
bufChange = true
|
||||
}
|
||||
}
|
||||
if chunkStart > bufStart && chunkStart < bufStart+int64(len(bufBytes)) {
|
||||
// next chunk is inside the existing buf
|
||||
bufBytes = bufBytes[chunkStart-bufStart:]
|
||||
bufStart = chunkStart
|
||||
chunkSize = len(bufBytes)
|
||||
bufChange = true
|
||||
}
|
||||
if chunkSize > 0 && chunkStart != bufStart {
|
||||
return d, fmt.Errorf("chunkStart (%d) != bufStart (%d)", chunkStart, bufStart)
|
||||
}
|
||||
if bufChange {
|
||||
// need to recreate the reader on a change to the slice length,
|
||||
// old reader is looking at the old slice metadata
|
||||
bufRdr = bytes.NewReader(bufBytes)
|
||||
}
|
||||
|
||||
if chunkSize > 0 {
|
||||
// write chunk
|
||||
header := http.Header{
|
||||
"Content-Type": {"application/octet-stream"},
|
||||
"Content-Range": {fmt.Sprintf("%d-%d", chunkStart, chunkStart+int64(chunkSize)-1)},
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Blob,
|
||||
Host: r.Registry,
|
||||
Method: "PATCH",
|
||||
Repository: r.Repository,
|
||||
DirectURL: &chunkURL,
|
||||
BodyFunc: bodyFunc,
|
||||
BodyLen: int64(chunkSize),
|
||||
Headers: header,
|
||||
NoMirrors: true,
|
||||
TransactLen: d.Size - int64(chunkSize),
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil && !errors.Is(err, errs.ErrHTTPStatus) && !errors.Is(err, errs.ErrNotFound) {
|
||||
return d, fmt.Errorf("failed to send blob (chunk), ref %s: http do: %w", r.CommonName(), err)
|
||||
}
|
||||
err = resp.Close()
|
||||
if err != nil {
|
||||
return d, fmt.Errorf("failed to close request: %w", err)
|
||||
}
|
||||
httpResp := resp.HTTPResponse()
|
||||
// distribution-spec is 202, AWS ECR returns a 201 and rejects the put
|
||||
if resp.HTTPResponse().StatusCode == 201 {
|
||||
reg.slog.Debug("Early accept of chunk in PATCH before PUT request",
|
||||
slog.String("ref", r.CommonName()),
|
||||
slog.Int64("chunkStart", chunkStart),
|
||||
slog.Int("chunkSize", chunkSize))
|
||||
} else if resp.HTTPResponse().StatusCode >= 400 && resp.HTTPResponse().StatusCode < 500 &&
|
||||
resp.HTTPResponse().Header.Get("Location") != "" &&
|
||||
resp.HTTPResponse().Header.Get("Range") != "" {
|
||||
retryCur++
|
||||
if retryCur > retryLimit {
|
||||
return d, fmt.Errorf("failed to send blob (chunk), ref %s: http status: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
reg.slog.Debug("Recoverable chunk upload error",
|
||||
slog.String("ref", r.CommonName()),
|
||||
slog.Int64("chunkStart", chunkStart),
|
||||
slog.Int("chunkSize", chunkSize),
|
||||
slog.String("range", resp.HTTPResponse().Header.Get("Range")))
|
||||
} else if resp.HTTPResponse().StatusCode != 202 {
|
||||
retryCur++
|
||||
statusResp, statusErr := reg.blobUploadStatus(ctx, r, &chunkURL)
|
||||
if retryCur > retryLimit || statusErr != nil {
|
||||
return d, fmt.Errorf("failed to send blob (chunk), ref %s: http status: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
httpResp = statusResp
|
||||
} else {
|
||||
// successful request
|
||||
if retryCur > 0 {
|
||||
retryCur--
|
||||
}
|
||||
}
|
||||
rangeEnd, err := blobUploadCurBytes(httpResp)
|
||||
if err == nil {
|
||||
chunkStart = rangeEnd + 1
|
||||
} else {
|
||||
chunkStart += int64(chunkSize)
|
||||
}
|
||||
location := httpResp.Header.Get("Location")
|
||||
if location != "" {
|
||||
reg.slog.Debug("Next chunk upload location received",
|
||||
slog.String("location", location))
|
||||
prevURL := httpResp.Request.URL
|
||||
parseURL, err := prevURL.Parse(location)
|
||||
if err != nil {
|
||||
return d, fmt.Errorf("failed to send blob (parse next chunk location), ref %s: %w", r.CommonName(), err)
|
||||
}
|
||||
chunkURL = *parseURL
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// compute digest
|
||||
dOut := digester.Digest()
|
||||
if d.Digest.Validate() == nil && dOut != d.Digest {
|
||||
return d, fmt.Errorf("%w, expected %s, computed %s", errs.ErrDigestMismatch, d.Digest.String(), dOut.String())
|
||||
}
|
||||
if d.Size != 0 && chunkStart != d.Size {
|
||||
return d, fmt.Errorf("blob content size does not match descriptor, expected %d, received %d%.0w", d.Size, chunkStart, errs.ErrMismatch)
|
||||
}
|
||||
d.Digest = dOut
|
||||
d.Size = chunkStart
|
||||
|
||||
// send the final put
|
||||
// append digest to request to use the monolithic upload option
|
||||
if chunkURL.RawQuery != "" {
|
||||
chunkURL.RawQuery = chunkURL.RawQuery + "&digest=" + url.QueryEscape(dOut.String())
|
||||
} else {
|
||||
chunkURL.RawQuery = "digest=" + url.QueryEscape(dOut.String())
|
||||
}
|
||||
|
||||
header := http.Header{
|
||||
"Content-Type": {"application/octet-stream"},
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
Method: "PUT",
|
||||
Repository: r.Repository,
|
||||
DirectURL: &chunkURL,
|
||||
BodyLen: int64(0),
|
||||
Headers: header,
|
||||
NoMirrors: true,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return d, fmt.Errorf("failed to send blob (chunk digest), digest %s, ref %s: %w", dOut, r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
// 201 follows distribution-spec, 204 is listed as possible in the Docker registry spec
|
||||
if resp.HTTPResponse().StatusCode != 201 && resp.HTTPResponse().StatusCode != 204 {
|
||||
return d, fmt.Errorf("failed to send blob (chunk digest), digest %s, ref %s: %w", dOut, r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// blobUploadCancel stops an upload, releasing resources on the server.
|
||||
func (reg *Reg) blobUploadCancel(ctx context.Context, r ref.Ref, putURL *url.URL) error {
|
||||
if putURL == nil {
|
||||
return fmt.Errorf("failed to cancel upload %s: url undefined", r.CommonName())
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
NoMirrors: true,
|
||||
Method: "DELETE",
|
||||
Repository: r.Repository,
|
||||
DirectURL: putURL,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to cancel upload %s: %w", r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 202 {
|
||||
return fmt.Errorf("failed to cancel upload %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// blobUploadStatus provides a response with headers indicating the progress of an upload
|
||||
func (reg *Reg) blobUploadStatus(ctx context.Context, r ref.Ref, putURL *url.URL) (*http.Response, error) {
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
Method: "GET",
|
||||
Repository: r.Repository,
|
||||
DirectURL: putURL,
|
||||
NoMirrors: true,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get upload status: %w", err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 204 {
|
||||
return resp.HTTPResponse(), fmt.Errorf("failed to get upload status: %w", reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
return resp.HTTPResponse(), nil
|
||||
}
|
||||
|
||||
func blobUploadCurBytes(resp *http.Response) (int64, error) {
|
||||
if resp == nil {
|
||||
return 0, fmt.Errorf("missing response")
|
||||
}
|
||||
r := resp.Header.Get("Range")
|
||||
if r == "" {
|
||||
return 0, fmt.Errorf("missing range header")
|
||||
}
|
||||
rSplit := strings.SplitN(r, "-", 2)
|
||||
if len(rSplit) < 2 {
|
||||
return 0, fmt.Errorf("missing offset in range header")
|
||||
}
|
||||
return strconv.ParseInt(rSplit[1], 10, 64)
|
||||
}
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
package reg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/internal/limitread"
|
||||
"github.com/regclient/regclient/internal/reghttp"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/manifest"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/warning"
|
||||
)
|
||||
|
||||
// ManifestDelete removes a manifest by reference (digest) from a registry.
|
||||
// This will implicitly delete all tags pointing to that manifest.
|
||||
func (reg *Reg) ManifestDelete(ctx context.Context, r ref.Ref, opts ...scheme.ManifestOpts) error {
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
if r.Digest == "" {
|
||||
return fmt.Errorf("digest required to delete manifest, reference %s%.0w", r.CommonName(), errs.ErrMissingDigest)
|
||||
}
|
||||
|
||||
mc := scheme.ManifestConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&mc)
|
||||
}
|
||||
|
||||
if mc.CheckReferrers && mc.Manifest == nil {
|
||||
m, err := reg.ManifestGet(ctx, r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to pull manifest for refers: %w", err)
|
||||
}
|
||||
mc.Manifest = m
|
||||
}
|
||||
if mc.Manifest != nil {
|
||||
if mr, ok := mc.Manifest.(manifest.Subjecter); ok {
|
||||
sDesc, err := mr.GetSubject()
|
||||
if err == nil && sDesc != nil && sDesc.Digest != "" {
|
||||
// attempt to delete the referrer, but ignore if the referrer entry wasn't found
|
||||
err = reg.referrerDelete(ctx, r, mc.Manifest)
|
||||
if err != nil && !errors.Is(err, errs.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rCache := r.SetDigest(r.Digest)
|
||||
reg.cacheMan.Delete(rCache)
|
||||
|
||||
// build/send request
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
NoMirrors: true,
|
||||
Method: "DELETE",
|
||||
Repository: r.Repository,
|
||||
Path: "manifests/" + r.Digest,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete manifest %s: %w", r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 202 {
|
||||
return fmt.Errorf("failed to delete manifest %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ManifestGet retrieves a manifest from the registry
|
||||
func (reg *Reg) ManifestGet(ctx context.Context, r ref.Ref) (manifest.Manifest, error) {
|
||||
var tagOrDigest string
|
||||
if r.Digest != "" {
|
||||
rCache := r.SetDigest(r.Digest)
|
||||
if m, err := reg.cacheMan.Get(rCache); err == nil {
|
||||
return m, nil
|
||||
}
|
||||
tagOrDigest = r.Digest
|
||||
} else if r.Tag != "" {
|
||||
tagOrDigest = r.Tag
|
||||
} else {
|
||||
return nil, fmt.Errorf("reference missing tag and digest: %s%.0w", r.CommonName(), errs.ErrMissingTagOrDigest)
|
||||
}
|
||||
|
||||
// build/send request
|
||||
headers := http.Header{
|
||||
"Accept": []string{
|
||||
mediatype.OCI1ManifestList,
|
||||
mediatype.OCI1Manifest,
|
||||
mediatype.Docker2ManifestList,
|
||||
mediatype.Docker2Manifest,
|
||||
mediatype.Docker1ManifestSigned,
|
||||
mediatype.Docker1Manifest,
|
||||
mediatype.OCI1Artifact,
|
||||
},
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Manifest,
|
||||
Host: r.Registry,
|
||||
Method: "GET",
|
||||
Repository: r.Repository,
|
||||
Path: "manifests/" + tagOrDigest,
|
||||
Headers: headers,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get manifest %s: %w", r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 200 {
|
||||
return nil, fmt.Errorf("failed to get manifest %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
// limit length
|
||||
size, _ := strconv.Atoi(resp.HTTPResponse().Header.Get("Content-Length"))
|
||||
if size > 0 && reg.manifestMaxPull > 0 && int64(size) > reg.manifestMaxPull {
|
||||
return nil, fmt.Errorf("manifest too large, received %d, limit %d: %s%.0w", size, reg.manifestMaxPull, r.CommonName(), errs.ErrSizeLimitExceeded)
|
||||
}
|
||||
rdr := &limitread.LimitRead{
|
||||
Reader: resp,
|
||||
Limit: reg.manifestMaxPull,
|
||||
}
|
||||
|
||||
// read manifest
|
||||
rawBody, err := io.ReadAll(rdr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading manifest for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
|
||||
m, err := manifest.New(
|
||||
manifest.WithRef(r),
|
||||
manifest.WithHeader(resp.HTTPResponse().Header),
|
||||
manifest.WithRaw(rawBody),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rCache := r.SetDigest(m.GetDescriptor().Digest.String())
|
||||
reg.cacheMan.Set(rCache, m)
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ManifestHead returns metadata on the manifest from the registry
|
||||
func (reg *Reg) ManifestHead(ctx context.Context, r ref.Ref) (manifest.Manifest, error) {
|
||||
// build the request
|
||||
var tagOrDigest string
|
||||
if r.Digest != "" {
|
||||
rCache := r.SetDigest(r.Digest)
|
||||
if m, err := reg.cacheMan.Get(rCache); err == nil {
|
||||
return m, nil
|
||||
}
|
||||
tagOrDigest = r.Digest
|
||||
} else if r.Tag != "" {
|
||||
tagOrDigest = r.Tag
|
||||
} else {
|
||||
return nil, fmt.Errorf("reference missing tag and digest: %s%.0w", r.CommonName(), errs.ErrMissingTagOrDigest)
|
||||
}
|
||||
|
||||
// build/send request
|
||||
headers := http.Header{
|
||||
"Accept": []string{
|
||||
mediatype.OCI1ManifestList,
|
||||
mediatype.OCI1Manifest,
|
||||
mediatype.Docker2ManifestList,
|
||||
mediatype.Docker2Manifest,
|
||||
mediatype.Docker1ManifestSigned,
|
||||
mediatype.Docker1Manifest,
|
||||
mediatype.OCI1Artifact,
|
||||
},
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Head,
|
||||
Host: r.Registry,
|
||||
Method: "HEAD",
|
||||
Repository: r.Repository,
|
||||
Path: "manifests/" + tagOrDigest,
|
||||
Headers: headers,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to request manifest head %s: %w", r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 200 {
|
||||
return nil, fmt.Errorf("failed to request manifest head %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
return manifest.New(
|
||||
manifest.WithRef(r),
|
||||
manifest.WithHeader(resp.HTTPResponse().Header),
|
||||
)
|
||||
}
|
||||
|
||||
// ManifestPut uploads a manifest to a registry
|
||||
func (reg *Reg) ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...scheme.ManifestOpts) error {
|
||||
var tagOrDigest string
|
||||
if r.Digest != "" {
|
||||
tagOrDigest = r.Digest
|
||||
} else if r.Tag != "" {
|
||||
tagOrDigest = r.Tag
|
||||
} else {
|
||||
reg.slog.Warn("Manifest put requires a tag",
|
||||
slog.String("ref", r.Reference))
|
||||
return errs.ErrMissingTag
|
||||
}
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
// create the request body
|
||||
mj, err := m.MarshalJSON()
|
||||
if err != nil {
|
||||
reg.slog.Warn("Error marshaling manifest",
|
||||
slog.String("ref", r.Reference),
|
||||
slog.String("err", err.Error()))
|
||||
return fmt.Errorf("error marshalling manifest for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
// limit length
|
||||
if reg.manifestMaxPush > 0 && int64(len(mj)) > reg.manifestMaxPush {
|
||||
return fmt.Errorf("manifest too large, calculated %d, limit %d: %s%.0w", len(mj), reg.manifestMaxPush, r.CommonName(), errs.ErrSizeLimitExceeded)
|
||||
}
|
||||
// if the ref provides a digest, verify it matches the manifest being pushed
|
||||
desc := m.GetDescriptor()
|
||||
if err := desc.Digest.Validate(); err != nil {
|
||||
return fmt.Errorf("invalid digest for manifest: %s: %w", string(desc.Digest), err)
|
||||
}
|
||||
if r.Digest != "" && desc.Digest.String() != r.Digest {
|
||||
// Digest algorithm may have changed, try recreating the manifest with the provided ref.
|
||||
// This will fail if the ref digest does not match the manifest.
|
||||
m, err = manifest.New(manifest.WithRef(r), manifest.WithRaw(mj))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rebuilding manifest with ref \"%s\": %w", r.CommonName(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// build/send request
|
||||
expectTags := []string{}
|
||||
headers := http.Header{
|
||||
"Content-Type": []string{manifest.GetMediaType(m)},
|
||||
}
|
||||
q := url.Values{}
|
||||
if tagOrDigest == r.Tag && m.GetDescriptor().Digest.Algorithm() != digest.Canonical {
|
||||
// TODO(bmitch): EXPERIMENTAL support for pushing tags with a digest: <https://github.com/opencontainers/distribution-spec/pull/600>
|
||||
tagOrDigest = m.GetDescriptor().Digest.String()
|
||||
q.Add("tag", r.Tag)
|
||||
expectTags = append(expectTags, r.Tag)
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Manifest,
|
||||
Host: r.Registry,
|
||||
NoMirrors: true,
|
||||
Method: "PUT",
|
||||
Repository: r.Repository,
|
||||
Path: "manifests/" + tagOrDigest,
|
||||
Query: q,
|
||||
Headers: headers,
|
||||
BodyLen: int64(len(mj)),
|
||||
BodyBytes: mj,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to put manifest %s: %w", r.CommonName(), err)
|
||||
}
|
||||
err = resp.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close request: %w", err)
|
||||
}
|
||||
if resp.HTTPResponse().StatusCode != 201 {
|
||||
return fmt.Errorf("failed to put manifest %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
// if Docker-Content-Digest header was returned, verify the digest matches
|
||||
if dig := resp.HTTPResponse().Header.Get("Docker-Content-Digest"); dig != "" && dig != m.GetDescriptor().Digest.String() {
|
||||
return fmt.Errorf("failed to put manifest, unexpected digest returned, expected %s, received %s", m.GetDescriptor().Digest.String(), dig)
|
||||
}
|
||||
|
||||
// if pushing tags by digest fails, fall back to pushing individual tags
|
||||
respTags := []string{}
|
||||
for _, hv := range resp.HTTPResponse().Header.Values("OCI-Tag") {
|
||||
for sv := range strings.SplitSeq(hv, ",") {
|
||||
respTags = append(respTags, strings.TrimSpace(sv))
|
||||
}
|
||||
}
|
||||
for _, t := range expectTags {
|
||||
if slices.Contains(respTags, t) {
|
||||
continue
|
||||
}
|
||||
q := url.Values{}
|
||||
if m.GetDescriptor().Digest.Algorithm() != digest.Canonical {
|
||||
// TODO(bmitch): EXPERIMENTAL parameter, registry support and OCI spec change needed: <https://github.com/opencontainers/distribution-spec/pull/543>
|
||||
q.Add(paramManifestDigest, m.GetDescriptor().Digest.String())
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Manifest,
|
||||
Host: r.Registry,
|
||||
NoMirrors: true,
|
||||
Method: "PUT",
|
||||
Repository: r.Repository,
|
||||
Path: "manifests/" + t,
|
||||
Query: q,
|
||||
Headers: headers,
|
||||
BodyLen: int64(len(mj)),
|
||||
BodyBytes: mj,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to put manifest %s: %w", r.CommonName(), err)
|
||||
}
|
||||
err = resp.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close request: %w", err)
|
||||
}
|
||||
if resp.HTTPResponse().StatusCode != 201 {
|
||||
return fmt.Errorf("failed to put manifest %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
// if Docker-Content-Digest header was returned, verify the digest matches
|
||||
if dig := resp.HTTPResponse().Header.Get("Docker-Content-Digest"); dig != "" && dig != m.GetDescriptor().Digest.String() {
|
||||
return fmt.Errorf("failed to put manifest, unexpected digest returned, expected %s, received %s", m.GetDescriptor().Digest.String(), dig)
|
||||
}
|
||||
}
|
||||
|
||||
rCache := r.SetDigest(m.GetDescriptor().Digest.String())
|
||||
reg.cacheMan.Set(rCache, m)
|
||||
|
||||
// update referrers if defined on this manifest
|
||||
if mr, ok := m.(manifest.Subjecter); ok {
|
||||
mDesc, err := mr.GetSubject()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mDesc != nil && mDesc.Digest.String() != "" {
|
||||
rSubj := r.SetDigest(mDesc.Digest.String())
|
||||
reg.cacheRL.Delete(rSubj)
|
||||
if mDesc.Digest.String() != resp.HTTPResponse().Header.Get(OCISubjectHeader) {
|
||||
err = reg.referrerPut(ctx, r, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
package reg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/regclient/regclient/internal/reghttp"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/types/ping"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
// Ping queries the /v2/ API of the registry to verify connectivity and access.
|
||||
func (reg *Reg) Ping(ctx context.Context, r ref.Ref) (ping.Result, error) {
|
||||
ret := ping.Result{}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
NoMirrors: true,
|
||||
Method: "GET",
|
||||
Path: "",
|
||||
}
|
||||
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if resp != nil && resp.HTTPResponse() != nil {
|
||||
ret.Header = resp.HTTPResponse().Header
|
||||
}
|
||||
if err != nil {
|
||||
return ret, fmt.Errorf("failed to ping registry %s: %w", r.Registry, err)
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
if resp.HTTPResponse().StatusCode != 200 {
|
||||
return ret, fmt.Errorf("failed to ping registry %s: %w",
|
||||
r.Registry, reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
package reg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
|
||||
"github.com/regclient/regclient/internal/httplink"
|
||||
"github.com/regclient/regclient/internal/reghttp"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/manifest"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
v1 "github.com/regclient/regclient/types/oci/v1"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/referrer"
|
||||
"github.com/regclient/regclient/types/warning"
|
||||
)
|
||||
|
||||
const OCISubjectHeader = "OCI-Subject"
|
||||
|
||||
// ReferrerList returns a list of referrers to a given reference.
|
||||
// The reference must include the digest. Use [regclient.ReferrerList] to resolve the platform or tag.
|
||||
func (reg *Reg) ReferrerList(ctx context.Context, rSubject ref.Ref, opts ...scheme.ReferrerOpts) (referrer.ReferrerList, error) {
|
||||
config := scheme.ReferrerConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&config)
|
||||
}
|
||||
var r ref.Ref
|
||||
if config.SrcRepo.IsSet() {
|
||||
r = config.SrcRepo.SetDigest(rSubject.Digest)
|
||||
} else {
|
||||
r = rSubject.SetDigest(rSubject.Digest)
|
||||
}
|
||||
rl := referrer.New(rSubject)
|
||||
if rSubject.Digest == "" {
|
||||
return rl, fmt.Errorf("digest required to query referrers %s", rSubject.CommonName())
|
||||
}
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
|
||||
found := false
|
||||
// try cache
|
||||
rl, err := reg.cacheRL.Get(r)
|
||||
if err == nil {
|
||||
found = true
|
||||
}
|
||||
// try referrers API
|
||||
if !found {
|
||||
referrerEnabled, ok := reg.featureGet("referrer", r.Registry, r.Repository)
|
||||
if !ok || referrerEnabled {
|
||||
// attempt to call the referrer API
|
||||
rl, err = reg.referrerListByAPI(ctx, r, config)
|
||||
if !ok {
|
||||
// save the referrer API state
|
||||
reg.featureSet("referrer", r.Registry, r.Repository, err == nil)
|
||||
}
|
||||
if err == nil {
|
||||
if config.MatchOpt.ArtifactType == "" {
|
||||
// only cache if successful and artifactType is not filtered
|
||||
reg.cacheRL.Set(r, rl)
|
||||
}
|
||||
found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
// fall back to tag
|
||||
if !found {
|
||||
rl, err = reg.referrerListByTag(ctx, r)
|
||||
if err == nil {
|
||||
reg.cacheRL.Set(r, rl)
|
||||
}
|
||||
}
|
||||
rl.Subject = rSubject
|
||||
if config.SrcRepo.IsSet() {
|
||||
rl.Source = config.SrcRepo
|
||||
}
|
||||
if err != nil {
|
||||
return rl, err
|
||||
}
|
||||
|
||||
// apply client side filters and return result
|
||||
rl = scheme.ReferrerFilter(config, rl)
|
||||
return rl, nil
|
||||
}
|
||||
|
||||
func (reg *Reg) referrerListByAPI(ctx context.Context, r ref.Ref, config scheme.ReferrerConfig) (referrer.ReferrerList, error) {
|
||||
rl := referrer.New(r)
|
||||
var link *url.URL
|
||||
// loop for paging
|
||||
for {
|
||||
rlAdd, linkNext, err := reg.referrerListByAPIPage(ctx, r, config, link)
|
||||
if err != nil {
|
||||
return rl, err
|
||||
}
|
||||
err = rl.Merge(rlAdd)
|
||||
if err != nil {
|
||||
return rl, err
|
||||
}
|
||||
if linkNext == nil {
|
||||
break
|
||||
}
|
||||
link = linkNext
|
||||
}
|
||||
return rl, nil
|
||||
}
|
||||
|
||||
func (reg *Reg) referrerListByAPIPage(ctx context.Context, r ref.Ref, config scheme.ReferrerConfig, link *url.URL) (referrer.ReferrerList, *url.URL, error) {
|
||||
rl := referrer.New(r)
|
||||
query := url.Values{}
|
||||
if config.MatchOpt.ArtifactType != "" {
|
||||
query.Set("artifactType", config.MatchOpt.ArtifactType)
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
Method: "GET",
|
||||
Repository: r.Repository,
|
||||
}
|
||||
if link == nil {
|
||||
req.Path = "referrers/" + r.Digest
|
||||
req.Query = query
|
||||
req.IgnoreErr = true
|
||||
}
|
||||
if link != nil {
|
||||
req.DirectURL = link
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return rl, nil, fmt.Errorf("failed to get referrers %s: %w", r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 200 {
|
||||
return rl, nil, fmt.Errorf("failed to get referrers %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
// read manifest
|
||||
rawBody, err := io.ReadAll(resp)
|
||||
if err != nil {
|
||||
return rl, nil, fmt.Errorf("error reading referrers for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
|
||||
m, err := manifest.New(
|
||||
manifest.WithRef(r.SetDigest("")),
|
||||
manifest.WithHeader(resp.HTTPResponse().Header),
|
||||
manifest.WithRaw(rawBody),
|
||||
)
|
||||
if err != nil {
|
||||
return rl, nil, err
|
||||
}
|
||||
ociML, ok := m.GetOrig().(v1.Index)
|
||||
if !ok {
|
||||
return rl, nil, fmt.Errorf("unexpected manifest type for referrers: %s, %w", m.GetDescriptor().MediaType, errs.ErrUnsupportedMediaType)
|
||||
}
|
||||
rl.Manifest = m
|
||||
rl.Descriptors = ociML.Manifests
|
||||
rl.Annotations = ociML.Annotations
|
||||
|
||||
// lookup next link
|
||||
respHead := resp.HTTPResponse().Header
|
||||
links, err := httplink.Parse(respHead.Values("Link"))
|
||||
if err != nil {
|
||||
return rl, nil, err
|
||||
}
|
||||
next, err := links.Get("rel", "next")
|
||||
if err != nil {
|
||||
// no next link
|
||||
link = nil
|
||||
} else {
|
||||
link = resp.HTTPResponse().Request.URL
|
||||
if link == nil {
|
||||
return rl, nil, fmt.Errorf("referrers list failed to get URL of previous request")
|
||||
}
|
||||
link, err = link.Parse(next.URI)
|
||||
if err != nil {
|
||||
return rl, nil, fmt.Errorf("referrers list failed to parse Link: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return rl, link, nil
|
||||
}
|
||||
|
||||
func (reg *Reg) referrerListByTag(ctx context.Context, r ref.Ref) (referrer.ReferrerList, error) {
|
||||
rl := referrer.New(r)
|
||||
rlTag, err := referrer.FallbackTag(r)
|
||||
if err != nil {
|
||||
return rl, err
|
||||
}
|
||||
m, err := reg.ManifestGet(ctx, rlTag)
|
||||
if err != nil {
|
||||
if errors.Is(err, errs.ErrNotFound) {
|
||||
// empty list, initialize a new manifest
|
||||
rl.Manifest, err = manifest.New(manifest.WithOrig(v1.Index{
|
||||
Versioned: v1.IndexSchemaVersion,
|
||||
MediaType: mediatype.OCI1ManifestList,
|
||||
}))
|
||||
if err != nil {
|
||||
return rl, err
|
||||
}
|
||||
return rl, nil
|
||||
}
|
||||
return rl, err
|
||||
}
|
||||
ociML, ok := m.GetOrig().(v1.Index)
|
||||
if !ok {
|
||||
return rl, fmt.Errorf("manifest is not an OCI index: %s", rlTag.CommonName())
|
||||
}
|
||||
// return resulting index
|
||||
rl.Manifest = m
|
||||
rl.Descriptors = ociML.Manifests
|
||||
rl.Annotations = ociML.Annotations
|
||||
rl.Tags = append(rl.Tags, rlTag.Tag)
|
||||
return rl, nil
|
||||
}
|
||||
|
||||
// referrerDelete deletes a referrer associated with a manifest
|
||||
func (reg *Reg) referrerDelete(ctx context.Context, r ref.Ref, m manifest.Manifest) error {
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
// get subject field
|
||||
mSubject, ok := m.(manifest.Subjecter)
|
||||
if !ok {
|
||||
return fmt.Errorf("manifest does not support the subject field: %w", errs.ErrUnsupportedMediaType)
|
||||
}
|
||||
subject, err := mSubject.GetSubject()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// validate/set subject descriptor
|
||||
if subject == nil || subject.Digest == "" {
|
||||
return fmt.Errorf("refers is not set%.0w", errs.ErrNotFound)
|
||||
}
|
||||
|
||||
// remove from cache
|
||||
rSubject := r.SetDigest(subject.Digest.String())
|
||||
reg.cacheRL.Delete(rSubject)
|
||||
|
||||
// if referrer API is available, nothing to do, return
|
||||
if reg.referrerPing(ctx, rSubject) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// fallback to using tag schema for refers
|
||||
rl, err := reg.referrerListByTag(ctx, rSubject)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = rl.Delete(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// push updated referrer list by tag
|
||||
rlTag, err := referrer.FallbackTag(rSubject)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rl.IsEmpty() {
|
||||
err = reg.TagDelete(ctx, rlTag)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
// if delete is not supported, fall back to pushing empty list
|
||||
}
|
||||
return reg.ManifestPut(ctx, rlTag, rl.Manifest)
|
||||
}
|
||||
|
||||
// referrerPut pushes a new referrer associated with a manifest
|
||||
func (reg *Reg) referrerPut(ctx context.Context, r ref.Ref, m manifest.Manifest) error {
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
// get subject field
|
||||
mSubject, ok := m.(manifest.Subjecter)
|
||||
if !ok {
|
||||
return fmt.Errorf("manifest does not support the subject field: %w", errs.ErrUnsupportedMediaType)
|
||||
}
|
||||
subject, err := mSubject.GetSubject()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// validate/set subject descriptor
|
||||
if subject == nil || subject.Digest == "" {
|
||||
return fmt.Errorf("subject is not set%.0w", errs.ErrNotFound)
|
||||
}
|
||||
|
||||
// lock to avoid internal race conditions between pulling and pushing tag
|
||||
reg.muRefTag.Lock()
|
||||
defer reg.muRefTag.Unlock()
|
||||
// fallback to using tag schema for refers
|
||||
rSubject := r.SetDigest(subject.Digest.String())
|
||||
rl, err := reg.referrerListByTag(ctx, rSubject)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = rl.Add(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// ensure the referrer list does not have a subject itself (avoiding circular locks)
|
||||
if ms, ok := rl.Manifest.(manifest.Subjecter); ok {
|
||||
mDesc, err := ms.GetSubject()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mDesc != nil && mDesc.Digest != "" {
|
||||
return fmt.Errorf("fallback referrers manifest should not have a subject: %s", rSubject.CommonName())
|
||||
}
|
||||
}
|
||||
// push updated referrer list by tag
|
||||
rlTag, err := referrer.FallbackTag(rSubject)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rl.Tags) == 0 {
|
||||
rl.Tags = []string{rlTag.Tag}
|
||||
}
|
||||
err = reg.ManifestPut(ctx, rlTag, rl.Manifest)
|
||||
if err == nil {
|
||||
reg.cacheRL.Set(rSubject, rl)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// referrerPing verifies the registry supports the referrers API
|
||||
func (reg *Reg) referrerPing(ctx context.Context, r ref.Ref) bool {
|
||||
referrerEnabled, ok := reg.featureGet("referrer", r.Registry, r.Repository)
|
||||
if ok {
|
||||
return referrerEnabled
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
Method: "GET",
|
||||
Repository: r.Repository,
|
||||
Path: "referrers/" + r.Digest,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
reg.featureSet("referrer", r.Registry, r.Repository, false)
|
||||
return false
|
||||
}
|
||||
_ = resp.Close()
|
||||
result := resp.HTTPResponse().StatusCode == 200
|
||||
reg.featureSet("referrer", r.Registry, r.Repository, result)
|
||||
return result
|
||||
}
|
||||
+272
@@ -0,0 +1,272 @@
|
||||
// Package reg implements the OCI registry scheme used by most images (host:port/repo:tag)
|
||||
package reg
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/regclient/regclient/config"
|
||||
"github.com/regclient/regclient/internal/cache"
|
||||
"github.com/regclient/regclient/internal/pqueue"
|
||||
"github.com/regclient/regclient/internal/reghttp"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/types/manifest"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/referrer"
|
||||
)
|
||||
|
||||
const (
|
||||
// blobChunkMinHeader is returned by registries requesting a minimum chunk size
|
||||
blobChunkMinHeader = "OCI-Chunk-Min-Length"
|
||||
// defaultBlobChunk 1M chunks, this is allocated in a memory buffer
|
||||
defaultBlobChunk = 1024 * 1024
|
||||
// defaultBlobChunkLimit 1G chunks, prevents a memory exhaustion attack
|
||||
defaultBlobChunkLimit = 1024 * 1024 * 1024
|
||||
// defaultBlobMax is disabled to support registries without chunked upload support
|
||||
defaultBlobMax = -1
|
||||
// defaultManifestMaxPull limits the largest manifest that will be pulled
|
||||
defaultManifestMaxPull = 1024 * 1024 * 8
|
||||
// defaultManifestMaxPush limits the largest manifest that will be pushed
|
||||
defaultManifestMaxPush = 1024 * 1024 * 4
|
||||
// paramBlobDigestAlgo specifies the query parameter to request a specific digest algorithm.
|
||||
// TODO(bmitch): EXPERIMENTAL field, registry support and OCI spec update needed
|
||||
paramBlobDigestAlgo = "digest-algorithm"
|
||||
// paramManifestDigest specifies the query parameter to specify the digest of a manifest pushed by tag.
|
||||
// TODO(bmitch): EXPERIMENTAL field, registry support and OCI spec update needed
|
||||
paramManifestDigest = "digest"
|
||||
)
|
||||
|
||||
// Reg is used for interacting with remote registry servers
|
||||
type Reg struct {
|
||||
reghttp *reghttp.Client
|
||||
reghttpOpts []reghttp.Opts
|
||||
slog *slog.Logger
|
||||
hosts map[string]*config.Host
|
||||
hostDefault *config.Host
|
||||
features map[featureKey]*featureVal
|
||||
blobChunkSize int64
|
||||
blobChunkLimit int64
|
||||
blobMaxPut int64
|
||||
manifestMaxPull int64
|
||||
manifestMaxPush int64
|
||||
cacheMan *cache.Cache[ref.Ref, manifest.Manifest]
|
||||
cacheRL *cache.Cache[ref.Ref, referrer.ReferrerList]
|
||||
muHost sync.Mutex
|
||||
muRefTag sync.Mutex
|
||||
}
|
||||
|
||||
type featureKey struct {
|
||||
kind string
|
||||
reg string
|
||||
repo string
|
||||
}
|
||||
type featureVal struct {
|
||||
enabled bool
|
||||
expire time.Time
|
||||
}
|
||||
|
||||
var featureExpire = time.Minute * time.Duration(5)
|
||||
|
||||
// Opts provides options to access registries
|
||||
type Opts func(*Reg)
|
||||
|
||||
// New returns a Reg pointer with any provided options
|
||||
func New(opts ...Opts) *Reg {
|
||||
r := Reg{
|
||||
reghttpOpts: []reghttp.Opts{},
|
||||
blobChunkSize: defaultBlobChunk,
|
||||
blobChunkLimit: defaultBlobChunkLimit,
|
||||
blobMaxPut: defaultBlobMax,
|
||||
manifestMaxPull: defaultManifestMaxPull,
|
||||
manifestMaxPush: defaultManifestMaxPush,
|
||||
hosts: map[string]*config.Host{},
|
||||
features: map[featureKey]*featureVal{},
|
||||
}
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithConfigHostFn(r.hostGet))
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
r.reghttp = reghttp.NewClient(r.reghttpOpts...)
|
||||
return &r
|
||||
}
|
||||
|
||||
// Throttle is used to limit concurrency
|
||||
func (reg *Reg) Throttle(r ref.Ref, put bool) []*pqueue.Queue[reqmeta.Data] {
|
||||
tList := []*pqueue.Queue[reqmeta.Data]{}
|
||||
host := reg.hostGet(r.Registry)
|
||||
t := reg.reghttp.GetThrottle(r.Registry)
|
||||
if t != nil {
|
||||
tList = append(tList, t)
|
||||
}
|
||||
if !put {
|
||||
for _, mirror := range host.Mirrors {
|
||||
t := reg.reghttp.GetThrottle(mirror)
|
||||
if t != nil {
|
||||
tList = append(tList, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
return tList
|
||||
}
|
||||
|
||||
func (reg *Reg) hostGet(hostname string) *config.Host {
|
||||
reg.muHost.Lock()
|
||||
defer reg.muHost.Unlock()
|
||||
if _, ok := reg.hosts[hostname]; !ok {
|
||||
newHost := config.HostNewDefName(reg.hostDefault, hostname)
|
||||
// check for normalized hostname
|
||||
if newHost.Name != hostname {
|
||||
hostname = newHost.Name
|
||||
if h, ok := reg.hosts[hostname]; ok {
|
||||
return h
|
||||
}
|
||||
}
|
||||
reg.hosts[hostname] = newHost
|
||||
}
|
||||
return reg.hosts[hostname]
|
||||
}
|
||||
|
||||
// featureGet returns enabled and ok
|
||||
func (reg *Reg) featureGet(kind, registry, repo string) (bool, bool) {
|
||||
reg.muHost.Lock()
|
||||
defer reg.muHost.Unlock()
|
||||
if v, ok := reg.features[featureKey{kind: kind, reg: registry, repo: repo}]; ok {
|
||||
if time.Now().Before(v.expire) {
|
||||
return v.enabled, true
|
||||
}
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
func (reg *Reg) featureSet(kind, registry, repo string, enabled bool) {
|
||||
reg.muHost.Lock()
|
||||
reg.features[featureKey{kind: kind, reg: registry, repo: repo}] = &featureVal{enabled: enabled, expire: time.Now().Add(featureExpire)}
|
||||
reg.muHost.Unlock()
|
||||
}
|
||||
|
||||
// WithBlobSize overrides default blob sizes
|
||||
func WithBlobSize(size, max int64) Opts {
|
||||
return func(r *Reg) {
|
||||
if size > 0 {
|
||||
r.blobChunkSize = size
|
||||
}
|
||||
if max != 0 {
|
||||
r.blobMaxPut = max
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithBlobLimit overrides default blob limit
|
||||
func WithBlobLimit(limit int64) Opts {
|
||||
return func(r *Reg) {
|
||||
if limit > 0 {
|
||||
r.blobChunkLimit = limit
|
||||
}
|
||||
if r.blobMaxPut > 0 && r.blobMaxPut < limit {
|
||||
r.blobMaxPut = limit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithCache defines a cache used for various requests
|
||||
func WithCache(timeout time.Duration, count int) Opts {
|
||||
return func(r *Reg) {
|
||||
cm := cache.New[ref.Ref, manifest.Manifest](cache.WithAge(timeout), cache.WithCount(count))
|
||||
r.cacheMan = &cm
|
||||
crl := cache.New[ref.Ref, referrer.ReferrerList](cache.WithAge(timeout), cache.WithCount(count))
|
||||
r.cacheRL = &crl
|
||||
}
|
||||
}
|
||||
|
||||
// WithCerts adds certificates
|
||||
func WithCerts(certs [][]byte) Opts {
|
||||
return func(r *Reg) {
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithCerts(certs))
|
||||
}
|
||||
}
|
||||
|
||||
// WithCertDirs adds certificate directories for host specific certs
|
||||
func WithCertDirs(dirs []string) Opts {
|
||||
return func(r *Reg) {
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithCertDirs(dirs))
|
||||
}
|
||||
}
|
||||
|
||||
// WithCertFiles adds certificates by filename
|
||||
func WithCertFiles(files []string) Opts {
|
||||
return func(r *Reg) {
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithCertFiles(files))
|
||||
}
|
||||
}
|
||||
|
||||
// WithConfigHostDefault provides default settings for hosts.
|
||||
func WithConfigHostDefault(ch *config.Host) Opts {
|
||||
return func(r *Reg) {
|
||||
r.hostDefault = ch
|
||||
}
|
||||
}
|
||||
|
||||
// WithConfigHosts adds host configs for credentials
|
||||
func WithConfigHosts(configHosts []*config.Host) Opts {
|
||||
return func(r *Reg) {
|
||||
for _, host := range configHosts {
|
||||
if host.Name == "" {
|
||||
continue
|
||||
}
|
||||
r.hosts[host.Name] = host
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithDelay initial time to wait between retries (increased with exponential backoff)
|
||||
func WithDelay(delayInit time.Duration, delayMax time.Duration) Opts {
|
||||
return func(r *Reg) {
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithDelay(delayInit, delayMax))
|
||||
}
|
||||
}
|
||||
|
||||
// WithHTTPClient uses a specific http client with retryable requests
|
||||
func WithHTTPClient(hc *http.Client) Opts {
|
||||
return func(r *Reg) {
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithHTTPClient(hc))
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifestMax sets the push and pull limits for manifests
|
||||
func WithManifestMax(push, pull int64) Opts {
|
||||
return func(r *Reg) {
|
||||
r.manifestMaxPush = push
|
||||
r.manifestMaxPull = pull
|
||||
}
|
||||
}
|
||||
|
||||
// WithRetryLimit restricts the number of retries (defaults to 5)
|
||||
func WithRetryLimit(l int) Opts {
|
||||
return func(r *Reg) {
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithRetryLimit(l))
|
||||
}
|
||||
}
|
||||
|
||||
// WithSlog injects a slog Logger configuration
|
||||
func WithSlog(slog *slog.Logger) Opts {
|
||||
return func(r *Reg) {
|
||||
r.slog = slog
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithLog(slog))
|
||||
}
|
||||
}
|
||||
|
||||
// WithTransport uses a specific http transport with retryable requests
|
||||
func WithTransport(t *http.Transport) Opts {
|
||||
return func(r *Reg) {
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithTransport(t))
|
||||
}
|
||||
}
|
||||
|
||||
// WithUserAgent sets a user agent header
|
||||
func WithUserAgent(ua string) Opts {
|
||||
return func(r *Reg) {
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithUserAgent(ua))
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
//go:build !wasm
|
||||
|
||||
package reg
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/regclient/regclient/internal/reghttp"
|
||||
"github.com/regclient/regclient/internal/sloghandle"
|
||||
)
|
||||
|
||||
// WithLog injects a logrus Logger configuration
|
||||
func WithLog(log *logrus.Logger) Opts {
|
||||
return func(r *Reg) {
|
||||
r.slog = slog.New(sloghandle.Logrus(log))
|
||||
r.reghttpOpts = append(r.reghttpOpts, reghttp.WithLog(r.slog))
|
||||
}
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package reg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
|
||||
"github.com/regclient/regclient/internal/reghttp"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
"github.com/regclient/regclient/types/repo"
|
||||
)
|
||||
|
||||
// RepoList returns a list of repositories on a registry
|
||||
// Note the underlying "_catalog" API is not supported on many cloud registries
|
||||
func (reg *Reg) RepoList(ctx context.Context, hostname string, opts ...scheme.RepoOpts) (*repo.RepoList, error) {
|
||||
config := scheme.RepoConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&config)
|
||||
}
|
||||
|
||||
query := url.Values{}
|
||||
if config.Last != "" {
|
||||
query.Set("last", config.Last)
|
||||
}
|
||||
if config.Limit > 0 {
|
||||
query.Set("n", strconv.Itoa(config.Limit))
|
||||
}
|
||||
|
||||
headers := http.Header{
|
||||
"Accept": []string{"application/json"},
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: hostname,
|
||||
NoMirrors: true,
|
||||
Method: "GET",
|
||||
Path: "_catalog",
|
||||
NoPrefix: true,
|
||||
Query: query,
|
||||
Headers: headers,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list repositories for %s: %w", hostname, err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 200 {
|
||||
return nil, fmt.Errorf("failed to list repositories for %s: %w", hostname, reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
|
||||
respBody, err := io.ReadAll(resp)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Failed to read repo list",
|
||||
slog.String("err", err.Error()),
|
||||
slog.String("host", hostname))
|
||||
return nil, fmt.Errorf("failed to read repo list for %s: %w", hostname, err)
|
||||
}
|
||||
mt := mediatype.Base(resp.HTTPResponse().Header.Get("Content-Type"))
|
||||
rl, err := repo.New(
|
||||
repo.WithMT(mt),
|
||||
repo.WithRaw(respBody),
|
||||
repo.WithHost(hostname),
|
||||
repo.WithHeaders(resp.HTTPResponse().Header),
|
||||
)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Failed to unmarshal repo list",
|
||||
slog.String("err", err.Error()),
|
||||
slog.String("body", string(respBody)),
|
||||
slog.String("host", hostname))
|
||||
return nil, fmt.Errorf("failed to parse repo list for %s: %w", hostname, err)
|
||||
}
|
||||
return rl, nil
|
||||
}
|
||||
+343
@@ -0,0 +1,343 @@
|
||||
package reg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
// crypto libraries included for go-digest
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/internal/httplink"
|
||||
"github.com/regclient/regclient/internal/reghttp"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types/descriptor"
|
||||
"github.com/regclient/regclient/types/docker/schema2"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/manifest"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
v1 "github.com/regclient/regclient/types/oci/v1"
|
||||
"github.com/regclient/regclient/types/platform"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/tag"
|
||||
"github.com/regclient/regclient/types/warning"
|
||||
)
|
||||
|
||||
// TagDelete removes a tag from a repository.
|
||||
// It first attempts the newer OCI API to delete by tag name (not widely supported).
|
||||
// If the OCI API fails, it falls back to pushing a unique empty manifest and deleting that.
|
||||
func (reg *Reg) TagDelete(ctx context.Context, r ref.Ref) error {
|
||||
var tempManifest manifest.Manifest
|
||||
if r.Tag == "" {
|
||||
return errs.ErrMissingTag
|
||||
}
|
||||
// dedup warnings
|
||||
if w := warning.FromContext(ctx); w == nil {
|
||||
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
|
||||
}
|
||||
|
||||
// attempt to delete the tag directly, available in OCI distribution-spec, and Hub API
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
NoMirrors: true,
|
||||
Method: "DELETE",
|
||||
Repository: r.Repository,
|
||||
Path: "manifests/" + r.Tag,
|
||||
IgnoreErr: true, // do not trigger backoffs if this fails
|
||||
}
|
||||
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if resp != nil {
|
||||
defer resp.Close()
|
||||
}
|
||||
if err == nil && resp != nil && resp.HTTPResponse().StatusCode == 202 {
|
||||
return nil
|
||||
}
|
||||
// ignore errors, fallback to creating a temporary manifest to replace the tag and deleting that manifest
|
||||
|
||||
// lookup the current manifest media type
|
||||
curManifest, err := reg.ManifestHead(ctx, r)
|
||||
if err != nil && errors.Is(err, errs.ErrUnsupportedAPI) {
|
||||
curManifest, err = reg.ManifestGet(ctx, r)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// create empty image config with single label
|
||||
// Note, this should be MediaType specific, but it appears that docker uses OCI for the config
|
||||
now := time.Now()
|
||||
conf := v1.Image{
|
||||
Created: &now,
|
||||
Config: v1.ImageConfig{
|
||||
Labels: map[string]string{
|
||||
"delete-tag": r.Tag,
|
||||
"delete-date": now.String(),
|
||||
},
|
||||
},
|
||||
Platform: platform.Platform{
|
||||
OS: "linux",
|
||||
Architecture: "amd64",
|
||||
},
|
||||
History: []v1.History{
|
||||
{
|
||||
Created: &now,
|
||||
CreatedBy: "# regclient",
|
||||
Comment: "empty JSON blob",
|
||||
},
|
||||
},
|
||||
RootFS: v1.RootFS{
|
||||
Type: "layers",
|
||||
DiffIDs: []digest.Digest{
|
||||
descriptor.EmptyDigest,
|
||||
},
|
||||
},
|
||||
}
|
||||
confB, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digester := digest.Canonical.Digester()
|
||||
confBuf := bytes.NewBuffer(confB)
|
||||
_, err = confBuf.WriteTo(digester.Hash())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
confDigest := digester.Digest()
|
||||
|
||||
// create manifest with config, matching the original tag manifest type
|
||||
switch manifest.GetMediaType(curManifest) {
|
||||
case mediatype.OCI1Manifest, mediatype.OCI1ManifestList:
|
||||
tempManifest, err = manifest.New(manifest.WithOrig(v1.Manifest{
|
||||
Versioned: v1.ManifestSchemaVersion,
|
||||
MediaType: mediatype.OCI1Manifest,
|
||||
Config: descriptor.Descriptor{
|
||||
MediaType: mediatype.OCI1ImageConfig,
|
||||
Digest: confDigest,
|
||||
Size: int64(len(confB)),
|
||||
},
|
||||
Layers: []descriptor.Descriptor{
|
||||
{
|
||||
MediaType: mediatype.OCI1Layer,
|
||||
Size: int64(len(descriptor.EmptyData)),
|
||||
Digest: descriptor.EmptyDigest,
|
||||
},
|
||||
},
|
||||
}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
default: // default to the docker v2 schema
|
||||
tempManifest, err = manifest.New(manifest.WithOrig(schema2.Manifest{
|
||||
Versioned: schema2.ManifestSchemaVersion,
|
||||
Config: descriptor.Descriptor{
|
||||
MediaType: mediatype.Docker2ImageConfig,
|
||||
Digest: confDigest,
|
||||
Size: int64(len(confB)),
|
||||
},
|
||||
Layers: []descriptor.Descriptor{
|
||||
{
|
||||
MediaType: mediatype.Docker2LayerGzip,
|
||||
Size: int64(len(descriptor.EmptyData)),
|
||||
Digest: descriptor.EmptyDigest,
|
||||
},
|
||||
},
|
||||
}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
reg.slog.Debug("Sending dummy manifest to replace tag",
|
||||
slog.String("ref", r.Reference))
|
||||
|
||||
// push empty layer
|
||||
_, err = reg.BlobPut(ctx, r, descriptor.Descriptor{Digest: descriptor.EmptyDigest, Size: int64(len(descriptor.EmptyData))}, bytes.NewReader(descriptor.EmptyData))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// push config
|
||||
_, err = reg.BlobPut(ctx, r, descriptor.Descriptor{Digest: confDigest, Size: int64(len(confB))}, bytes.NewReader(confB))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed sending dummy config to delete %s: %w", r.CommonName(), err)
|
||||
}
|
||||
|
||||
// push manifest to tag
|
||||
err = reg.ManifestPut(ctx, r, tempManifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed sending dummy manifest to delete %s: %w", r.CommonName(), err)
|
||||
}
|
||||
|
||||
// delete manifest by digest
|
||||
r = r.AddDigest(tempManifest.GetDescriptor().Digest.String())
|
||||
reg.slog.Debug("Deleting dummy manifest",
|
||||
slog.String("ref", r.Reference),
|
||||
slog.String("digest", r.Digest))
|
||||
err = reg.ManifestDelete(ctx, r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed deleting dummy manifest for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TagList returns a listing to tags from the repository
|
||||
func (reg *Reg) TagList(ctx context.Context, r ref.Ref, opts ...scheme.TagOpts) (*tag.List, error) {
|
||||
var config scheme.TagConfig
|
||||
for _, opt := range opts {
|
||||
opt(&config)
|
||||
}
|
||||
|
||||
tl, err := reg.tagListOCI(ctx, r, config)
|
||||
if err != nil {
|
||||
return tl, err
|
||||
}
|
||||
|
||||
for {
|
||||
// if limit reached, stop searching
|
||||
if config.Limit > 0 && len(tl.Tags) >= config.Limit {
|
||||
break
|
||||
}
|
||||
tlHead, err := tl.RawHeaders()
|
||||
if err != nil {
|
||||
return tl, err
|
||||
}
|
||||
links, err := httplink.Parse(tlHead.Values("Link"))
|
||||
if err != nil {
|
||||
return tl, err
|
||||
}
|
||||
next, err := links.Get("rel", "next")
|
||||
// if Link header with rel="next" is defined
|
||||
if err == nil {
|
||||
link := tl.GetURL()
|
||||
if link == nil {
|
||||
return tl, fmt.Errorf("tag list, failed to get URL of previous request")
|
||||
}
|
||||
link, err = link.Parse(next.URI)
|
||||
if err != nil {
|
||||
return tl, fmt.Errorf("tag list failed to parse Link: %w", err)
|
||||
}
|
||||
tlAdd, err := reg.tagListLink(ctx, r, config, link)
|
||||
if err != nil {
|
||||
return tl, fmt.Errorf("tag list failed to get Link: %w", err)
|
||||
}
|
||||
err = tl.Append(tlAdd)
|
||||
if err != nil {
|
||||
return tl, fmt.Errorf("tag list failed to append entries: %w", err)
|
||||
}
|
||||
} else {
|
||||
// do not automatically expand tags with OCI methods,
|
||||
// OCI registries should send all possible entries up to the specified limit
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return tl, nil
|
||||
}
|
||||
|
||||
func (reg *Reg) tagListOCI(ctx context.Context, r ref.Ref, config scheme.TagConfig) (*tag.List, error) {
|
||||
query := url.Values{}
|
||||
if config.Last != "" {
|
||||
query.Set("last", config.Last)
|
||||
}
|
||||
if config.Limit > 0 {
|
||||
query.Set("n", strconv.Itoa(config.Limit))
|
||||
}
|
||||
headers := http.Header{
|
||||
"Accept": []string{"application/json"},
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
Method: "GET",
|
||||
Repository: r.Repository,
|
||||
Path: "tags/list",
|
||||
Query: query,
|
||||
Headers: headers,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list tags for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 200 {
|
||||
return nil, fmt.Errorf("failed to list tags for %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
respBody, err := io.ReadAll(resp)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Failed to read tag list",
|
||||
slog.String("err", err.Error()),
|
||||
slog.String("ref", r.CommonName()))
|
||||
return nil, fmt.Errorf("failed to read tags for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
tl, err := tag.New(
|
||||
tag.WithRef(r),
|
||||
tag.WithRaw(respBody),
|
||||
tag.WithResp(resp.HTTPResponse()),
|
||||
)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Failed to unmarshal tag list",
|
||||
slog.String("err", err.Error()),
|
||||
slog.String("body", string(respBody)),
|
||||
slog.String("ref", r.CommonName()))
|
||||
return tl, fmt.Errorf("failed to unmarshal tag list for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
|
||||
return tl, nil
|
||||
}
|
||||
|
||||
func (reg *Reg) tagListLink(ctx context.Context, r ref.Ref, _ scheme.TagConfig, link *url.URL) (*tag.List, error) {
|
||||
headers := http.Header{
|
||||
"Accept": []string{"application/json"},
|
||||
}
|
||||
req := ®http.Req{
|
||||
MetaKind: reqmeta.Query,
|
||||
Host: r.Registry,
|
||||
Method: "GET",
|
||||
DirectURL: link,
|
||||
Repository: r.Repository,
|
||||
Headers: headers,
|
||||
}
|
||||
resp, err := reg.reghttp.Do(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list tags for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
defer resp.Close()
|
||||
if resp.HTTPResponse().StatusCode != 200 {
|
||||
return nil, fmt.Errorf("failed to list tags for %s: %w", r.CommonName(), reghttp.HTTPError(resp.HTTPResponse().StatusCode))
|
||||
}
|
||||
respBody, err := io.ReadAll(resp)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Failed to read tag list",
|
||||
slog.String("err", err.Error()),
|
||||
slog.String("ref", r.CommonName()))
|
||||
return nil, fmt.Errorf("failed to read tags for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
tl, err := tag.New(
|
||||
tag.WithRef(r),
|
||||
tag.WithRaw(respBody),
|
||||
tag.WithResp(resp.HTTPResponse()),
|
||||
)
|
||||
if err != nil {
|
||||
reg.slog.Warn("Failed to unmarshal tag list",
|
||||
slog.String("err", err.Error()),
|
||||
slog.String("body", string(respBody)),
|
||||
slog.String("ref", r.CommonName()))
|
||||
return tl, fmt.Errorf("failed to unmarshal tag list for %s: %w", r.CommonName(), err)
|
||||
}
|
||||
|
||||
return tl, nil
|
||||
}
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
// Package scheme defines the interface for various reference schemes.
|
||||
package scheme
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/regclient/regclient/internal/pqueue"
|
||||
"github.com/regclient/regclient/internal/reqmeta"
|
||||
"github.com/regclient/regclient/types/blob"
|
||||
"github.com/regclient/regclient/types/descriptor"
|
||||
"github.com/regclient/regclient/types/manifest"
|
||||
"github.com/regclient/regclient/types/ping"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/referrer"
|
||||
"github.com/regclient/regclient/types/tag"
|
||||
)
|
||||
|
||||
// API is used to interface between different methods to store images.
|
||||
type API interface {
|
||||
// BlobDelete removes a blob from the repository.
|
||||
BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error
|
||||
// BlobGet retrieves a blob, returning a reader.
|
||||
BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error)
|
||||
// BlobHead verifies the existence of a blob, the reader contains the headers but no body to read.
|
||||
BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error)
|
||||
// BlobMount attempts to perform a server side copy of the blob.
|
||||
BlobMount(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor) error
|
||||
// BlobPut sends a blob to the repository, returns the digest and size when successful.
|
||||
BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error)
|
||||
|
||||
// ManifestDelete removes a manifest, including all tags that point to that manifest.
|
||||
ManifestDelete(ctx context.Context, r ref.Ref, opts ...ManifestOpts) error
|
||||
// ManifestGet retrieves a manifest from a repository.
|
||||
ManifestGet(ctx context.Context, r ref.Ref) (manifest.Manifest, error)
|
||||
// ManifestHead gets metadata about the manifest (existence, digest, mediatype, size).
|
||||
ManifestHead(ctx context.Context, r ref.Ref) (manifest.Manifest, error)
|
||||
// ManifestPut sends a manifest to the repository.
|
||||
ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...ManifestOpts) error
|
||||
|
||||
// Ping verifies access to a registry or equivalent.
|
||||
Ping(ctx context.Context, r ref.Ref) (ping.Result, error)
|
||||
|
||||
// ReferrerList returns a list of referrers to a given reference.
|
||||
ReferrerList(ctx context.Context, r ref.Ref, opts ...ReferrerOpts) (referrer.ReferrerList, error)
|
||||
|
||||
// TagDelete removes a tag from the repository.
|
||||
TagDelete(ctx context.Context, r ref.Ref) error
|
||||
// TagList returns a list of tags from the repository.
|
||||
TagList(ctx context.Context, r ref.Ref, opts ...TagOpts) (*tag.List, error)
|
||||
}
|
||||
|
||||
// Closer is used to check if a scheme implements the Close API.
|
||||
type Closer interface {
|
||||
Close(ctx context.Context, r ref.Ref) error
|
||||
}
|
||||
|
||||
// GCLocker is used to indicate locking is available for GC management.
|
||||
type GCLocker interface {
|
||||
// GCLock a reference to prevent GC from triggering during a put, locks are not exclusive.
|
||||
GCLock(r ref.Ref)
|
||||
// GCUnlock a reference to allow GC (once all locks are released).
|
||||
// The reference should be closed after this step and unlock should only be called once per each Lock call.
|
||||
GCUnlock(r ref.Ref)
|
||||
}
|
||||
|
||||
// Throttler is used to indicate the scheme implements Throttle.
|
||||
type Throttler interface {
|
||||
Throttle(r ref.Ref, put bool) []*pqueue.Queue[reqmeta.Data]
|
||||
}
|
||||
|
||||
// ManifestConfig is used by schemes to import [ManifestOpts].
|
||||
type ManifestConfig struct {
|
||||
CheckReferrers bool
|
||||
Child bool // used when pushing a child of a manifest list, skips indexing in ocidir
|
||||
Manifest manifest.Manifest
|
||||
}
|
||||
|
||||
// ManifestOpts is used to set options on manifest APIs.
|
||||
type ManifestOpts func(*ManifestConfig)
|
||||
|
||||
// WithManifestCheckReferrers is used when deleting a manifest.
|
||||
// It indicates the manifest should be fetched and referrers should be deleted if defined.
|
||||
func WithManifestCheckReferrers() ManifestOpts {
|
||||
return func(config *ManifestConfig) {
|
||||
config.CheckReferrers = true
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifestChild indicates the API call is on a child manifest.
|
||||
// This is used internally when copying multi-platform manifests.
|
||||
// This bypasses tracking of an untagged digest in ocidir which is needed for garbage collection.
|
||||
func WithManifestChild() ManifestOpts {
|
||||
return func(config *ManifestConfig) {
|
||||
config.Child = true
|
||||
}
|
||||
}
|
||||
|
||||
// WithManifest is used to pass the manifest to a method to avoid an extra GET request.
|
||||
// This is used on a delete to check for referrers.
|
||||
func WithManifest(m manifest.Manifest) ManifestOpts {
|
||||
return func(mc *ManifestConfig) {
|
||||
mc.Manifest = m
|
||||
}
|
||||
}
|
||||
|
||||
// ReferrerConfig is used by schemes to import [ReferrerOpts].
|
||||
type ReferrerConfig struct {
|
||||
MatchOpt descriptor.MatchOpt // filter/sort results
|
||||
Platform string // get referrers for a specific platform
|
||||
SlowSearch bool // search every manifest in an OCI Layout index.json file
|
||||
SrcRepo ref.Ref // repo used to query referrers
|
||||
}
|
||||
|
||||
// ReferrerOpts is used to set options on referrer APIs.
|
||||
type ReferrerOpts func(*ReferrerConfig)
|
||||
|
||||
// WithReferrerMatchOpt filters results using [descriptor.MatchOpt].
|
||||
func WithReferrerMatchOpt(mo descriptor.MatchOpt) ReferrerOpts {
|
||||
return func(config *ReferrerConfig) {
|
||||
config.MatchOpt = config.MatchOpt.Merge(mo)
|
||||
}
|
||||
}
|
||||
|
||||
// WithReferrerPlatform gets referrers for a single platform from a multi-platform manifest.
|
||||
// Note that this is implemented by [regclient.ReferrerList] and not the individual scheme implementations.
|
||||
func WithReferrerPlatform(p string) ReferrerOpts {
|
||||
return func(config *ReferrerConfig) {
|
||||
config.Platform = p
|
||||
}
|
||||
}
|
||||
|
||||
// WithReferrerSlowSearch looks for referrers in ways that increases compatibility but takes longer.
|
||||
// When used with an OCI Layout, this searches every manifest for a subject for ORAS compatibility.
|
||||
func WithReferrerSlowSearch() ReferrerOpts {
|
||||
return func(config *ReferrerConfig) {
|
||||
config.SlowSearch = true
|
||||
}
|
||||
}
|
||||
|
||||
// WithReferrerSource pulls referrers from a separate source.
|
||||
// Note that this is implemented by [regclient.ReferrerList] and not the individual scheme implementations.
|
||||
func WithReferrerSource(r ref.Ref) ReferrerOpts {
|
||||
return func(config *ReferrerConfig) {
|
||||
config.SrcRepo = r
|
||||
}
|
||||
}
|
||||
|
||||
// WithReferrerAT filters by a specific artifactType value.
|
||||
//
|
||||
// Deprecated: replace with [WithReferrerMatchOpt].
|
||||
//
|
||||
//go:fix inline
|
||||
func WithReferrerAT(at string) ReferrerOpts {
|
||||
return WithReferrerMatchOpt(descriptor.MatchOpt{ArtifactType: at})
|
||||
}
|
||||
|
||||
// WithReferrerAnnotations filters by a list of annotations, all of which must match.
|
||||
//
|
||||
// Deprecated: replace with [WithReferrerMatchOpt].
|
||||
//
|
||||
//go:fix inline
|
||||
func WithReferrerAnnotations(annotations map[string]string) ReferrerOpts {
|
||||
return WithReferrerMatchOpt(descriptor.MatchOpt{Annotations: annotations})
|
||||
}
|
||||
|
||||
// WithReferrerSort orders the resulting referrers listing according to a specified annotation.
|
||||
//
|
||||
// Deprecated: replace with [WithReferrerMatchOpt].
|
||||
//
|
||||
//go:fix inline
|
||||
func WithReferrerSort(annotation string, desc bool) ReferrerOpts {
|
||||
return WithReferrerMatchOpt(descriptor.MatchOpt{SortAnnotation: annotation, SortDesc: desc})
|
||||
}
|
||||
|
||||
// ReferrerFilter filters the referrer list according to the config.
|
||||
func ReferrerFilter(config ReferrerConfig, rlIn referrer.ReferrerList) referrer.ReferrerList {
|
||||
return referrer.ReferrerList{
|
||||
Subject: rlIn.Subject,
|
||||
Source: rlIn.Source,
|
||||
Manifest: rlIn.Manifest,
|
||||
Annotations: rlIn.Annotations,
|
||||
Tags: rlIn.Tags,
|
||||
Descriptors: descriptor.DescriptorListFilter(rlIn.Descriptors, config.MatchOpt),
|
||||
}
|
||||
}
|
||||
|
||||
// RepoConfig is used by schemes to import [RepoOpts].
|
||||
type RepoConfig struct {
|
||||
Limit int
|
||||
Last string
|
||||
}
|
||||
|
||||
// RepoOpts is used to set options on repo APIs.
|
||||
type RepoOpts func(*RepoConfig)
|
||||
|
||||
// WithRepoLimit passes a maximum number of repositories to return to the repository list API.
|
||||
// Registries may ignore this.
|
||||
func WithRepoLimit(l int) RepoOpts {
|
||||
return func(config *RepoConfig) {
|
||||
config.Limit = l
|
||||
}
|
||||
}
|
||||
|
||||
// WithRepoLast passes the last received repository for requesting the next batch of repositories.
|
||||
// Registries may ignore this.
|
||||
func WithRepoLast(l string) RepoOpts {
|
||||
return func(config *RepoConfig) {
|
||||
config.Last = l
|
||||
}
|
||||
}
|
||||
|
||||
// TagConfig is used by schemes to import [TagOpts].
|
||||
type TagConfig struct {
|
||||
Limit int
|
||||
Last string
|
||||
}
|
||||
|
||||
// TagOpts is used to set options on tag APIs.
|
||||
type TagOpts func(*TagConfig)
|
||||
|
||||
// WithTagLimit passes a maximum number of tags to return to the tag list API.
|
||||
// Registries may ignore this.
|
||||
func WithTagLimit(limit int) TagOpts {
|
||||
return func(t *TagConfig) {
|
||||
t.Limit = limit
|
||||
}
|
||||
}
|
||||
|
||||
// WithTagLast passes the last received tag for requesting the next batch of tags.
|
||||
// Registries may ignore this.
|
||||
func WithTagLast(last string) TagOpts {
|
||||
return func(t *TagConfig) {
|
||||
t.Last = last
|
||||
}
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package regclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/regclient/regclient/scheme"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
"github.com/regclient/regclient/types/tag"
|
||||
)
|
||||
|
||||
// TagDelete deletes a tag from the registry. Since there's no API for this,
|
||||
// you'd want to normally just delete the manifest. However multiple tags may
|
||||
// point to the same manifest, so instead you must:
|
||||
// 1. Make a manifest, for this we put a few labels and timestamps to be unique.
|
||||
// 2. Push that manifest to the tag.
|
||||
// 3. Delete the digest for that new manifest that is only used by that tag.
|
||||
func (rc *RegClient) TagDelete(ctx context.Context, r ref.Ref) error {
|
||||
if !r.IsSet() {
|
||||
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return schemeAPI.TagDelete(ctx, r)
|
||||
}
|
||||
|
||||
// TagList returns a tag list from a repository
|
||||
func (rc *RegClient) TagList(ctx context.Context, r ref.Ref, opts ...scheme.TagOpts) (*tag.List, error) {
|
||||
if !r.IsSetRepo() {
|
||||
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
|
||||
}
|
||||
schemeAPI, err := rc.schemeGet(r.Scheme)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return schemeAPI.TagList(ctx, r, opts...)
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
// Content in this file comes from OCI
|
||||
// Copyright 2016 The Linux Foundation
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package types
|
||||
|
||||
const (
|
||||
// AnnotationCreated is the annotation key for the date and time on which the image was built (date-time string as defined by RFC 3339).
|
||||
AnnotationCreated = "org.opencontainers.image.created"
|
||||
|
||||
// AnnotationAuthors is the annotation key for the contact details of the people or organization responsible for the image (freeform string).
|
||||
AnnotationAuthors = "org.opencontainers.image.authors"
|
||||
|
||||
// AnnotationURL is the annotation key for the URL to find more information on the image.
|
||||
AnnotationURL = "org.opencontainers.image.url"
|
||||
|
||||
// AnnotationDocumentation is the annotation key for the URL to get documentation on the image.
|
||||
AnnotationDocumentation = "org.opencontainers.image.documentation"
|
||||
|
||||
// AnnotationSource is the annotation key for the URL to get source code for building the image.
|
||||
AnnotationSource = "org.opencontainers.image.source"
|
||||
|
||||
// AnnotationVersion is the annotation key for the version of the packaged software.
|
||||
// The version MAY match a label or tag in the source code repository.
|
||||
// The version MAY be Semantic versioning-compatible.
|
||||
AnnotationVersion = "org.opencontainers.image.version"
|
||||
|
||||
// AnnotationRevision is the annotation key for the source control revision identifier for the packaged software.
|
||||
AnnotationRevision = "org.opencontainers.image.revision"
|
||||
|
||||
// AnnotationVendor is the annotation key for the name of the distributing entity, organization or individual.
|
||||
AnnotationVendor = "org.opencontainers.image.vendor"
|
||||
|
||||
// AnnotationLicenses is the annotation key for the license(s) under which contained software is distributed as an SPDX License Expression.
|
||||
AnnotationLicenses = "org.opencontainers.image.licenses"
|
||||
|
||||
// AnnotationRefName is the annotation key for the name of the reference for a target.
|
||||
// SHOULD only be considered valid when on descriptors on `index.json` within image layout.
|
||||
AnnotationRefName = "org.opencontainers.image.ref.name"
|
||||
|
||||
// AnnotationTitle is the annotation key for the human-readable title of the image.
|
||||
AnnotationTitle = "org.opencontainers.image.title"
|
||||
|
||||
// AnnotationDescription is the annotation key for the human-readable description of the software packaged in the image.
|
||||
AnnotationDescription = "org.opencontainers.image.description"
|
||||
|
||||
// AnnotationBaseImageDigest is the annotation key for the digest of the image's base image.
|
||||
AnnotationBaseImageDigest = "org.opencontainers.image.base.digest"
|
||||
|
||||
// AnnotationBaseImageName is the annotation key for the image reference of the image's base image.
|
||||
AnnotationBaseImageName = "org.opencontainers.image.base.name"
|
||||
|
||||
// AnnotationArtifactCreated is the annotation key for the date and time on which the artifact was built, conforming to RFC 3339.
|
||||
AnnotationArtifactCreated = "org.opencontainers.artifact.created"
|
||||
|
||||
// AnnotationArtifactDescription is the annotation key for the human readable description for the artifact.
|
||||
AnnotationArtifactDescription = "org.opencontainers.artifact.description"
|
||||
|
||||
// AnnotationReferrersFiltersApplied is the annotation key for the comma separated list of filters applied by the registry in the referrers listing.
|
||||
AnnotationReferrersFiltersApplied = "org.opencontainers.referrers.filtersApplied"
|
||||
)
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
// Package blob is the underlying type for pushing and pulling blobs.
|
||||
package blob
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/types/descriptor"
|
||||
v1 "github.com/regclient/regclient/types/oci/v1"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
// Blob interface is used for returning blobs.
|
||||
type Blob interface {
|
||||
// GetDescriptor returns the descriptor associated with the blob.
|
||||
GetDescriptor() descriptor.Descriptor
|
||||
// RawBody returns the raw content of the blob.
|
||||
RawBody() ([]byte, error)
|
||||
// RawHeaders returns the headers received from the registry.
|
||||
RawHeaders() http.Header
|
||||
// Response returns the response associated with the blob.
|
||||
Response() *http.Response
|
||||
|
||||
// Digest returns the provided or calculated digest of the blob.
|
||||
//
|
||||
// Deprecated: Digest should be replaced by GetDescriptor().Digest.
|
||||
Digest() digest.Digest
|
||||
// Length returns the provided or calculated length of the blob.
|
||||
//
|
||||
// Deprecated: Length should be replaced by GetDescriptor().Size.
|
||||
Length() int64
|
||||
// MediaType returns the Content-Type header received from the registry.
|
||||
//
|
||||
// Deprecated: MediaType should be replaced by GetDescriptor().MediaType.
|
||||
MediaType() string
|
||||
}
|
||||
|
||||
type blobConfig struct {
|
||||
desc descriptor.Descriptor
|
||||
header http.Header
|
||||
image *v1.Image
|
||||
r ref.Ref
|
||||
rdr io.Reader
|
||||
resp *http.Response
|
||||
rawBody []byte
|
||||
}
|
||||
|
||||
// Opts is used for options to create a new blob.
|
||||
type Opts func(*blobConfig)
|
||||
|
||||
// WithDesc specifies the descriptor associated with the blob.
|
||||
func WithDesc(d descriptor.Descriptor) Opts {
|
||||
return func(bc *blobConfig) {
|
||||
bc.desc = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithHeader defines the headers received when pulling a blob.
|
||||
func WithHeader(header http.Header) Opts {
|
||||
return func(bc *blobConfig) {
|
||||
bc.header = header
|
||||
}
|
||||
}
|
||||
|
||||
// WithImage provides the OCI Image config needed for config blobs.
|
||||
func WithImage(image v1.Image) Opts {
|
||||
return func(bc *blobConfig) {
|
||||
bc.image = &image
|
||||
}
|
||||
}
|
||||
|
||||
// WithRawBody defines the raw blob contents for OCIConfig.
|
||||
func WithRawBody(raw []byte) Opts {
|
||||
return func(bc *blobConfig) {
|
||||
bc.rawBody = raw
|
||||
}
|
||||
}
|
||||
|
||||
// WithReader defines the reader for a new blob.
|
||||
func WithReader(rc io.Reader) Opts {
|
||||
return func(bc *blobConfig) {
|
||||
bc.rdr = rc
|
||||
}
|
||||
}
|
||||
|
||||
// WithRef specifies the reference where the blob was pulled from.
|
||||
func WithRef(r ref.Ref) Opts {
|
||||
return func(bc *blobConfig) {
|
||||
bc.r = r
|
||||
}
|
||||
}
|
||||
|
||||
// WithResp includes the http response, which is used to extract the headers and reader.
|
||||
func WithResp(resp *http.Response) Opts {
|
||||
return func(bc *blobConfig) {
|
||||
bc.resp = resp
|
||||
if bc.header == nil && resp != nil {
|
||||
bc.header = resp.Header
|
||||
}
|
||||
}
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package blob
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
// crypto libraries included for go-digest
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/types/descriptor"
|
||||
"github.com/regclient/regclient/types/ref"
|
||||
)
|
||||
|
||||
// Common was previously an interface. A type alias is provided for upgrades.
|
||||
type Common = *BCommon
|
||||
|
||||
// BCommon is a common struct for all blobs which includes various shared methods.
|
||||
type BCommon struct {
|
||||
r ref.Ref
|
||||
desc descriptor.Descriptor
|
||||
blobSet bool
|
||||
rawHeader http.Header
|
||||
resp *http.Response
|
||||
}
|
||||
|
||||
// GetDescriptor returns the descriptor associated with the blob.
|
||||
func (c *BCommon) GetDescriptor() descriptor.Descriptor {
|
||||
return c.desc
|
||||
}
|
||||
|
||||
// Digest returns the provided or calculated digest of the blob.
|
||||
//
|
||||
// Deprecated: Digest should be replaced by GetDescriptor().Digest, see [GetDescriptor].
|
||||
//
|
||||
//go:fix inline
|
||||
func (c *BCommon) Digest() digest.Digest {
|
||||
return c.desc.Digest
|
||||
}
|
||||
|
||||
// Length returns the provided or calculated length of the blob.
|
||||
//
|
||||
// Deprecated: Length should be replaced by GetDescriptor().Size, see [GetDescriptor].
|
||||
//
|
||||
//go:fix inline
|
||||
func (c *BCommon) Length() int64 {
|
||||
return c.desc.Size
|
||||
}
|
||||
|
||||
// MediaType returns the Content-Type header received from the registry.
|
||||
//
|
||||
// Deprecated: MediaType should be replaced by GetDescriptor().MediaType, see [GetDescriptor].
|
||||
//
|
||||
//go:fix inline
|
||||
func (c *BCommon) MediaType() string {
|
||||
return c.desc.MediaType
|
||||
}
|
||||
|
||||
// RawHeaders returns the headers received from the registry.
|
||||
func (c *BCommon) RawHeaders() http.Header {
|
||||
return c.rawHeader
|
||||
}
|
||||
|
||||
// Response returns the response associated with the blob.
|
||||
func (c *BCommon) Response() *http.Response {
|
||||
return c.resp
|
||||
}
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package blob
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
// crypto libraries included for go-digest
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
v1 "github.com/regclient/regclient/types/oci/v1"
|
||||
)
|
||||
|
||||
// OCIConfig was previously an interface. A type alias is provided for upgrading.
|
||||
type OCIConfig = *BOCIConfig
|
||||
|
||||
// BOCIConfig includes an OCI Image Config struct that may be extracted from or pushed to a blob.
|
||||
type BOCIConfig struct {
|
||||
BCommon
|
||||
rawBody []byte
|
||||
image v1.Image
|
||||
}
|
||||
|
||||
// NewOCIConfig creates a new BOCIConfig.
|
||||
// When created from an existing blob, a BOCIConfig will be created using BReader.ToOCIConfig().
|
||||
func NewOCIConfig(opts ...Opts) *BOCIConfig {
|
||||
bc := blobConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&bc)
|
||||
}
|
||||
if bc.image != nil && len(bc.rawBody) == 0 {
|
||||
var err error
|
||||
bc.rawBody, err = json.Marshal(bc.image)
|
||||
if err != nil {
|
||||
bc.rawBody = []byte{}
|
||||
}
|
||||
}
|
||||
if len(bc.rawBody) > 0 {
|
||||
if bc.image == nil {
|
||||
bc.image = &v1.Image{}
|
||||
err := json.Unmarshal(bc.rawBody, bc.image)
|
||||
if err != nil {
|
||||
bc.image = nil
|
||||
}
|
||||
}
|
||||
// force descriptor to match raw body, even if we generated the raw body
|
||||
bc.desc.Digest = bc.desc.DigestAlgo().FromBytes(bc.rawBody)
|
||||
bc.desc.Size = int64(len(bc.rawBody))
|
||||
if bc.desc.MediaType == "" {
|
||||
bc.desc.MediaType = mediatype.OCI1ImageConfig
|
||||
}
|
||||
}
|
||||
b := BOCIConfig{
|
||||
BCommon: BCommon{
|
||||
desc: bc.desc,
|
||||
r: bc.r,
|
||||
rawHeader: bc.header,
|
||||
resp: bc.resp,
|
||||
},
|
||||
rawBody: bc.rawBody,
|
||||
}
|
||||
if bc.image != nil {
|
||||
b.image = *bc.image
|
||||
b.blobSet = true
|
||||
}
|
||||
return &b
|
||||
}
|
||||
|
||||
// GetConfig returns OCI config.
|
||||
func (oc *BOCIConfig) GetConfig() v1.Image {
|
||||
return oc.image
|
||||
}
|
||||
|
||||
// RawBody returns the original body from the request.
|
||||
func (oc *BOCIConfig) RawBody() ([]byte, error) {
|
||||
var err error
|
||||
if !oc.blobSet {
|
||||
return []byte{}, fmt.Errorf("Blob is not defined")
|
||||
}
|
||||
if len(oc.rawBody) == 0 {
|
||||
oc.rawBody, err = json.Marshal(oc.image)
|
||||
}
|
||||
return oc.rawBody, err
|
||||
}
|
||||
|
||||
// SetConfig updates the config, including raw body and descriptor.
|
||||
func (oc *BOCIConfig) SetConfig(image v1.Image) {
|
||||
oc.image = image
|
||||
oc.rawBody, _ = json.Marshal(oc.image)
|
||||
if oc.desc.MediaType == "" {
|
||||
oc.desc.MediaType = mediatype.OCI1ImageConfig
|
||||
}
|
||||
oc.desc.Digest = oc.desc.DigestAlgo().FromBytes(oc.rawBody)
|
||||
oc.desc.Size = int64(len(oc.rawBody))
|
||||
oc.blobSet = true
|
||||
}
|
||||
|
||||
// MarshalJSON passes through the marshalling to the underlying image if rawBody is not available.
|
||||
func (oc *BOCIConfig) MarshalJSON() ([]byte, error) {
|
||||
if !oc.blobSet {
|
||||
return []byte{}, fmt.Errorf("Blob is not defined")
|
||||
}
|
||||
if len(oc.rawBody) > 0 {
|
||||
return oc.rawBody, nil
|
||||
}
|
||||
return json.Marshal(oc.image)
|
||||
}
|
||||
|
||||
// UnmarshalJSON extracts json content and populates the content.
|
||||
func (oc *BOCIConfig) UnmarshalJSON(data []byte) error {
|
||||
image := v1.Image{}
|
||||
err := json.Unmarshal(data, &image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oc.image = image
|
||||
oc.rawBody = make([]byte, len(data))
|
||||
copy(oc.rawBody, data)
|
||||
if oc.desc.MediaType == "" {
|
||||
oc.desc.MediaType = mediatype.OCI1ImageConfig
|
||||
}
|
||||
oc.desc.Digest = oc.desc.DigestAlgo().FromBytes(oc.rawBody)
|
||||
oc.desc.Size = int64(len(oc.rawBody))
|
||||
oc.blobSet = true
|
||||
return nil
|
||||
}
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
package blob
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
// crypto libraries included for go-digest
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/internal/limitread"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
)
|
||||
|
||||
// Reader was previously an interface. A type alias is provided for upgrading.
|
||||
type Reader = *BReader
|
||||
|
||||
// BReader is used to read blobs.
|
||||
type BReader struct {
|
||||
BCommon
|
||||
readBytes int64
|
||||
reader io.Reader
|
||||
origRdr io.Reader
|
||||
digester digest.Digester
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewReader creates a new BReader.
|
||||
func NewReader(opts ...Opts) *BReader {
|
||||
bc := blobConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&bc)
|
||||
}
|
||||
if bc.resp != nil {
|
||||
// extract headers and reader if other fields not passed
|
||||
if bc.header == nil {
|
||||
bc.header = bc.resp.Header
|
||||
}
|
||||
if bc.rdr == nil {
|
||||
bc.rdr = bc.resp.Body
|
||||
}
|
||||
}
|
||||
if bc.header != nil {
|
||||
// extract fields from header if descriptor not passed
|
||||
if bc.desc.MediaType == "" {
|
||||
bc.desc.MediaType = mediatype.Base(bc.header.Get("Content-Type"))
|
||||
}
|
||||
if bc.desc.Size == 0 {
|
||||
cl, _ := strconv.Atoi(bc.header.Get("Content-Length"))
|
||||
bc.desc.Size = int64(cl)
|
||||
}
|
||||
if bc.desc.Digest == "" {
|
||||
bc.desc.Digest, _ = digest.Parse(bc.header.Get("Docker-Content-Digest"))
|
||||
}
|
||||
}
|
||||
br := BReader{
|
||||
BCommon: BCommon{
|
||||
r: bc.r,
|
||||
desc: bc.desc,
|
||||
rawHeader: bc.header,
|
||||
resp: bc.resp,
|
||||
},
|
||||
origRdr: bc.rdr,
|
||||
}
|
||||
if bc.rdr != nil {
|
||||
br.blobSet = true
|
||||
br.digester = br.desc.DigestAlgo().Digester()
|
||||
rdr := bc.rdr
|
||||
if br.desc.Size > 0 {
|
||||
rdr = &limitread.LimitRead{
|
||||
Reader: rdr,
|
||||
Limit: br.desc.Size,
|
||||
}
|
||||
}
|
||||
br.reader = io.TeeReader(rdr, br.digester.Hash())
|
||||
}
|
||||
return &br
|
||||
}
|
||||
|
||||
// Close attempts to close the reader and populates/validates the digest.
|
||||
func (r *BReader) Close() error {
|
||||
if r == nil || r.origRdr == nil {
|
||||
return nil
|
||||
}
|
||||
// attempt to close if available in original reader
|
||||
bc, ok := r.origRdr.(io.Closer)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return bc.Close()
|
||||
}
|
||||
|
||||
// RawBody returns the original body from the request.
|
||||
func (r *BReader) RawBody() ([]byte, error) {
|
||||
return io.ReadAll(r)
|
||||
}
|
||||
|
||||
// Read passes through the read operation while computing the digest and tracking the size.
|
||||
func (r *BReader) Read(p []byte) (int, error) {
|
||||
if r == nil || r.reader == nil {
|
||||
return 0, fmt.Errorf("blob has no reader: %w", io.ErrUnexpectedEOF)
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
size, err := r.reader.Read(p)
|
||||
r.readBytes = r.readBytes + int64(size)
|
||||
if err == io.EOF {
|
||||
// check/save size
|
||||
if r.desc.Size == 0 {
|
||||
r.desc.Size = r.readBytes
|
||||
} else if r.readBytes < r.desc.Size {
|
||||
err = fmt.Errorf("%w [expected %d, received %d]: %w", errs.ErrShortRead, r.desc.Size, r.readBytes, err)
|
||||
} else if r.readBytes > r.desc.Size {
|
||||
err = fmt.Errorf("%w [expected %d, received %d]: %w", errs.ErrSizeLimitExceeded, r.desc.Size, r.readBytes, err)
|
||||
}
|
||||
// check/save digest
|
||||
if r.desc.Digest.Validate() != nil {
|
||||
r.desc.Digest = r.digester.Digest()
|
||||
} else if r.desc.Digest != r.digester.Digest() {
|
||||
err = fmt.Errorf("%w [expected %s, calculated %s]: %w", errs.ErrDigestMismatch, r.desc.Digest.String(), r.digester.Digest().String(), err)
|
||||
}
|
||||
}
|
||||
return size, err
|
||||
}
|
||||
|
||||
// Seek passes through the seek operation, reseting or invalidating the digest
|
||||
func (r *BReader) Seek(offset int64, whence int) (int64, error) {
|
||||
if r == nil || r.origRdr == nil {
|
||||
return 0, fmt.Errorf("blob has no reader")
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if offset == 0 && whence == io.SeekCurrent {
|
||||
return r.readBytes, nil
|
||||
}
|
||||
// cannot do an arbitrary seek and still digest without a lot more complication
|
||||
if offset != 0 || whence != io.SeekStart {
|
||||
return r.readBytes, fmt.Errorf("unable to seek to arbitrary position")
|
||||
}
|
||||
rdrSeek, ok := r.origRdr.(io.Seeker)
|
||||
if !ok {
|
||||
return r.readBytes, fmt.Errorf("Seek unsupported")
|
||||
}
|
||||
o, err := rdrSeek.Seek(offset, whence)
|
||||
if err != nil || o != 0 {
|
||||
return r.readBytes, err
|
||||
}
|
||||
// reset internal offset and digest calculation
|
||||
rdr := r.origRdr
|
||||
if r.desc.Size > 0 {
|
||||
rdr = &limitread.LimitRead{
|
||||
Reader: rdr,
|
||||
Limit: r.desc.Size,
|
||||
}
|
||||
}
|
||||
r.digester = r.desc.DigestAlgo().Digester()
|
||||
r.reader = io.TeeReader(rdr, r.digester.Hash())
|
||||
r.readBytes = 0
|
||||
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// ToOCIConfig converts a BReader to a BOCIConfig.
|
||||
func (r *BReader) ToOCIConfig() (*BOCIConfig, error) {
|
||||
if r == nil || !r.blobSet {
|
||||
return nil, fmt.Errorf("blob is not defined")
|
||||
}
|
||||
if r.readBytes != 0 {
|
||||
return nil, fmt.Errorf("unable to convert after read has been performed")
|
||||
}
|
||||
blobBody, err := io.ReadAll(r)
|
||||
errC := r.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading image config for %s: %w", r.r.CommonName(), err)
|
||||
}
|
||||
if errC != nil {
|
||||
return nil, fmt.Errorf("error closing blob reader: %w", err)
|
||||
}
|
||||
return NewOCIConfig(
|
||||
WithDesc(r.desc),
|
||||
WithHeader(r.rawHeader),
|
||||
WithRawBody(blobBody),
|
||||
WithRef(r.r),
|
||||
WithResp(r.resp),
|
||||
), nil
|
||||
}
|
||||
|
||||
// ToTarReader converts a BReader to a BTarReader
|
||||
func (r *BReader) ToTarReader() (*BTarReader, error) {
|
||||
if r == nil || !r.blobSet {
|
||||
return nil, fmt.Errorf("blob is not defined")
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.readBytes != 0 {
|
||||
return nil, fmt.Errorf("unable to convert after read has been performed")
|
||||
}
|
||||
return NewTarReader(
|
||||
WithDesc(r.desc),
|
||||
WithHeader(r.rawHeader),
|
||||
WithRef(r.r),
|
||||
WithResp(r.resp),
|
||||
WithReader(r.reader),
|
||||
), nil
|
||||
}
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
package blob
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/internal/limitread"
|
||||
"github.com/regclient/regclient/pkg/archive"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
)
|
||||
|
||||
// TarReader was previously an interface. A type alias is provided for upgrading.
|
||||
type TarReader = *BTarReader
|
||||
|
||||
// BTarReader is used to read individual files from an image layer.
|
||||
type BTarReader struct {
|
||||
BCommon
|
||||
origRdr io.Reader
|
||||
reader io.Reader
|
||||
digester digest.Digester
|
||||
tr *tar.Reader
|
||||
}
|
||||
|
||||
// NewTarReader creates a BTarReader.
|
||||
// Typically a BTarReader will be created using BReader.ToTarReader().
|
||||
func NewTarReader(opts ...Opts) *BTarReader {
|
||||
bc := blobConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&bc)
|
||||
}
|
||||
tr := BTarReader{
|
||||
BCommon: BCommon{
|
||||
desc: bc.desc,
|
||||
r: bc.r,
|
||||
rawHeader: bc.header,
|
||||
resp: bc.resp,
|
||||
},
|
||||
origRdr: bc.rdr,
|
||||
}
|
||||
if bc.rdr != nil {
|
||||
tr.blobSet = true
|
||||
tr.digester = tr.desc.DigestAlgo().Digester()
|
||||
rdr := bc.rdr
|
||||
if tr.desc.Size > 0 {
|
||||
rdr = &limitread.LimitRead{
|
||||
Reader: rdr,
|
||||
Limit: tr.desc.Size,
|
||||
}
|
||||
}
|
||||
tr.reader = io.TeeReader(rdr, tr.digester.Hash())
|
||||
}
|
||||
return &tr
|
||||
}
|
||||
|
||||
// Close attempts to close the reader and populates/validates the digest.
|
||||
func (tr *BTarReader) Close() error {
|
||||
// attempt to close if available in original reader
|
||||
if trc, ok := tr.origRdr.(io.Closer); ok && trc != nil {
|
||||
return trc.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTarReader returns the tar.Reader for the blob.
|
||||
func (tr *BTarReader) GetTarReader() (*tar.Reader, error) {
|
||||
if tr.reader == nil {
|
||||
return nil, fmt.Errorf("blob has no reader defined")
|
||||
}
|
||||
if tr.tr == nil {
|
||||
dr, err := archive.Decompress(tr.reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tr.tr = tar.NewReader(dr)
|
||||
}
|
||||
return tr.tr, nil
|
||||
}
|
||||
|
||||
// RawBody returns the original body from the request.
|
||||
func (tr *BTarReader) RawBody() ([]byte, error) {
|
||||
if !tr.blobSet {
|
||||
return []byte{}, fmt.Errorf("Blob is not defined")
|
||||
}
|
||||
if tr.tr != nil {
|
||||
return []byte{}, fmt.Errorf("RawBody cannot be returned after TarReader returned")
|
||||
}
|
||||
b, err := io.ReadAll(tr.reader)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
if tr.digester != nil {
|
||||
dig := tr.digester.Digest()
|
||||
tr.digester = nil
|
||||
if tr.desc.Digest.String() != "" && dig != tr.desc.Digest {
|
||||
return b, fmt.Errorf("%w, expected %s, received %s", errs.ErrDigestMismatch, tr.desc.Digest.String(), dig.String())
|
||||
}
|
||||
tr.desc.Digest = dig
|
||||
}
|
||||
err = tr.Close()
|
||||
return b, err
|
||||
}
|
||||
|
||||
// ReadFile parses the tar to find a file.
|
||||
func (tr *BTarReader) ReadFile(filename string) (*tar.Header, io.Reader, error) {
|
||||
if strings.HasPrefix(filename, ".wh.") {
|
||||
return nil, nil, fmt.Errorf(".wh. prefix is reserved for whiteout files")
|
||||
}
|
||||
// normalize filenames,
|
||||
filename = filepath.Clean(filename)
|
||||
if filename[0] == '/' {
|
||||
filename = filename[1:]
|
||||
}
|
||||
// get reader
|
||||
rdr, err := tr.GetTarReader()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// loop through files until whiteout or target file is found
|
||||
whiteout := false
|
||||
for {
|
||||
th, err := rdr.Next()
|
||||
if err != nil {
|
||||
// break on eof, everything else is an error
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
thFile := filepath.Clean(th.Name)
|
||||
if thFile[0] == '/' {
|
||||
thFile = thFile[1:]
|
||||
}
|
||||
// found the target file
|
||||
if thFile == filename {
|
||||
return th, rdr, nil
|
||||
}
|
||||
// check/track whiteout file
|
||||
name := filepath.Base(th.Name)
|
||||
if !whiteout && strings.HasPrefix(name, ".wh.") && tarCmpWhiteout(th.Name, filename) {
|
||||
// continue searching after finding a whiteout file
|
||||
// a new file may be created in the same layer
|
||||
whiteout = true
|
||||
}
|
||||
}
|
||||
// EOF encountered
|
||||
if whiteout {
|
||||
return nil, nil, errs.ErrFileDeleted
|
||||
}
|
||||
if tr.digester != nil {
|
||||
_, _ = io.Copy(io.Discard, tr.reader) // process/digest any trailing bytes from reader
|
||||
dig := tr.digester.Digest()
|
||||
tr.digester = nil
|
||||
if tr.desc.Digest.String() != "" && dig != tr.desc.Digest {
|
||||
return nil, nil, fmt.Errorf("%w, expected %s, received %s", errs.ErrDigestMismatch, tr.desc.Digest.String(), dig.String())
|
||||
}
|
||||
tr.desc.Digest = dig
|
||||
}
|
||||
return nil, nil, errs.ErrFileNotFound
|
||||
}
|
||||
|
||||
func tarCmpWhiteout(whFile, tgtFile string) bool {
|
||||
whSplit := strings.Split(whFile, "/")
|
||||
tgtSplit := strings.Split(tgtFile, "/")
|
||||
// the -1 handles the opaque whiteout
|
||||
if len(whSplit)-1 > len(tgtSplit) {
|
||||
return false
|
||||
}
|
||||
// verify the path matches up to the whiteout
|
||||
for i := range whSplit[:len(whSplit)-1] {
|
||||
if whSplit[i] != tgtSplit[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
i := len(whSplit) - 1
|
||||
// opaque whiteout of entire directory
|
||||
if whSplit[i] == ".wh..wh..opq" {
|
||||
return true
|
||||
}
|
||||
// compare whiteout name to next path entry
|
||||
if i > len(tgtSplit)-1 {
|
||||
return false
|
||||
}
|
||||
whName := strings.TrimPrefix(whSplit[i], ".wh.")
|
||||
return whName == tgtSplit[i]
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package types
|
||||
|
||||
type CallbackState int
|
||||
|
||||
const (
|
||||
CallbackUndef CallbackState = iota
|
||||
CallbackSkipped
|
||||
CallbackStarted
|
||||
CallbackActive
|
||||
CallbackFinished
|
||||
CallbackArchived
|
||||
)
|
||||
|
||||
type CallbackKind int
|
||||
|
||||
const (
|
||||
CallbackManifest CallbackKind = iota
|
||||
CallbackBlob
|
||||
)
|
||||
|
||||
func (k CallbackKind) String() string {
|
||||
switch k {
|
||||
case CallbackBlob:
|
||||
return "blob"
|
||||
case CallbackManifest:
|
||||
return "manifest"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package types
|
||||
|
||||
import "github.com/regclient/regclient/types/descriptor"
|
||||
|
||||
type (
|
||||
// Descriptor is used in manifests to refer to content by media type, size, and digest.
|
||||
//
|
||||
// Deprecated: replace with [descriptor.Descriptor].
|
||||
//go:fix inline
|
||||
Descriptor = descriptor.Descriptor
|
||||
// MatchOpt defines conditions for a match descriptor.
|
||||
//
|
||||
// Deprecated: replace with [descriptor.MatchOpt].
|
||||
//go:fix inline
|
||||
MatchOpt = descriptor.MatchOpt
|
||||
)
|
||||
|
||||
var (
|
||||
// EmptyData is the content of the empty JSON descriptor. See [mediatype.OCI1Empty].
|
||||
//
|
||||
// Deprecated: replace with [descriptor.EmptyData].
|
||||
//go:fix inline
|
||||
EmptyData = descriptor.EmptyData
|
||||
// EmptyDigest is the digest of the empty JSON descriptor. See [mediatype.OCI1Empty].
|
||||
//
|
||||
// Deprecated: replace with [descriptor.EmptyDigest].
|
||||
//go:fix inline
|
||||
EmptyDigest = descriptor.EmptyDigest
|
||||
// DescriptorListFilter returns a list of descriptors from the list matching the search options.
|
||||
// When opt.SortAnnotation is set, the order of descriptors with matching annotations is undefined.
|
||||
//
|
||||
// Deprecated: replace with [descriptor.DescriptorListFilter]
|
||||
//go:fix inline
|
||||
DescriptorListFilter = descriptor.DescriptorListFilter
|
||||
// DescriptorListSearch returns the first descriptor from the list matching the search options.
|
||||
//
|
||||
// Deprecated: replace with [descriptor.DescriptorListSearch]
|
||||
//go:fix inline
|
||||
DescriptorListSearch = descriptor.DescriptorListSearch
|
||||
)
|
||||
+349
@@ -0,0 +1,349 @@
|
||||
// Package descriptor defines the OCI descriptor data structure used in manifests to reference content addressable data.
|
||||
package descriptor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
// crypto libraries included for go-digest
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/internal/units"
|
||||
"github.com/regclient/regclient/types/errs"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
"github.com/regclient/regclient/types/platform"
|
||||
)
|
||||
|
||||
// Descriptor is used in manifests to refer to content by media type, size, and digest.
|
||||
type Descriptor struct {
|
||||
// MediaType describe the type of the content.
|
||||
MediaType string `json:"mediaType"`
|
||||
|
||||
// Digest uniquely identifies the content.
|
||||
Digest digest.Digest `json:"digest"`
|
||||
|
||||
// Size in bytes of content.
|
||||
Size int64 `json:"size"`
|
||||
|
||||
// URLs contains the source URLs of this content.
|
||||
URLs []string `json:"urls,omitempty"`
|
||||
|
||||
// Annotations contains arbitrary metadata relating to the targeted content.
|
||||
Annotations map[string]string `json:"annotations,omitempty"`
|
||||
|
||||
// Data is an embedding of the targeted content. This is encoded as a base64
|
||||
// string when marshalled to JSON (automatically, by encoding/json). If
|
||||
// present, Data can be used directly to avoid fetching the targeted content.
|
||||
Data []byte `json:"data,omitempty"`
|
||||
|
||||
// Platform describes the platform which the image in the manifest runs on.
|
||||
// This should only be used when referring to a manifest.
|
||||
Platform *platform.Platform `json:"platform,omitempty"`
|
||||
|
||||
// ArtifactType is the media type of the artifact this descriptor refers to.
|
||||
ArtifactType string `json:"artifactType,omitempty"`
|
||||
|
||||
// digestAlgo is the preferred digest algorithm for when the digest is unset.
|
||||
digestAlgo digest.Algorithm
|
||||
}
|
||||
|
||||
var (
|
||||
// EmptyData is the content of the empty JSON descriptor. See [mediatype.OCI1Empty].
|
||||
EmptyData = []byte("{}")
|
||||
// EmptyDigest is the digest of the empty JSON descriptor. See [mediatype.OCI1Empty].
|
||||
EmptyDigest = digest.SHA256.FromBytes(EmptyData)
|
||||
mtToOCI map[string]string
|
||||
)
|
||||
|
||||
func init() {
|
||||
mtToOCI = map[string]string{
|
||||
mediatype.Docker2ManifestList: mediatype.OCI1ManifestList,
|
||||
mediatype.Docker2Manifest: mediatype.OCI1Manifest,
|
||||
mediatype.Docker2ImageConfig: mediatype.OCI1ImageConfig,
|
||||
mediatype.Docker2Layer: mediatype.OCI1Layer,
|
||||
mediatype.Docker2LayerGzip: mediatype.OCI1LayerGzip,
|
||||
mediatype.Docker2LayerZstd: mediatype.OCI1LayerZstd,
|
||||
mediatype.OCI1ManifestList: mediatype.OCI1ManifestList,
|
||||
mediatype.OCI1Manifest: mediatype.OCI1Manifest,
|
||||
mediatype.OCI1ImageConfig: mediatype.OCI1ImageConfig,
|
||||
mediatype.OCI1Layer: mediatype.OCI1Layer,
|
||||
mediatype.OCI1LayerGzip: mediatype.OCI1LayerGzip,
|
||||
mediatype.OCI1LayerZstd: mediatype.OCI1LayerZstd,
|
||||
}
|
||||
}
|
||||
|
||||
// DigestAlgo returns the algorithm for computing the digest.
|
||||
// This prefers the algorithm used by the digest when set, falling back to the preferred digest algorithm, and finally the canonical algorithm.
|
||||
func (d Descriptor) DigestAlgo() digest.Algorithm {
|
||||
if d.Digest != "" && d.Digest.Validate() == nil {
|
||||
return d.Digest.Algorithm()
|
||||
}
|
||||
if d.digestAlgo != "" && d.digestAlgo.Available() {
|
||||
return d.digestAlgo
|
||||
}
|
||||
return digest.Canonical
|
||||
}
|
||||
|
||||
// DigestAlgoPrefer sets the preferred digest algorithm for when the digest is unset.
|
||||
func (d *Descriptor) DigestAlgoPrefer(algo digest.Algorithm) error {
|
||||
if !algo.Available() {
|
||||
return fmt.Errorf("digest algorithm is not available: %s%.0w", algo.String(), errs.ErrUnsupported)
|
||||
}
|
||||
d.digestAlgo = algo
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetData decodes the Data field from the descriptor if available
|
||||
func (d Descriptor) GetData() ([]byte, error) {
|
||||
// verify length
|
||||
if int64(len(d.Data)) != d.Size {
|
||||
return nil, errs.ErrParsingFailed
|
||||
}
|
||||
// generate and verify digest
|
||||
if d.Digest != d.DigestAlgo().FromBytes(d.Data) {
|
||||
return nil, errs.ErrParsingFailed
|
||||
}
|
||||
// return data
|
||||
return d.Data, nil
|
||||
}
|
||||
|
||||
// Equal indicates the two descriptors are identical, effectively a DeepEqual.
|
||||
func (d Descriptor) Equal(d2 Descriptor) bool {
|
||||
if !d.Same(d2) {
|
||||
return false
|
||||
}
|
||||
if d.MediaType != d2.MediaType {
|
||||
return false
|
||||
}
|
||||
if d.ArtifactType != d2.ArtifactType {
|
||||
return false
|
||||
}
|
||||
if d.Platform == nil || d2.Platform == nil {
|
||||
if d.Platform != nil || d2.Platform != nil {
|
||||
return false
|
||||
}
|
||||
} else if !platform.Match(*d.Platform, *d2.Platform) {
|
||||
return false
|
||||
}
|
||||
if d.URLs == nil || d2.URLs == nil {
|
||||
if d.URLs != nil || d2.URLs != nil {
|
||||
return false
|
||||
}
|
||||
} else if len(d.URLs) != len(d2.URLs) {
|
||||
return false
|
||||
} else {
|
||||
for i := range d.URLs {
|
||||
if d.URLs[i] != d2.URLs[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
if d.Annotations == nil || d2.Annotations == nil {
|
||||
if d.Annotations != nil || d2.Annotations != nil {
|
||||
return false
|
||||
}
|
||||
} else if len(d.Annotations) != len(d2.Annotations) {
|
||||
return false
|
||||
} else {
|
||||
for i := range d.Annotations {
|
||||
if d.Annotations[i] != d2.Annotations[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Same indicates two descriptors point to the same CAS object.
|
||||
// This verifies the digest, media type, and size all match.
|
||||
func (d Descriptor) Same(d2 Descriptor) bool {
|
||||
if d.Digest != d2.Digest || d.Size != d2.Size {
|
||||
return false
|
||||
}
|
||||
// loosen the check on media type since this can be converted from a build
|
||||
if d.MediaType != d2.MediaType && (mtToOCI[d.MediaType] != mtToOCI[d2.MediaType] || mtToOCI[d.MediaType] == "") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (d Descriptor) MarshalPrettyTW(tw *tabwriter.Writer, prefix string) error {
|
||||
fmt.Fprintf(tw, "%sDigest:\t%s\n", prefix, string(d.Digest))
|
||||
fmt.Fprintf(tw, "%sMediaType:\t%s\n", prefix, d.MediaType)
|
||||
if d.ArtifactType != "" {
|
||||
fmt.Fprintf(tw, "%sArtifactType:\t%s\n", prefix, d.ArtifactType)
|
||||
}
|
||||
switch d.MediaType {
|
||||
case mediatype.Docker1Manifest, mediatype.Docker1ManifestSigned,
|
||||
mediatype.Docker2Manifest, mediatype.Docker2ManifestList,
|
||||
mediatype.OCI1Manifest, mediatype.OCI1ManifestList:
|
||||
// skip printing size for descriptors to manifests
|
||||
default:
|
||||
if d.Size > 100000 {
|
||||
fmt.Fprintf(tw, "%sSize:\t%s\n", prefix, units.HumanSize(float64(d.Size)))
|
||||
} else {
|
||||
fmt.Fprintf(tw, "%sSize:\t%dB\n", prefix, d.Size)
|
||||
}
|
||||
}
|
||||
if p := d.Platform; p != nil && p.OS != "" {
|
||||
fmt.Fprintf(tw, "%sPlatform:\t%s\n", prefix, p.String())
|
||||
if p.OSVersion != "" {
|
||||
fmt.Fprintf(tw, "%sOSVersion:\t%s\n", prefix, p.OSVersion)
|
||||
}
|
||||
if len(p.OSFeatures) > 0 {
|
||||
fmt.Fprintf(tw, "%sOSFeatures:\t%s\n", prefix, strings.Join(p.OSFeatures, ", "))
|
||||
}
|
||||
}
|
||||
if len(d.URLs) > 0 {
|
||||
fmt.Fprintf(tw, "%sURLs:\t%s\n", prefix, strings.Join(d.URLs, ", "))
|
||||
}
|
||||
if d.Annotations != nil {
|
||||
fmt.Fprintf(tw, "%sAnnotations:\t\n", prefix)
|
||||
for k, v := range d.Annotations {
|
||||
fmt.Fprintf(tw, "%s %s:\t%s\n", prefix, k, v)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MatchOpt defines conditions for a match descriptor.
|
||||
type MatchOpt struct {
|
||||
Platform *platform.Platform // Platform to match including compatible platforms (darwin/arm64 matches linux/arm64)
|
||||
ArtifactType string // Match ArtifactType in the descriptor
|
||||
Annotations map[string]string // Match each of the specified annotations and their value, an empty value verifies the key is set
|
||||
SortAnnotation string // Sort the results by an annotation, string based comparison, descriptors without the annotation are sorted last
|
||||
SortDesc bool // Set to true to sort in descending order
|
||||
}
|
||||
|
||||
// Merge applies changes to a MatchOpt, overwriting existing values, and returning a new MatchOpt.
|
||||
func (mo MatchOpt) Merge(changes MatchOpt) MatchOpt {
|
||||
ret := MatchOpt{
|
||||
ArtifactType: changes.ArtifactType,
|
||||
SortAnnotation: changes.SortAnnotation,
|
||||
SortDesc: changes.SortDesc,
|
||||
}
|
||||
if ret.ArtifactType == "" {
|
||||
ret.ArtifactType = mo.ArtifactType
|
||||
}
|
||||
if changes.Platform != nil {
|
||||
p := *changes.Platform
|
||||
ret.Platform = &p
|
||||
} else if mo.Platform != nil {
|
||||
p := *mo.Platform
|
||||
ret.Platform = &p
|
||||
}
|
||||
if ret.SortAnnotation == "" {
|
||||
ret.SortAnnotation = mo.SortAnnotation
|
||||
}
|
||||
if !ret.SortDesc {
|
||||
ret.SortDesc = mo.SortDesc
|
||||
}
|
||||
if len(mo.Annotations) > 0 {
|
||||
ret.Annotations = maps.Clone(mo.Annotations)
|
||||
}
|
||||
if len(changes.Annotations) > 0 {
|
||||
if ret.Annotations == nil {
|
||||
ret.Annotations = changes.Annotations
|
||||
} else {
|
||||
maps.Copy(ret.Annotations, changes.Annotations)
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Match returns true if the descriptor matches the options, including compatible platforms.
|
||||
func (d Descriptor) Match(opt MatchOpt) bool {
|
||||
if opt.ArtifactType != "" && d.ArtifactType != opt.ArtifactType {
|
||||
return false
|
||||
}
|
||||
if len(opt.Annotations) > 0 {
|
||||
if d.Annotations == nil {
|
||||
return false
|
||||
}
|
||||
for k, v := range opt.Annotations {
|
||||
if dv, ok := d.Annotations[k]; !ok || (v != "" && v != dv) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
if opt.Platform != nil {
|
||||
if d.Platform == nil {
|
||||
return false
|
||||
}
|
||||
if !platform.Compatible(*opt.Platform, *d.Platform) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// DescriptorListFilter returns a list of descriptors from the list matching the search options.
|
||||
// When opt.SortAnnotation is set, the order of descriptors with matching annotations is undefined.
|
||||
func DescriptorListFilter(dl []Descriptor, opt MatchOpt) []Descriptor {
|
||||
ret := []Descriptor{}
|
||||
for _, d := range dl {
|
||||
if d.Match(opt) {
|
||||
ret = append(ret, d)
|
||||
}
|
||||
}
|
||||
if opt.SortAnnotation != "" {
|
||||
sort.Slice(ret, func(i, j int) bool {
|
||||
// if annotations are not defined, sort to the very end
|
||||
if ret[i].Annotations == nil {
|
||||
return false
|
||||
}
|
||||
if _, ok := ret[i].Annotations[opt.SortAnnotation]; !ok {
|
||||
return false
|
||||
}
|
||||
if ret[j].Annotations == nil {
|
||||
return true
|
||||
}
|
||||
if _, ok := ret[j].Annotations[opt.SortAnnotation]; !ok {
|
||||
return true
|
||||
}
|
||||
// else sort by string
|
||||
if strings.Compare(ret[i].Annotations[opt.SortAnnotation], ret[j].Annotations[opt.SortAnnotation]) < 0 {
|
||||
return !opt.SortDesc
|
||||
}
|
||||
return opt.SortDesc
|
||||
})
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// DescriptorListSearch returns the first descriptor from the list matching the search options.
|
||||
func DescriptorListSearch(dl []Descriptor, opt MatchOpt) (Descriptor, error) {
|
||||
if opt.ArtifactType != "" || opt.SortAnnotation != "" || len(opt.Annotations) > 0 {
|
||||
dl = DescriptorListFilter(dl, opt)
|
||||
}
|
||||
var ret Descriptor
|
||||
var retPlat platform.Platform
|
||||
if len(dl) == 0 {
|
||||
return ret, errs.ErrNotFound
|
||||
}
|
||||
if opt.Platform == nil {
|
||||
return dl[0], nil
|
||||
}
|
||||
found := false
|
||||
comp := platform.NewCompare(*opt.Platform)
|
||||
for _, d := range dl {
|
||||
if d.Platform == nil {
|
||||
continue
|
||||
}
|
||||
if comp.Better(*d.Platform, retPlat) {
|
||||
found = true
|
||||
ret = d
|
||||
retPlat = *d.Platform
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return ret, errs.ErrNotFound
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
// Package types defines various types that have no other internal imports
|
||||
// This allows them to be used between other packages without creating import loops
|
||||
package types
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
// Package schema1 defines the manifest and json marshal/unmarshal for docker schema1
|
||||
package schema1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
// crypto libraries included for go-digest
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/docker/libtrust"
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/regclient/regclient/types/docker"
|
||||
"github.com/regclient/regclient/types/mediatype"
|
||||
)
|
||||
|
||||
var (
|
||||
// ManifestSchemaVersion provides a pre-initialized version structure schema1 manifests.
|
||||
ManifestSchemaVersion = docker.Versioned{
|
||||
SchemaVersion: 1,
|
||||
MediaType: mediatype.Docker1Manifest,
|
||||
}
|
||||
// ManifestSignedSchemaVersion provides a pre-initialized version structure schema1 signed manifests.
|
||||
ManifestSignedSchemaVersion = docker.Versioned{
|
||||
SchemaVersion: 1,
|
||||
MediaType: mediatype.Docker1ManifestSigned,
|
||||
}
|
||||
)
|
||||
|
||||
// FSLayer is a container struct for BlobSums defined in an image manifest
|
||||
type FSLayer struct {
|
||||
// BlobSum is the tarsum of the referenced filesystem image layer
|
||||
BlobSum digest.Digest `json:"blobSum"`
|
||||
}
|
||||
|
||||
// History stores unstructured v1 compatibility information
|
||||
type History struct {
|
||||
// V1Compatibility is the raw v1 compatibility information
|
||||
V1Compatibility string `json:"v1Compatibility"`
|
||||
}
|
||||
|
||||
// Manifest defines the schema v1 docker manifest
|
||||
type Manifest struct {
|
||||
docker.Versioned
|
||||
|
||||
// Name is the name of the image's repository
|
||||
Name string `json:"name"`
|
||||
|
||||
// Tag is the tag of the image specified by this manifest
|
||||
Tag string `json:"tag"`
|
||||
|
||||
// Architecture is the host architecture on which this image is intended to run
|
||||
Architecture string `json:"architecture"`
|
||||
|
||||
// FSLayers is a list of filesystem layer blobSums contained in this image
|
||||
FSLayers []FSLayer `json:"fsLayers"`
|
||||
|
||||
// History is a list of unstructured historical data for v1 compatibility
|
||||
History []History `json:"history"`
|
||||
}
|
||||
|
||||
// SignedManifest provides an envelope for a signed image manifest, including the format sensitive raw bytes.
|
||||
type SignedManifest struct {
|
||||
Manifest
|
||||
|
||||
// Canonical is the canonical byte representation of the ImageManifest, without any attached signatures.
|
||||
// The manifest byte representation cannot change or it will have to be re-signed.
|
||||
Canonical []byte `json:"-"`
|
||||
|
||||
// all contains the byte representation of the Manifest including signatures and is returned by Payload()
|
||||
all []byte
|
||||
}
|
||||
|
||||
// UnmarshalJSON populates a new SignedManifest struct from JSON data.
|
||||
func (sm *SignedManifest) UnmarshalJSON(b []byte) error {
|
||||
sm.all = make([]byte, len(b))
|
||||
// store manifest and signatures in all
|
||||
copy(sm.all, b)
|
||||
|
||||
jsig, err := libtrust.ParsePrettySignature(b, "signatures")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Resolve the payload in the manifest.
|
||||
bytes, err := jsig.Payload()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// sm.Canonical stores the canonical manifest JSON
|
||||
sm.Canonical = make([]byte, len(bytes))
|
||||
copy(sm.Canonical, bytes)
|
||||
|
||||
// Unmarshal canonical JSON into Manifest object
|
||||
var manifest Manifest
|
||||
if err := json.Unmarshal(sm.Canonical, &manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sm.Manifest = manifest
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON returns the contents of raw.
|
||||
// If Raw is nil, marshals the inner contents.
|
||||
// Applications requiring a marshaled signed manifest should simply use Raw directly, since the the content produced by json.Marshal will be compacted and will fail signature checks.
|
||||
func (sm *SignedManifest) MarshalJSON() ([]byte, error) {
|
||||
if len(sm.all) > 0 {
|
||||
return sm.all, nil
|
||||
}
|
||||
|
||||
// If the raw data is not available, just dump the inner content.
|
||||
return json.Marshal(&sm.Manifest)
|
||||
}
|
||||
|
||||
// TODO: verify Payload and Signatures methods are required
|
||||
|
||||
// Payload returns the signed content of the signed manifest.
|
||||
func (sm SignedManifest) Payload() (string, []byte, error) {
|
||||
return mediatype.Docker1ManifestSigned, sm.all, nil
|
||||
}
|
||||
|
||||
// Signatures returns the signatures as provided by (*libtrust.JSONSignature).Signatures.
|
||||
// The byte slices are opaque jws signatures.
|
||||
func (sm *SignedManifest) Signatures() ([][]byte, error) {
|
||||
jsig, err := libtrust.ParsePrettySignature(sm.all, "signatures")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Resolve the payload in the manifest.
|
||||
return jsig.Signatures()
|
||||
}
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
// Package schema2 contains structs for Docker schema v2 manifests.
|
||||
package schema2
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user