Compare commits

..
Author SHA1 Message Date
sixsmith 309fc72c38 temp 2026-10-10 01:04:50 -04:00
sixsmith 455b5fef21 revert Makefile change, moved to another CL 2026-10-09 21:58:43 -04:00
sixsmith d8742e728c Merge remote-tracking branch 'toolshed/main' 2026-10-09 21:56:00 -04:00
sixsmith eaec0cc733 update digest pinning work
- formatter.RemoveSha
- make pindigests TRUE by default
- more strict matching of digests
- unit and integration tests
2026-10-09 21:51:51 -04:00
sixsmith 0f7d991709 add lint rule for images with digest 2026-09-10 00:13:35 -04:00
sixsmith 39050a85ef run make-i18n 2026-09-10 00:13:35 -04:00
sixsmith 92bab8637d add --pindigests flag
* handle digests for the recipe subcommand
* add make parameter to pass args
2026-09-10 00:11:52 -04:00
sixsmith 7932356746 add regclient to fetch image manifest/digest
* also run `go mod vendor`
2026-09-10 00:11:52 -04:00
sixsmith 91fe901264 add docs and test 2026-09-10 00:11:52 -04:00
231 changed files with 29624 additions and 3283 deletions
+1
View File
@@ -7,3 +7,4 @@
/bin
dist/
tests/integration/.bats
tests/resources/path_to_repo
+50
View File
@@ -84,3 +84,53 @@ build-mo:
release:
@goreleaser release --clean
CLIENT_VM := abra-client
vm-client-status:
sudo systemctl status microvm@$(CLIENT_VM).service
vm-client-create:test-integration-hosts
sudo microvm -f git+file://$$(pwd) -c $(CLIENT_VM)
vm-client-start:
sudo systemctl start microvm@$(CLIENT_VM).service
vm-client-update:test-integration-hosts
sudo microvm -R -f git+file://$$(pwd) -u $(CLIENT_VM)
vm-client-run:
sudo microvm -f git+file://$$(pwd) -r $(CLIENT_VM)
vm-client-stop:
sudo systemctl stop microvm@$(CLIENT_VM)
vm-client-delete: vm-client-stop
sudo rm -rf /var/lib/microvms/$(CLIENT_VM)
vm-client-connect:
ssh abra@10.0.0.2 -i ./tests/resources/local_integration_ssh
SERVER_VM := abra-server
vm-server-status:
sudo systemctl status microvm@$(SERVER_VM).service
vm-server-create:
sudo microvm -f git+file://$$(pwd) -c $(SERVER_VM)
vm-server-start:
sudo systemctl start microvm@$(SERVER_VM).service
vm-server-update:
sudo microvm -R -f git+file://$$(pwd) -u $(SERVER_VM)
vm-server-run:
sudo microvm -f git+file://$$(pwd) -r $(SERVER_VM)
vm-server-stop:
sudo systemctl stop microvm@$(SERVER_VM)
vm-server-delete: vm-server-stop
sudo rm -rf /var/lib/microvms/$(SERVER_VM)
vm-server-connect:
ssh abra@10.0.0.3 -i ./tests/resources/local_integration_ssh
test-integration-hosts:
./scripts/tests/extra_hosts
# TEST_INTEGRATION_FILES overrides the default of running all test files
ifdef TEST_INTEGRATION_FILES
test_files := $(TEST_INTEGRATION_FILES);
else
test_files := *;
endif
test-integration: build
chmod 600 ./tests/resources/local_integration_ssh
ssh-add ./tests/resources/local_integration_ssh
@bats -Tp --verbose-run tests/integration/$(test_files)
+1
View File
@@ -19,5 +19,6 @@ var (
Minor bool
NoDomainChecks bool
Patch bool
PinDigests bool
ShowUnchanged bool
)
+5 -4
View File
@@ -92,7 +92,7 @@ your private key and enter your passphrase beforehand.
log.Fatal(err)
}
mainAppVersion := imagesTmp[mainApp]
mainAppVersion := formatter.RemoveSha(imagesTmp[mainApp])
if mainAppVersion == "" {
log.Fatal(i18n.G("main app service version for %s is empty?", recipe.Name))
}
@@ -192,6 +192,9 @@ likely to change.
serviceMeta := serviceVersions[serviceName]
existingImageTag := fmt.Sprintf("%s:%s", serviceMeta.Image, serviceMeta.Tag)
if serviceMeta.Digest != "" {
existingImageTag += "@" + serviceMeta.Digest
}
newImageTag := fmt.Sprintf("%s:%s", serviceMeta.Image, imagesTmp[serviceMeta.Image])
if existingImageTag == newImageTag {
@@ -331,10 +334,8 @@ func GetImageVersions(recipe recipePkg.Recipe) (map[string]string, error) {
path = formatter.StripTagMeta(path)
var tag string
switch img.(type) {
case reference.NamedTagged:
tag = img.(reference.NamedTagged).Tag()
case reference.Named:
if service.Name == "app" {
missingTag = true
@@ -342,7 +343,7 @@ func GetImageVersions(recipe recipePkg.Recipe) (map[string]string, error) {
continue
}
services[path] = tag
services[path] = formatter.GetTagAndDigest(img)
}
if missingTag {
+42 -8
View File
@@ -130,7 +130,7 @@ interface.`),
}
for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(service.Image)
img, err := reference.ParseNormalizedNamed(formatter.RemoveSha(service.Image))
if err != nil {
log.Fatal(err)
}
@@ -178,9 +178,14 @@ interface.`),
sort.Sort(tagcmp.ByTagDesc(compatible))
var upgradeTag string
if len(compatible) == 0 && !allTags {
log.Info(i18n.G("no new versions available for %s, assuming %s is the latest (use -a/--all-tags to see all anyway)", image, tag))
continue // skip on to the next tag and don't update any compose files
if !shouldPin(service.Image, internal.PinDigests) {
continue
}
upgradeTag = tag.String()
}
catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name, internal.Offline)
@@ -203,9 +208,8 @@ interface.`),
log.Debug(i18n.G("detected compatible upgradable tags %s for %s", compatibleStrings, service.Name))
var upgradeTag string
_, ok := servicePins[service.Name]
if versionsPresent && ok {
if upgradeTag == "" && versionsPresent && ok {
pinnedTag := servicePins[service.Name].version
if tag.IsLessThan(pinnedTag) {
pinnedTagString := pinnedTag.String()
@@ -227,7 +231,7 @@ interface.`),
log.Fatal(i18n.G("service %s is at version %s, but pinned to %s, please correct your compose.yml file manually!", service.Name, tag.String(), pinnedTag.String()))
continue
}
} else {
} else if upgradeTag == "" {
if bumpType != 0 {
for _, upTag := range compatible {
upElement, err := tag.UpgradeDelta(upTag)
@@ -242,7 +246,10 @@ interface.`),
}
if upgradeTag == "" {
log.Warn(i18n.G("not upgrading from %s to %s for %s, because the upgrade type is more serious than what user wants", tag.String(), compatible[0].String(), image))
continue
if !shouldPin(service.Image, internal.PinDigests) {
continue
}
upgradeTag = tag.String()
}
} else {
msg := i18n.G("upgrade to which tag? (service: %s, image: %s, tag: %s)", service.Name, image, tag)
@@ -293,12 +300,25 @@ interface.`),
}
}
if upgradeTag != "skip" {
ok, err := recipe.UpdateTag(image, upgradeTag)
var resolvedDigest string
if shouldPin(service.Image, internal.PinDigests) {
resolvedDigest, err = client.GetImageMultiplatformDigest(img, upgradeTag)
if err != nil {
log.Fatal(i18n.G("failed to resolve digest for %s:%s: %s", image, upgradeTag, err.Error()))
}
}
ok, err := recipe.UpdateTag(service.Image, upgradeTag, resolvedDigest)
if err != nil {
log.Fatal(err)
}
if ok {
log.Info(i18n.G("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image))
if shouldPin(service.Image, internal.PinDigests) {
log.Info(i18n.G("tag upgraded and pinned from %s to %s@%s for %s", tag.String(), upgradeTag, resolvedDigest, image))
} else {
log.Info(i18n.G("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image))
}
}
} else {
if !internal.NoInput {
@@ -364,6 +384,12 @@ interface.`),
},
}
// shouldPin reports whether a service image should be pinned to a digest.
// An image that is already pinned stays pinned, even without --pindigests.
func shouldPin(image string, pinDigestsFlag bool) bool {
return pinDigestsFlag || strings.Contains(image, "@sha256:")
}
var (
allTags bool
createCommit bool
@@ -417,4 +443,12 @@ func init() {
false,
i18n.G("commit changes"),
)
RecipeUpgradeCommand.Flags().BoolVarP(
&internal.PinDigests,
i18n.G("pindigests"),
i18n.GC("p", "pin container image by digest"),
true,
i18n.G("pin the container image version by manifest digest (image:tag@sha256:...)"),
)
}
+22
View File
@@ -0,0 +1,22 @@
package recipe
import "testing"
func TestShouldPin(t *testing.T) {
tests := []struct {
image string
pinDigests bool
want bool
}{
{"nginx:1.21.0", false, false},
{"nginx:1.21.0", true, true},
{"nginx:1.21.0@sha256:2bcabc23b45489fb0885d69a06ba1d648aeda973fae7bb981bafbb884165e514", false, true},
{"nginx:1.21.0@sha256:2bcabc23b45489fb0885d69a06ba1d648aeda973fae7bb981bafbb884165e514", true, true},
}
for _, test := range tests {
if got := shouldPin(test.image, test.pinDigests); got != test.want {
t.Errorf("shouldPin(%q, %v) = %v, want %v", test.image, test.pinDigests, got, test.want)
}
}
}
Generated
+38
View File
@@ -18,6 +18,27 @@
"type": "github"
}
},
"microvm": {
"inputs": {
"nixpkgs": [
"nixpkgs"
],
"spectrum": "spectrum"
},
"locked": {
"lastModified": 1784666190,
"narHash": "sha256-xgfS6slV7J3baMooNN1UuBi51RIgg9y0DbCxfSA0668=",
"owner": "astro",
"repo": "microvm.nix",
"rev": "fa5340ac684cdce8a22b6d4a0bcebb0cc999275e",
"type": "github"
},
"original": {
"owner": "astro",
"repo": "microvm.nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1778443072,
@@ -37,9 +58,26 @@
"root": {
"inputs": {
"flake-utils": "flake-utils",
"microvm": "microvm",
"nixpkgs": "nixpkgs"
}
},
"spectrum": {
"flake": false,
"locked": {
"lastModified": 1783694892,
"narHash": "sha256-xO8f7Qng+18FK2UlB9vcrkxCaQMCt5WjCH24aW/11eg=",
"ref": "refs/heads/main",
"rev": "24c4346e30fdea8d8e80f34aec3554a15a667d24",
"revCount": 1410,
"type": "git",
"url": "https://spectrum-os.org/git/spectrum"
},
"original": {
"type": "git",
"url": "https://spectrum-os.org/git/spectrum"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
+87 -18
View File
@@ -4,34 +4,103 @@
inputs = {
nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable";
flake-utils.url = "github:numtide/flake-utils";
microvm = {
url = "github:astro/microvm.nix";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs =
{
self,
nixpkgs,
microvm,
flake-utils,
}:
flake-utils.lib.eachDefaultSystem (
system:
let
pkgs = nixpkgs.legacyPackages.${system};
in
{
packages = rec {
abra = pkgs.callPackage ./package.nix { };
default = abra;
let
system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system};
publicKey = builtins.readFile ./tests/resources/local_integration_ssh.pub;
privateKey = builtins.readFile ./tests/resources/local_integration_ssh;
pathToRepo = builtins.readFile ./tests/resources/path_to_repo;
# local DNS aliases for the server host
extraHosts = builtins.readFile ./tests/resources/extra_hosts;
username = "abra";
password = "abra";
defaultDNS = [
# Quad9.net
"9.9.9.9"
"149.112.112.112"
"2620:fe::fe"
"2620:fe::9"
];
in
{
nixosConfigurations = {
abra-client = nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
modules = [
microvm.nixosModules.microvm
./nix/hosts/client/configuration.nix
];
specialArgs = {
inherit
publicKey
privateKey
username
password
defaultDNS
extraHosts
pathToRepo
;
};
};
apps = rec {
abra = flake-utils.lib.mkApp { drv = self.packages.${system}.abra; };
default = abra;
abra-server = nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
modules = [
microvm.nixosModules.microvm
./nix/hosts/server/configuration.nix
];
specialArgs = {
inherit
publicKey
privateKey
username
password
defaultDNS
;
};
};
devShells.default = pkgs.mkShell {
packages = with pkgs; [
go_1_26
gnumake
};
nixosModules = rec {
host = {
imports = [
microvm.nixosModules.host
./nix/modules/host.nix
];
};
}
);
default = host;
};
packages = rec {
abra = pkgs.callPackage ./nix/package.nix { };
default = abra;
};
apps = rec {
abra = flake-utils.lib.mkApp { drv = self.packages.${system}.abra; };
default = abra;
};
devShells.${system}.default = pkgs.mkShell {
# testing env variables
BATS_LIB_PATH = "~/.local/share/bats/";
TEST_SERVER = "abra.local";
ABRA_DIR = "$HOME/.abra_test";
packages = with pkgs; [
go_1_26
gnumake
gopls
];
};
};
}
+26 -21
View File
@@ -1,6 +1,6 @@
module coopcloud.tech/abra
go 1.27
go 1.26.0
require (
coopcloud.tech/tagcmp v0.0.0-20260515102403-c26951b55977
@@ -9,23 +9,18 @@ require (
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
github.com/charmbracelet/log v1.0.0
github.com/containers/image v3.0.2+incompatible
github.com/decentral1se/passgen v1.0.1
github.com/distribution/reference v0.6.0
github.com/docker/cli v28.4.0+incompatible
github.com/docker/docker v28.5.2+incompatible
github.com/docker/go-units v0.5.0
github.com/go-git/go-git/v5 v5.19.3
github.com/google/go-cmp v0.7.0
github.com/hashicorp/go-retryablehttp v0.7.8
github.com/leonelquinteros/gotext v1.8.0
github.com/leonelquinteros/gotext v1.7.2
github.com/moby/sys/signal v0.7.1
github.com/moby/term v0.5.2
github.com/pkg/errors v0.9.1
github.com/regclient/regclient v0.11.5
github.com/schollz/progressbar/v3 v3.19.1
github.com/spf13/cobra v1.10.1
github.com/stretchr/testify v1.12.1
golang.org/x/sys v0.49.0
golang.org/x/term v0.47.0
gopkg.in/yaml.v3 v3.0.1
gotest.tools/v3 v3.5.2
@@ -54,18 +49,15 @@ require (
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/containers/storage v1.38.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/docker-credential-helpers v0.9.5 // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fvbommel/sortorder v1.1.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.9.2 // indirect
@@ -74,16 +66,14 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kevinburke/ssh_config v1.6.0 // indirect
github.com/klauspost/compress v1.18.5 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect
@@ -94,9 +84,7 @@ require (
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.1.0 // indirect
github.com/moby/patternmatcher v0.6.1 // indirect
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/morikuni/aec v1.1.0 // indirect
@@ -105,23 +93,19 @@ require (
github.com/muesli/termenv v0.16.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/opencontainers/runc v1.1.13 // indirect
github.com/opencontainers/runtime-spec v1.1.0 // indirect
github.com/pjbgf/sha1cd v0.7.0 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.5 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.4.0 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/skeema/knownhosts v1.3.2 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/theupdateframework/notary v0.7.0 // indirect
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
@@ -152,6 +136,27 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
)
require (
github.com/containers/image v3.0.2+incompatible
github.com/containers/storage v1.38.2 // indirect
github.com/decentral1se/passgen v1.0.1
github.com/docker/docker-credential-helpers v0.9.5 // indirect
github.com/fvbommel/sortorder v1.1.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/hashicorp/go-retryablehttp v0.7.8
github.com/moby/patternmatcher v0.6.1 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/sergi/go-diff v1.4.0 // indirect
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.12.1
github.com/theupdateframework/notary v0.7.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
golang.org/x/sys v0.49.0
)
replace github.com/docker/cli v28.4.0+incompatible => git.coopcloud.tech/toolshed/docker-cli v28.5.3-0.20260202112816-30df2d0b3a00+incompatible
replace github.com/spf13/cobra => github.com/decentral1se/cobra v1.10.2
+14 -4
View File
@@ -426,6 +426,8 @@ github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
@@ -583,8 +585,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -602,8 +604,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/leonelquinteros/gotext v1.8.0 h1:wwpxINWmWq3JN3WFI9BoSupBFrvAqxZTtf3qppijYkU=
github.com/leonelquinteros/gotext v1.8.0/go.mod h1:Wqke380+8unZq2F4oTmmqmbUY9zWQMzwImdC34XOAo8=
github.com/leonelquinteros/gotext v1.7.2 h1:bDPndU8nt+/kRo1m4l/1OXiiy2v7Z7dfPQ9+YP7G1Mc=
github.com/leonelquinteros/gotext v1.7.2/go.mod h1:9/haCkm5P7Jay1sxKDGJ5WIg4zkz8oZKw4ekNpALob8=
github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
@@ -696,6 +698,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/olareg/olareg v0.2.1 h1:RPHGIaqlVWPbKAsOYUj7e2WfEEW5M7F8I6hKMrwd4jU=
github.com/olareg/olareg v0.2.1/go.mod h1:dhr8QetC7U7jJ2m93oxDhEEOKCRbPgOK1oGyKfB4QNo=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -792,6 +796,8 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
github.com/regclient/regclient v0.11.5 h1:OHRsXO0F3qHGfa4HEUv+EkMH9NXNcCTBKjNzyC/UhIA=
github.com/regclient/regclient v0.11.5/go.mod h1:DZUOfIT14WFTK2Pj4vjd93avy9O4Fdpjrf9ir23TbRE=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -855,6 +861,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/sudo-bmitch/oci-digest v0.1.2 h1:are0qzWTsFZGZ3Uvdi9OSztJszSWaab6iqquMEEB7rw=
github.com/sudo-bmitch/oci-digest v0.1.2/go.mod h1:SH6l5OIe0islKBZBedjiPOeET/0QwGL+/oYfQt51uQo=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
@@ -866,6 +874,8 @@ github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnH
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-54
View File
@@ -1,54 +0,0 @@
{
buildGo126Module,
fetchgit,
lib,
installShellFiles,
}:
buildGo126Module rec {
pname = "abra";
version = "0.13.0-beta";
rev = "06a57ded025a43c80f94d4e65299add8a31830dc";
src = fetchgit {
url = "https://git.coopcloud.tech/toolshed/abra.git";
tag = version;
hash = "sha256-rgoK0TY0WLSQ39lPvVM80zW/qJF40VFBSxYDOaKXZQo=";
};
vendorHash = null;
nativeBuildInputs = [
installShellFiles
];
env.CGO_ENABLED = 0;
buildPhase = ''
runHook preBuild
go build -ldflags="-s -w -X 'main.Commit=${rev}' -X 'main.Version=${version}'" ./cmd/abra
runHook postBuild
'';
installPhase = ''
runHook preInstall
install -D abra $out/bin/abra
runHook postInstall
'';
postInstall = ''
export ABRA_DIR="$out"
$out/bin/abra autocomplete bash >abra.bash
$out/bin/abra autocomplete fish >abra.fish
$out/bin/abra autocomplete zsh >abra.zsh
installShellCompletion abra.{bash,fish,zsh}
'';
meta = with lib; {
description = "The Co-op Cloud utility belt";
homepage = "https://docs.coopcloud.tech/abra";
changelog = "https://git.coopcloud.tech/toolshed/abra/releases/tag/${version}";
mainProgram = "abra";
license = licenses.gpl3Plus;
maintainers = "devydave";
};
}
+32
View File
@@ -9,6 +9,8 @@ import (
"github.com/containers/image/docker"
"github.com/containers/image/types"
"github.com/distribution/reference"
"github.com/regclient/regclient"
"github.com/regclient/regclient/types/ref"
)
// GetRegistryTags retrieves all tags of an image from a container registry.
@@ -28,3 +30,33 @@ func GetRegistryTags(img reference.Named) ([]string, error) {
return tags, nil
}
// GetImageDigest resolves the content digest (sha256:...) for a specific
// image:tag reference by querying the registry manifest. This gives us a
// digest to pin to while keeping support for multi-arch deployments
// See: https://docs.docker.com/build/building/multi-platform/#why-multi-platform-builds
// and: https://docs.docker.com/dhi/explore/security-concepts/digests/#multi-platform-images-and-manifests
func GetImageMultiplatformDigest(img reference.Named, tag string) (string, error) {
tagged, err := reference.WithTag(reference.TrimNamed(img), tag)
if err != nil {
return "", errors.New(i18n.G("failed to parse image %s:%s, saw: %s", img.Name(), tag, err.Error()))
}
r, err := ref.New(tagged.String())
if err != nil {
return "", errors.New(i18n.G("failed to parse image %s:%s, saw: %s", img.Name(), tag, err.Error()))
}
rc := regclient.New(regclient.WithDockerCreds())
m, err := rc.ManifestHead(context.Background(), r)
if err != nil {
return "", errors.New(i18n.G("failed to fetch manifest for %s:%s, saw: %s", img.Name(), tag, err.Error()))
}
digest := m.GetDescriptor().Digest.String()
if digest == "" {
return "", errors.New(i18n.G("registry did not return a digest for %s:%s", img.Name(), tag))
}
return digest, nil
}
+6 -2
View File
@@ -171,7 +171,7 @@ func GetImagesForStack(cl *dockerClient.Client, app appPkg.App) (map[string]stri
images[imageBaseName] = ""
continue
}
imageTag := namedTag.Tag()
imageTag := formatter.GetTagAndDigest(namedTag)
existingImageVersion, ok := images[imageBaseName]
if !ok {
@@ -294,7 +294,7 @@ func GatherImagesForDeploy(cl *dockerClient.Client, app appPkg.App, compose *com
newImages[imageBaseName] = ""
continue
}
imageTag := namedTag.Tag()
imageTag := formatter.GetTagAndDigest(namedTag)
existingImageVersion, ok := newImages[imageBaseName]
if !ok {
@@ -312,6 +312,10 @@ func GatherImagesForDeploy(cl *dockerClient.Client, app appPkg.App, compose *com
var imageInfo []string
for newImageName, newImageVersion := range newImages {
if currentVersion, exists := currentImages[newImageName]; exists {
// Swarm adds a resolved digest even for unpinned compose images.
if !strings.Contains(newImageVersion, "@") {
currentVersion = formatter.RemoveSha(currentVersion)
}
if currentVersion == newImageVersion {
if showUnchanged {
imageInfo = append(imageInfo, i18n.G("%s: %s (unchanged)", formatter.StripTagMeta(newImageName), newImageVersion))
+17 -2
View File
@@ -11,6 +11,7 @@ import (
"coopcloud.tech/abra/pkg/i18n"
"github.com/charmbracelet/lipgloss"
"github.com/charmbracelet/lipgloss/table"
"github.com/distribution/reference"
"github.com/docker/go-units"
"golang.org/x/term"
@@ -34,11 +35,24 @@ func SmallSHA(hash string) string {
return hash[:8]
}
// RemoveSha remove image sha from a string that are added in some docker outputs
// RemoveSha remove image sha (digest suffix) from a string that are added in some docker outputs
// e.g., "image:tag@sha256:..." -> "image:tag"
func RemoveSha(str string) string {
return strings.Split(str, "@")[0]
}
// GetTagAndDigest returns the tag and any digest for comparison and display.
func GetTagAndDigest(image reference.Named) string {
var version string
if tagged, ok := image.(reference.NamedTagged); ok {
version = tagged.Tag()
}
if pinned, ok := image.(reference.Digested); ok {
version += "@" + pinned.Digest().String()
}
return version
}
// HumanDuration from docker/cli RunningFor() to be accessible outside of the class
func HumanDuration(timestamp int64) string {
date := time.Unix(timestamp, 0)
@@ -222,7 +236,8 @@ func CreateProgressbar(length int, title string) *progressbar.ProgressBar {
)
}
// StripTagMeta strips front-matter image tag data that we don't need for parsing.
// StripTagMeta strips front-matter image tag data that we don't need for parsing,
// like registry host and library prefix.
func StripTagMeta(image string) string {
originalImage := image
+26
View File
@@ -3,9 +3,35 @@ package formatter
import (
"testing"
"github.com/distribution/reference"
"github.com/stretchr/testify/assert"
)
func TestRemoveSha(t *testing.T) {
assert.Equal(t, "nginx:latest", RemoveSha("nginx:latest@sha256:1234567890abcdef"))
assert.Equal(t, "nginx:latest", RemoveSha("nginx:latest"))
assert.Equal(t, "my-repo/my-image:v1", RemoveSha("my-repo/my-image:v1@sha256:abcdef1234567890"))
assert.Equal(t, "my-repo/my-image", RemoveSha("my-repo/my-image@sha256:abcdef1234567890"))
}
func TestBoldDirtyDefault(t *testing.T) {
assert.Equal(t, "foo", BoldDirtyDefault("foo"))
}
func TestGetTagAndDigest(t *testing.T) {
digest := "sha256:908dc23e643a1447dbfb2e189ed268bfde6a51a5bf9a34d3dd3440a24f58ccf7"
for _, tc := range []struct{ image, want string }{
{"nginx:1.21.6", "1.21.6"},
{"ghcr.io/foo/bar:1.0@" + digest, "1.0@" + digest},
{"nginx@" + digest, "@" + digest},
{"nginx", ""},
} {
img, err := reference.ParseNormalizedNamed(tc.image)
if err != nil {
t.Fatal(err)
}
if got := GetTagAndDigest(img); got != tc.want {
t.Errorf("%s: got %s, want %s", tc.image, got, tc.want)
}
}
}
+161 -95
View File
@@ -7,7 +7,7 @@
msgid ""
msgstr "Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2026-10-14 10:32-0400\n"
"POT-Creation-Date: 2026-10-02 16:58-0400\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -165,7 +165,7 @@ msgid " # standard rollback\n"
" abra app rollback 1312.net 2.0.0+1.2.3"
msgstr ""
#: ./pkg/formatter/formatter.go:46
#: ./pkg/formatter/formatter.go:47
msgid " ago"
msgstr ""
@@ -219,7 +219,7 @@ msgstr ""
msgid "%s created (version: %s)"
msgstr ""
#: ./cli/recipe/upgrade.go:335
#: ./cli/recipe/upgrade.go:364
#, c-format
msgid "%s currently has these unstaged changes 👇"
msgstr ""
@@ -304,7 +304,7 @@ msgstr ""
msgid "%s has no secrets to generate, skipping..."
msgstr ""
#: ./pkg/recipe/compose.go:107
#: ./pkg/recipe/compose.go:110
#, c-format
msgid "%s has no version label? try running \"abra recipe sync %s\" first?"
msgstr ""
@@ -324,12 +324,12 @@ msgstr ""
msgid "%s is already fetched"
msgstr ""
#: ./pkg/recipe/compose.go:233
#: ./pkg/recipe/compose.go:374
#, c-format
msgid "%s is already set, nothing to do?"
msgstr ""
#: ./pkg/recipe/compose.go:72
#: ./pkg/recipe/compose.go:75
#, c-format
msgid "%s is missing a compose.yml or compose.*.yml file?"
msgstr ""
@@ -568,12 +568,12 @@ msgstr ""
msgid "--local detected, running %s on local work station"
msgstr ""
#: ./pkg/lint/recipe.go:261 ./pkg/recipe/files.go:18
#: ./pkg/lint/recipe.go:268 ./pkg/recipe/files.go:18
#, c-format
msgid ".env.sample for %s couldn't be read: %s"
msgstr ""
#: ./pkg/lint/recipe.go:133
#: ./pkg/lint/recipe.go:140
msgid ".env.sample provided"
msgstr ""
@@ -638,17 +638,17 @@ msgstr ""
msgid "CHAOS DEPLOY"
msgstr ""
#: ./pkg/recipe/compose.go:47
#: ./pkg/recipe/compose.go:50
#, c-format
msgid "COMPOSE_FILE (=\"%s\") parsing failed?"
msgstr ""
#: ./pkg/recipe/compose.go:58
#: ./pkg/recipe/compose.go:61
#, c-format
msgid "COMPOSE_FILE detected (%s), loading %s"
msgstr ""
#: ./pkg/recipe/compose.go:38
#: ./pkg/recipe/compose.go:41
#, c-format
msgid "COMPOSE_FILE detected, loading %s"
msgstr ""
@@ -739,7 +739,7 @@ msgid "Create a new version of a recipe.\n"
" ssh-add ~/.ssh/<my-ssh-private-key-for-git-coopcloud-tech>"
msgstr ""
#: ./pkg/formatter/formatter.go:134
#: ./pkg/formatter/formatter.go:135
msgid "CreateOverview: only accepts rows of len == 2"
msgstr ""
@@ -1644,7 +1644,7 @@ msgid "\n"
"\n"
msgstr ""
#: ./pkg/lint/recipe.go:203
#: ./pkg/lint/recipe.go:210
#, c-format
msgid "\n"
"lint %s: %s"
@@ -1687,7 +1687,7 @@ msgctxt "recipe fetch"
msgid "a"
msgstr ""
#: ./cli/recipe/upgrade.go:408
#: ./cli/recipe/upgrade.go:444
msgctxt "recipe upgrade"
msgid "a"
msgstr ""
@@ -1737,7 +1737,7 @@ msgstr ""
msgid "accepts at most 2 args with --local/-l"
msgstr ""
#: ./pkg/recipe/compose.go:249
#: ./pkg/recipe/compose.go:390
#, c-format
msgid "add '- \"%s\"' manually to the 'app' service in %s"
msgstr ""
@@ -1764,11 +1764,15 @@ msgstr ""
msgid "all containers did really go away"
msgstr ""
#: ./pkg/lint/recipe.go:131
msgid "all images use a digest"
msgstr ""
#: ./pkg/lint/recipe.go:82
msgid "all images use a tag"
msgstr ""
#: ./pkg/lint/recipe.go:162
#: ./pkg/lint/recipe.go:169
msgid "all services have images"
msgstr ""
@@ -1780,7 +1784,7 @@ msgstr ""
msgid "all-services"
msgstr ""
#: ./cli/recipe/upgrade.go:407
#: ./cli/recipe/upgrade.go:443
msgid "all-tags"
msgstr ""
@@ -1919,7 +1923,7 @@ msgstr ""
msgid "c"
msgstr ""
#: ./cli/recipe/upgrade.go:416
#: ./cli/recipe/upgrade.go:452
msgctxt "recipe upgrade"
msgid "c"
msgstr ""
@@ -1928,7 +1932,7 @@ msgstr ""
msgid "can not insert from file and read from stdin"
msgstr ""
#: ./cli/recipe/upgrade.go:323
#: ./cli/recipe/upgrade.go:352
#, c-format
msgid "can upgrade service: %s, image: %s, tag: %s ::"
msgstr ""
@@ -2108,7 +2112,7 @@ msgstr ""
msgid "chore: publish new catalogue release changes"
msgstr ""
#: ./cli/recipe/upgrade.go:352
#: ./cli/recipe/upgrade.go:381
msgid "chore: update image tags"
msgstr ""
@@ -2170,19 +2174,19 @@ msgstr ""
msgid "commandconn: starting %s with %v"
msgstr ""
#: ./cli/recipe/upgrade.go:415
#: ./cli/recipe/upgrade.go:451
msgid "commit"
msgstr ""
#: ./cli/recipe/upgrade.go:418
#: ./cli/recipe/upgrade.go:454
msgid "commit changes"
msgstr ""
#: ./cli/recipe/upgrade.go:342
#: ./cli/recipe/upgrade.go:371
msgid "commit changes?"
msgstr ""
#: ./cli/recipe/upgrade.go:356
#: ./cli/recipe/upgrade.go:385
#, c-format
msgid "committed changes as '%s'"
msgstr ""
@@ -2221,7 +2225,7 @@ msgstr ""
msgid "config not found: %s"
msgstr ""
#: ./pkg/lint/recipe.go:169
#: ./pkg/lint/recipe.go:176
msgid "config version are vendored"
msgstr ""
@@ -2230,14 +2234,14 @@ msgstr ""
msgid "connection timed out for %s"
msgstr ""
#: ./pkg/recipe/compose.go:191
#: ./pkg/recipe/compose.go:146
#, c-format
msgid "considering %s config(s) for label update"
msgid "considering %s config(s) for image update"
msgstr ""
#: ./pkg/recipe/compose.go:123
#: ./pkg/recipe/compose.go:332
#, c-format
msgid "considering %s config(s) for tag update"
msgid "considering %s config(s) for label update"
msgstr ""
#: ./cli/app/undeploy.go:141 ./cli/server/prune.go:53
@@ -2258,7 +2262,7 @@ msgstr ""
msgid "context lacks Docker endpoint"
msgstr ""
#: ./pkg/recipe/compose.go:229
#: ./pkg/recipe/compose.go:370
#, c-format
msgid "coop-cloud.${STACK_NAME}.version=%s"
msgstr ""
@@ -2303,7 +2307,7 @@ msgstr ""
msgid "create <domain> [flags]"
msgstr ""
#: ./pkg/lint/recipe.go:134
#: ./pkg/lint/recipe.go:141
msgid "create an example .env.sample"
msgstr ""
@@ -2418,7 +2422,7 @@ msgstr ""
msgid "deploy in progress 🟠"
msgstr ""
#: ./pkg/lint/recipe.go:147
#: ./pkg/lint/recipe.go:154
msgid "deploy labels stanza present"
msgstr ""
@@ -2461,11 +2465,11 @@ msgstr ""
msgid "detected %s as tags for recipe %s"
msgstr ""
#: ./pkg/formatter/formatter.go:80
#: ./pkg/formatter/formatter.go:81
msgid "detected ABRA_CI=1"
msgstr ""
#: ./cli/recipe/upgrade.go:204
#: ./cli/recipe/upgrade.go:208
#, c-format
msgid "detected compatible upgradable tags %s for %s"
msgstr ""
@@ -2644,7 +2648,7 @@ msgstr ""
msgid "ensure 'version: \"3.8\"' in compose configs"
msgstr ""
#: ./pkg/lint/recipe.go:163
#: ./pkg/lint/recipe.go:170
msgid "ensure \"image: ...\" set on all services"
msgstr ""
@@ -2658,7 +2662,7 @@ msgstr ""
msgid "ensuring env version %s"
msgstr ""
#: ./cli/recipe/upgrade.go:285
#: ./cli/recipe/upgrade.go:291
msgid "enter / return to confirm, choose 'skip' to not upgrade this tag, vim mode is enabled"
msgstr ""
@@ -2682,7 +2686,7 @@ msgstr ""
msgid "err: %v, "
msgstr ""
#: ./cli/recipe/lint.go:82 ./cli/recipe/lint.go:144 ./pkg/lint/recipe.go:132 ./pkg/lint/recipe.go:139 ./pkg/lint/recipe.go:146 ./pkg/lint/recipe.go:153 ./pkg/lint/recipe.go:161 ./pkg/lint/recipe.go:168 ./pkg/lint/recipe.go:175
#: ./cli/recipe/lint.go:82 ./cli/recipe/lint.go:144 ./pkg/lint/recipe.go:139 ./pkg/lint/recipe.go:146 ./pkg/lint/recipe.go:153 ./pkg/lint/recipe.go:160 ./pkg/lint/recipe.go:168 ./pkg/lint/recipe.go:175 ./pkg/lint/recipe.go:182
msgid "error"
msgstr ""
@@ -2801,6 +2805,11 @@ msgstr ""
msgid "failed to create volume %s on %s: %s"
msgstr ""
#: ./pkg/client/registry.go:54
#, c-format
msgid "failed to fetch manifest for %s:%s, saw: %s"
msgstr ""
#: ./pkg/secret/secret.go:104
#, c-format
msgid "failed to generate random bytes: %w"
@@ -2839,11 +2848,16 @@ msgstr ""
msgid "failed to migrate app config: %s"
msgstr ""
#: ./pkg/client/registry.go:20
#: ./pkg/client/registry.go:22
#, c-format
msgid "failed to parse image %s, saw: %s"
msgstr ""
#: ./pkg/client/registry.go:45
#, c-format
msgid "failed to parse image %s:%s, saw: %s"
msgstr ""
#: ./pkg/recipe/git.go:437
#, c-format
msgid "failed to parse image for %s in %s: %s"
@@ -2898,6 +2912,11 @@ msgstr ""
msgid "failed to resize tty, using default size"
msgstr ""
#: ./cli/recipe/upgrade.go:313
#, c-format
msgid "failed to resolve digest for %s:%s: %s"
msgstr ""
#: ./cli/app/new.go:138
#, c-format
msgid "failed to retrieve latest commit for %s: %s"
@@ -3206,16 +3225,26 @@ msgstr ""
msgid "ignore uncommitted recipes changes"
msgstr ""
#: ./pkg/recipe/compose.go:248
#, c-format
msgid "image %s for %s uses a variable, skipping update, please update it manually"
msgstr ""
#: ./pkg/recipe/compose.go:183
#, c-format
msgid "image for %s uses a variable, skipping update, please update %s manually"
msgstr ""
#: ./cli/app/undeploy.go:156 ./cli/server/prune.go:74
#, c-format
msgid "images pruned: %d; space reclaimed: %s"
msgstr ""
#: ./pkg/lint/recipe.go:148
#: ./pkg/lint/recipe.go:155
msgid "include \"deploy: labels: ...\" stanza"
msgstr ""
#: ./pkg/lint/recipe.go:155
#: ./pkg/lint/recipe.go:162
msgid "include \"traefik.enable=true\" deploy label"
msgstr ""
@@ -3291,15 +3320,15 @@ msgstr ""
msgid "including VOLUMES=%v in backupbot exec invocation"
msgstr ""
#: ./cli/recipe/release.go:634 ./cli/recipe/upgrade.go:378
#: ./cli/recipe/release.go:634 ./cli/recipe/upgrade.go:414
msgid "increase the major part of the version"
msgstr ""
#: ./cli/recipe/release.go:642 ./cli/recipe/upgrade.go:386
#: ./cli/recipe/release.go:642 ./cli/recipe/upgrade.go:422
msgid "increase the minor part of the version"
msgstr ""
#: ./cli/recipe/release.go:650 ./cli/recipe/upgrade.go:394
#: ./cli/recipe/release.go:650 ./cli/recipe/upgrade.go:430
msgid "increase the patch part of the version"
msgstr ""
@@ -3369,7 +3398,12 @@ msgstr ""
msgid "invalid detach escape keys, using default: %s"
msgstr ""
#: ./pkg/lint/recipe.go:504
#: ./pkg/recipe/compose.go:308
#, c-format
msgid "invalid digest %s, want sha256:<64 hex digits>"
msgstr ""
#: ./pkg/lint/recipe.go:533
msgid "invalid lightweight tag detected"
msgstr ""
@@ -3430,12 +3464,12 @@ msgstr ""
msgid "lint <recipe> [flags]"
msgstr ""
#: ./pkg/lint/recipe.go:187
#: ./pkg/lint/recipe.go:194
#, c-format
msgid "linting for critical errors in %s configs"
msgstr ""
#: ./pkg/lint/recipe.go:215
#: ./pkg/lint/recipe.go:222
#, c-format
msgid "linting successful, %s is well configured"
msgstr ""
@@ -3464,7 +3498,7 @@ msgstr ""
msgid "list [flags]"
msgstr ""
#: ./cli/recipe/upgrade.go:410
#: ./cli/recipe/upgrade.go:446
msgid "list all tags, not just upgrades"
msgstr ""
@@ -3534,11 +3568,11 @@ msgstr ""
#. with no spaces in between
#. translators: `abra man` aliases. use a comma separated list of aliases
#. with no spaces in between
#: ./cli/app/list.go:319 ./cli/app/move.go:34 ./cli/app/ps.go:205 ./cli/app/secret.go:553 ./cli/app/secret.go:649 ./cli/recipe/list.go:104 ./cli/recipe/upgrade.go:400 ./cli/recipe/version.go:139 ./cli/run.go:152 ./cli/server/list.go:106
#: ./cli/app/list.go:319 ./cli/app/move.go:34 ./cli/app/ps.go:205 ./cli/app/secret.go:553 ./cli/app/secret.go:649 ./cli/recipe/list.go:104 ./cli/recipe/upgrade.go:436 ./cli/recipe/version.go:139 ./cli/run.go:152 ./cli/server/list.go:106
msgid "m"
msgstr ""
#: ./cli/app/list.go:318 ./cli/app/ps.go:204 ./cli/app/secret.go:552 ./cli/app/secret.go:648 ./cli/recipe/list.go:103 ./cli/recipe/upgrade.go:399 ./cli/recipe/version.go:138 ./cli/server/list.go:105
#: ./cli/app/list.go:318 ./cli/app/ps.go:204 ./cli/app/secret.go:552 ./cli/app/secret.go:648 ./cli/recipe/list.go:103 ./cli/recipe/upgrade.go:435 ./cli/recipe/version.go:138 ./cli/server/list.go:105
msgid "machine"
msgstr ""
@@ -3547,7 +3581,7 @@ msgstr ""
msgid "main app service version for %s is empty?"
msgstr ""
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:66 ./cli/internal/recipe.go:80 ./cli/recipe/release.go:631 ./cli/recipe/upgrade.go:375
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:66 ./cli/internal/recipe.go:80 ./cli/recipe/release.go:631 ./cli/recipe/upgrade.go:411
msgid "major"
msgstr ""
@@ -3576,7 +3610,7 @@ msgstr ""
msgid "migrating app config from %s to %s"
msgstr ""
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:68 ./cli/internal/recipe.go:82 ./cli/recipe/release.go:639 ./cli/recipe/upgrade.go:383
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:68 ./cli/internal/recipe.go:82 ./cli/recipe/release.go:639 ./cli/recipe/upgrade.go:419
msgid "minor"
msgstr ""
@@ -3627,7 +3661,7 @@ msgstr ""
msgid "name"
msgstr ""
#: ./pkg/lint/recipe.go:141
#: ./pkg/lint/recipe.go:148
msgid "name a servce 'app'"
msgstr ""
@@ -3705,7 +3739,7 @@ msgstr ""
msgid "no .git directory in %s?"
msgstr ""
#: ./pkg/recipe/compose.go:29
#: ./pkg/recipe/compose.go:32
#, c-format
msgid "no COMPOSE_FILE detected, loading default: %s"
msgstr ""
@@ -3731,7 +3765,7 @@ msgstr ""
msgid "no changes discovered in %s, nothing to publish?"
msgstr ""
#: ./cli/recipe/upgrade.go:361
#: ./cli/recipe/upgrade.go:390
msgid "no changes, skip creating commit"
msgstr ""
@@ -3756,7 +3790,7 @@ msgstr ""
msgid "no domain provided"
msgstr ""
#: ./pkg/recipe/compose.go:248
#: ./pkg/recipe/compose.go:389
msgid "no existing label found, automagic insertion not supported yet"
msgstr ""
@@ -3765,7 +3799,7 @@ msgstr ""
msgid "no git tags found for %s"
msgstr ""
#: ./cli/recipe/upgrade.go:182
#: ./cli/recipe/upgrade.go:183
#, c-format
msgid "no new versions available for %s, assuming %s is the latest (use -a/--all-tags to see all anyway)"
msgstr ""
@@ -3897,12 +3931,12 @@ msgstr ""
msgid "not requesting a remote TTY"
msgstr ""
#: ./cli/recipe/upgrade.go:305
#: ./cli/recipe/upgrade.go:334
#, c-format
msgid "not upgrading %s, skipping as requested"
msgstr ""
#: ./cli/recipe/upgrade.go:244
#: ./cli/recipe/upgrade.go:247
#, c-format
msgid "not upgrading from %s to %s for %s, because the upgrade type is more serious than what user wants"
msgstr ""
@@ -3928,11 +3962,11 @@ msgstr ""
msgid "one of <src>/<dest> arguments must take $SERVICE:$PATH form"
msgstr ""
#: ./pkg/lint/recipe.go:140
#: ./pkg/lint/recipe.go:147
msgid "one service named 'app'"
msgstr ""
#: ./pkg/lint/recipe.go:176
#: ./pkg/lint/recipe.go:183
msgid "only annotated tags used for recipe version"
msgstr ""
@@ -3956,16 +3990,16 @@ msgstr ""
msgid "p"
msgstr ""
#: ./cli/recipe/upgrade.go:460
msgctxt "pin container image by digest"
msgid "p"
msgstr ""
#: ./cli/recipe/upgrade.go:163
#, c-format
msgid "parsed %s for %s"
msgstr ""
#: ./pkg/recipe/compose.go:159
#, c-format
msgid "parsed %s from %s"
msgstr ""
#: ./cli/app/cmd.go:134 ./cli/app/cmd.go:180
#, c-format
msgid "parsed following command arguments: %s"
@@ -3999,7 +4033,7 @@ msgstr ""
msgid "pass command not found on $PATH, is it installed?"
msgstr ""
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:70 ./cli/internal/recipe.go:84 ./cli/recipe/release.go:647 ./cli/recipe/upgrade.go:391
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:70 ./cli/internal/recipe.go:84 ./cli/recipe/release.go:647 ./cli/recipe/upgrade.go:427
msgid "patch"
msgstr ""
@@ -4015,6 +4049,14 @@ msgstr ""
msgid "perform action without further prompt"
msgstr ""
#: ./cli/recipe/upgrade.go:462
msgid "pin the container image version by manifest digest (image:tag@sha256:...)"
msgstr ""
#: ./cli/recipe/upgrade.go:459
msgid "pindigests"
msgstr ""
#. translators: `abra app env pull` aliases. use a comma separated list of
#. aliases with no spaces in between
#: ./cli/app/env.go:39
@@ -4053,7 +4095,7 @@ msgstr ""
msgid "prefer offline & filesystem access"
msgstr ""
#: ./cli/app/list.go:321 ./cli/app/ps.go:207 ./cli/app/secret.go:555 ./cli/app/secret.go:651 ./cli/recipe/list.go:106 ./cli/recipe/upgrade.go:402 ./cli/recipe/version.go:141 ./cli/server/list.go:108
#: ./cli/app/list.go:321 ./cli/app/ps.go:207 ./cli/app/secret.go:555 ./cli/app/secret.go:651 ./cli/recipe/list.go:106 ./cli/recipe/upgrade.go:438 ./cli/recipe/version.go:141 ./cli/server/list.go:108
msgid "print machine-readable output"
msgstr ""
@@ -4206,7 +4248,7 @@ msgstr ""
msgid "recipe %s does not exist?"
msgstr ""
#: ./pkg/lint/recipe.go:212
#: ./pkg/lint/recipe.go:219
#, c-format
msgid "recipe '%s' failed lint checks:\n"
"%s"
@@ -4230,6 +4272,11 @@ msgstr ""
msgid "ref"
msgstr ""
#: ./pkg/client/registry.go:59
#, c-format
msgid "registry did not return a digest for %s:%s"
msgstr ""
#. translators: `recipe release` command
#: ./cli/recipe/release.go:35
msgid "release <recipe> [version] [flags]"
@@ -4328,6 +4375,11 @@ msgstr ""
msgid "removing configs failed: %s"
msgstr ""
#: ./pkg/recipe/compose.go:122
#, c-format
msgid "removing digest from %s for %s, use --pindigests to pin the new tag"
msgstr ""
#: ./pkg/upstream/stack/remove.go:148
#, c-format
msgid "removing network %s"
@@ -4353,7 +4405,7 @@ msgstr ""
msgid "removing volumes failed: %s"
msgstr ""
#: ./pkg/lint/recipe.go:177
#: ./pkg/lint/recipe.go:184
msgid "replace lightweight tag with annotated tag"
msgstr ""
@@ -4436,7 +4488,7 @@ msgstr ""
msgid "retrieved %s as target container on %s"
msgstr ""
#: ./pkg/recipe/compose.go:59
#: ./pkg/recipe/compose.go:62
#, c-format
msgid "retrieved %s configs for %s"
msgstr ""
@@ -4475,7 +4527,7 @@ msgstr ""
msgid "retrieving docker auth token: failed create docker cli: %s"
msgstr ""
#: ./pkg/recipe/recipe.go:656
#: ./pkg/recipe/recipe.go:659
msgid "retrieving recipes"
msgstr ""
@@ -4607,7 +4659,7 @@ msgstr ""
msgid "secret %s is > %d chars when combined with %s"
msgstr ""
#: ./pkg/lint/recipe.go:481
#: ./pkg/lint/recipe.go:510
#, c-format
msgid "secret %s is longer than 12 characters"
msgstr ""
@@ -4723,12 +4775,12 @@ msgstr ""
msgid "serverAdd: cleanUp: unable to list files in %s: %s"
msgstr ""
#: ./cli/recipe/upgrade.go:227
#: ./cli/recipe/upgrade.go:230
#, c-format
msgid "service %s is at version %s, but pinned to %s, please correct your compose.yml file manually!"
msgstr ""
#: ./cli/recipe/upgrade.go:223
#: ./cli/recipe/upgrade.go:226
#, c-format
msgid "service %s, image %s pinned to %s, no compatible upgrade found"
msgstr ""
@@ -4969,7 +5021,7 @@ msgstr ""
msgid "store secrets in a local pass store"
msgstr ""
#: ./pkg/formatter/formatter.go:238
#: ./pkg/formatter/formatter.go:239
#, c-format
msgid "stripped %s to %s for parsing"
msgstr ""
@@ -5007,7 +5059,7 @@ msgstr ""
msgid "sync [flags]"
msgstr ""
#: ./pkg/recipe/compose.go:243
#: ./pkg/recipe/compose.go:384
#, c-format
msgid "synced label %s to service %s"
msgstr ""
@@ -5025,7 +5077,12 @@ msgstr ""
msgid "tag at commit %s is unannotated or otherwise broken"
msgstr ""
#: ./cli/recipe/upgrade.go:301
#: ./cli/recipe/upgrade.go:327
#, c-format
msgid "tag upgraded and pinned from %s to %s@%s for %s"
msgstr ""
#: ./cli/recipe/upgrade.go:329
#, c-format
msgid "tag upgraded from %s to %s for %s"
msgstr ""
@@ -5099,7 +5156,7 @@ msgstr ""
msgid "toggle non-interactive mode"
msgstr ""
#: ./pkg/lint/recipe.go:154
#: ./pkg/lint/recipe.go:161
msgid "traefik routing enabled"
msgstr ""
@@ -5212,7 +5269,7 @@ msgstr ""
msgid "unable to determine server of app %s, please pass --server/-s"
msgstr ""
#: ./cli/recipe/upgrade.go:252
#: ./cli/recipe/upgrade.go:258
#, c-format
msgid "unable to determine versioning semantics of %s, listing all tags"
msgstr ""
@@ -5227,6 +5284,11 @@ msgstr ""
msgid "unable to fetch tags in %s: %s"
msgstr ""
#: ./pkg/recipe/compose.go:238 ./pkg/recipe/compose.go:245 ./pkg/recipe/compose.go:255 ./pkg/recipe/compose.go:260
#, c-format
msgid "unable to find image %s for %s, please update it manually"
msgstr ""
#: ./cli/app/move.go:301
#, c-format
msgid "unable to get container matching %s: %s"
@@ -5247,7 +5309,7 @@ msgstr ""
msgid "unable to inspect container for %s: %s"
msgstr ""
#: ./pkg/lint/recipe.go:496
#: ./pkg/lint/recipe.go:525
#, c-format
msgid "unable to list local tags for %s"
msgstr ""
@@ -5261,7 +5323,7 @@ msgstr ""
msgid "unable to look up server context for %s: %s"
msgstr ""
#: ./cli/recipe/fetch.go:77 ./pkg/git/read.go:26 ./pkg/lint/recipe.go:491 ./pkg/recipe/git.go:243
#: ./cli/recipe/fetch.go:77 ./pkg/git/read.go:26 ./pkg/lint/recipe.go:520 ./pkg/recipe/git.go:243
#, c-format
msgid "unable to open %s: %s"
msgstr ""
@@ -5291,11 +5353,6 @@ msgstr ""
msgid "unable to parse %s, error was: %s, skipping upgrade for %s"
msgstr ""
#: ./pkg/recipe/compose.go:153
#, c-format
msgid "unable to parse %s, skipping"
msgstr ""
#: ./cli/app/list.go:167
#, c-format
msgid "unable to parse %s, skipping as upgrade option"
@@ -5311,6 +5368,11 @@ msgstr ""
msgid "unable to parse deploy version: %s"
msgstr ""
#: ./pkg/recipe/compose.go:305
#, c-format
msgid "unable to parse image %s: %s"
msgstr ""
#: ./cli/run.go:176
#, c-format
msgid "unable to proceed, %s does not exist?"
@@ -5518,7 +5580,7 @@ msgstr ""
msgid "updating %s"
msgstr ""
#: ./pkg/recipe/compose.go:171 ./pkg/recipe/compose.go:237
#: ./pkg/recipe/compose.go:203 ./pkg/recipe/compose.go:378
#, c-format
msgid "updating %s to %s in %s"
msgstr ""
@@ -5543,17 +5605,17 @@ msgstr ""
msgid "upgrade [flags]"
msgstr ""
#: ./cli/recipe/upgrade.go:248
#: ./cli/recipe/upgrade.go:254
#, c-format
msgid "upgrade to which tag? (service: %s, image: %s, tag: %s)"
msgstr ""
#: ./cli/recipe/upgrade.go:254
#: ./cli/recipe/upgrade.go:260
#, c-format
msgid "upgrade to which tag? (service: %s, tag: %s)"
msgstr ""
#: ./cli/recipe/upgrade.go:221
#: ./cli/recipe/upgrade.go:224
#, c-format
msgid "upgrading service %s from %s to %s (pinned tag: %s)"
msgstr ""
@@ -5562,6 +5624,10 @@ msgstr ""
msgid "upload your recipe to git.coopcloud.tech/coop-cloud/..."
msgstr ""
#: ./pkg/lint/recipe.go:132
msgid "use a digest for all images"
msgstr ""
#: ./pkg/lint/recipe.go:83
msgid "use a tag for all images"
msgstr ""
@@ -5612,7 +5678,7 @@ msgstr ""
msgid "validated %s as server argument"
msgstr ""
#: ./pkg/lint/recipe.go:170
#: ./pkg/lint/recipe.go:177
msgid "vendor config versions in an abra.sh"
msgstr ""
@@ -5763,7 +5829,7 @@ msgstr ""
msgid "waiting for %d task(s) to reach terminal state"
msgstr ""
#: ./pkg/lint/recipe.go:21 ./pkg/lint/recipe.go:67 ./pkg/lint/recipe.go:74 ./pkg/lint/recipe.go:81 ./pkg/lint/recipe.go:88 ./pkg/lint/recipe.go:95 ./pkg/lint/recipe.go:102 ./pkg/lint/recipe.go:109 ./pkg/lint/recipe.go:116 ./pkg/lint/recipe.go:123
#: ./pkg/lint/recipe.go:21 ./pkg/lint/recipe.go:67 ./pkg/lint/recipe.go:74 ./pkg/lint/recipe.go:81 ./pkg/lint/recipe.go:88 ./pkg/lint/recipe.go:95 ./pkg/lint/recipe.go:102 ./pkg/lint/recipe.go:109 ./pkg/lint/recipe.go:116 ./pkg/lint/recipe.go:123 ./pkg/lint/recipe.go:130
msgid "warn"
msgstr ""
@@ -5806,11 +5872,11 @@ msgstr ""
msgid "writing recipe version failed: %s"
msgstr ""
#: ./cli/recipe/release.go:632 ./cli/recipe/upgrade.go:376
#: ./cli/recipe/release.go:632 ./cli/recipe/upgrade.go:412
msgid "x"
msgstr ""
#: ./cli/recipe/release.go:640 ./cli/recipe/upgrade.go:384
#: ./cli/recipe/release.go:640 ./cli/recipe/upgrade.go:420
msgid "y"
msgstr ""
@@ -5822,7 +5888,7 @@ msgstr ""
msgid "you can only use one version flag: --major, --minor or --patch"
msgstr ""
#: ./cli/recipe/release.go:648 ./cli/recipe/upgrade.go:392
#: ./cli/recipe/release.go:648 ./cli/recipe/upgrade.go:428
msgid "z"
msgstr ""
Binary file not shown.
+673 -312
View File
File diff suppressed because it is too large Load Diff
+29
View File
@@ -125,6 +125,13 @@ var LintRules = map[string][]LintRule{
HowToResolve: i18n.G("reduce length of secret names to 12 chars"),
Function: LintSecretLengths,
},
{
Ref: "R016",
Level: i18n.G("warn"),
Description: i18n.G("all images use a digest"),
HowToResolve: i18n.G("use a digest for all images"),
Function: LintAllImagesTaggedWithDigest,
},
},
"error": {
{
@@ -333,6 +340,28 @@ func LintAllImagesTagged(recipe recipe.Recipe) (bool, error) {
return true, nil
}
func LintAllImagesTaggedWithDigest(recipe recipe.Recipe) (bool, error) {
config, err := recipe.GetComposeConfig(nil)
if err != nil {
return false, err
}
for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(service.Image)
if err != nil {
return false, err
}
if reference.IsNameOnly(img) {
return false, nil
}
_, ok := img.(reference.Digested)
if !ok {
return false, nil
}
}
return true, nil
}
func LintNoUnstableTags(recipe recipe.Recipe) (bool, error) {
config, err := recipe.GetComposeConfig(nil)
if err != nil {
+82
View File
@@ -0,0 +1,82 @@
package recipe
import (
"encoding/json"
"strings"
"testing"
"coopcloud.tech/abra/pkg/config"
gitPkg "coopcloud.tech/abra/pkg/git"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
)
// Use the test recipe at the given commit, in a temporary recipes directory.
func setupTestRecipe(t *testing.T, commit string) Recipe {
t.Helper()
// Other tests in this package use the real RECIPES_DIR.
recipesDir := config.RECIPES_DIR
config.RECIPES_DIR = t.TempDir()
t.Cleanup(func() { config.RECIPES_DIR = recipesDir })
r := Get("abra-test-recipe")
if err := gitPkg.Clone(r.Dir, "https://git.coopcloud.tech/toolshed/abra-test-recipe"); err != nil {
t.Fatal(err)
}
if _, err := r.EnsureVersion(commit); err != nil {
t.Fatal(err)
}
return r
}
func TestGetRecipeVersionsIncludesDigest(t *testing.T) {
if testing.Short() {
t.Skip("skipping e2e test in short mode")
}
// https://git.coopcloud.tech/toolshed/abra-test-recipe/commit/4f0642719d2e2a568f4350cf7eea182438d58be3
r := setupTestRecipe(t, "4f0642719d2e2a568f4350cf7eea182438d58be3")
repo, err := git.PlainOpen(r.Dir)
if err != nil {
t.Fatal(err)
}
for version, commit := range map[string]string{
"9.0.0+1.21.5": "4f0642719d2e2a568f4350cf7eea182438d58be3",
"9.0.1+1.21.6": "77d752b3c61a71bfb2c45f40ec07c9d626a0e580",
} {
if _, err := repo.CreateTag(version, plumbing.NewHash(commit), nil); err != nil {
t.Fatal(err)
}
}
versions, _, err := r.GetRecipeVersions()
if err != nil {
t.Fatal(err)
}
found := map[string]ServiceMeta{}
for _, version := range versions {
for tag, services := range version {
if strings.HasPrefix(tag, "9.0.") {
found[tag] = services["app"]
}
}
}
pinned := found["9.0.0+1.21.5"]
if pinned.Tag != "1.21.5" || pinned.Digest != "sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31" {
t.Fatalf("got pinned metadata %+v", pinned)
}
unpinned := found["9.0.1+1.21.6"]
if unpinned.Tag != "1.21.6" || unpinned.Digest != "" {
t.Fatalf("got unpinned metadata %+v", unpinned)
}
for _, tc := range []struct {
meta ServiceMeta
wantDigest bool
}{{pinned, true}, {unpinned, false}} {
encoded, err := json.Marshal(tc.meta)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(encoded), `"digest"`) != tc.wantDigest {
t.Fatalf("unexpected JSON %s", encoded)
}
}
}
+42 -13
View File
@@ -6,6 +6,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"
"coopcloud.tech/abra/pkg/formatter"
@@ -110,10 +111,23 @@ func (r Recipe) GetVersionLabelLocal() (string, error) {
return label, nil
}
var sha256Digest = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`)
// UpdateTag updates an image tag in-place on file system local compose files.
func (r Recipe) UpdateTag(image, tag string) (bool, error) {
// A non-empty digest pins the tag (image:tag@sha256:...).
func (r Recipe) UpdateTag(image, tag, digest string) (bool, error) {
// Optional parameter, but must be a valid sha256 digest if provided.
if digest != "" && !sha256Digest.MatchString(digest) {
return false, errors.New(i18n.G("invalid digest %s, want sha256:<64 hex digits>", digest))
}
fullPattern := fmt.Sprintf("%s/compose**yml", r.Dir)
image = formatter.StripTagMeta(image)
source, parseErr := reference.ParseNormalizedNamed(image)
_, exactImage := source.(reference.NamedTagged)
if parseErr != nil {
return false, parseErr
}
imageName := formatter.StripTagMeta(reference.Path(source))
composeFiles, err := filepath.Glob(fullPattern)
if err != nil {
@@ -122,6 +136,7 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
log.Debug(i18n.G("considering %s config(s) for tag update", strings.Join(composeFiles, ", ")))
updated := false
for _, composeFile := range composeFiles {
opts := stack.Deploy{Composefiles: []string{composeFile}}
@@ -140,44 +155,58 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
continue // may be a compose.$optional.yml file
}
img, _ := reference.ParseNormalizedNamed(service.Image)
img, err := reference.ParseNormalizedNamed(formatter.RemoveSha(service.Image))
if err != nil {
return false, err
}
var composeTag string
switch img.(type) {
case reference.NamedTagged:
composeTag = img.(reference.NamedTagged).Tag()
default:
log.Debug(i18n.G("unable to parse %s, skipping", img))
continue
}
// Found image that needs replacing
composeImage := formatter.StripTagMeta(reference.Path(img))
log.Debug(i18n.G("parsed %s from %s", composeTag, service.Image))
if image == composeImage {
if imageName == composeImage && (!exactImage || service.Image == image) {
bytes, err := ioutil.ReadFile(composeFile)
if err != nil {
return false, err
}
old := fmt.Sprintf("%s:%s", composeImage, composeTag)
new := fmt.Sprintf("%s:%s", composeImage, tag)
replacedBytes := strings.Replace(string(bytes), old, new, -1)
old := service.Image
tagged, err := reference.WithTag(reference.TrimNamed(img), tag)
if err != nil {
return false, err
}
new := reference.FamiliarString(tagged)
if digest != "" {
new += "@" + digest
}
if old == new {
continue
}
// Don't confuse digests if the image isn't an exact match. E.g. if old is postgres:13, don't
// match postgres:13-alpine
pattern := regexp.MustCompile(`(?m)(^\s*image:\s*["']?)` + regexp.QuoteMeta(old) + `(["']?\s*(?:#.*)?$)`)
replacedBytes := pattern.ReplaceAllString(string(bytes), "${1}"+new+"${2}")
if replacedBytes == string(bytes) {
continue
}
log.Debug(i18n.G("updating %s to %s in %s", old, new, compose.Filename))
if err := os.WriteFile(compose.Filename, []byte(replacedBytes), 0o764); err != nil {
return false, err
}
updated = true
}
}
}
return false, nil
return updated, nil
}
// UpdateLabel updates a label in-place on file system local compose files.
+7 -6
View File
@@ -164,7 +164,7 @@ func (r Recipe) EnsureVersion(version string) (bool, error) {
hash, err := repo.ResolveRevision(plumbing.Revision(version))
if err != nil {
log.Fatal(i18n.G("unable to resolve '%s': %s", version, err))
return isChaosCommit, errors.New(i18n.G("unable to resolve '%s': %s", version, err))
}
opts = &git.CheckoutOptions{Hash: *hash, Create: false, Force: true}
@@ -175,11 +175,11 @@ func (r Recipe) EnsureVersion(version string) (bool, error) {
worktree, err := repo.Worktree()
if err != nil {
return isChaosCommit, nil
return isChaosCommit, err
}
if err := worktree.Checkout(opts); err != nil {
return isChaosCommit, nil
return isChaosCommit, err
}
log.Debug(i18n.G("successfully checked %s out to %s in %s", r.Name, tagRef.Short(), r.Dir))
@@ -452,10 +452,11 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, []string, error) {
continue
}
versionMeta[service.Name] = ServiceMeta{
Image: path,
Tag: tag,
meta := ServiceMeta{Image: path, Tag: tag}
if pinned, ok := img.(reference.Digested); ok {
meta.Digest = pinned.Digest().String()
}
versionMeta[service.Name] = meta
}
versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})
+21 -17
View File
@@ -39,8 +39,9 @@ type service = string
// ServiceMeta represents meta info associated with a service.
type ServiceMeta struct {
Image string `json:"image"`
Tag string `json:"tag"`
Image string `json:"image"`
Tag string `json:"tag"`
Digest string `json:"digest,omitempty"`
}
// RecipeVersions are the versions associated with a recipe.
@@ -592,7 +593,15 @@ func ReadReposMetadata(debug bool) (RepoCatalogue, error) {
// sortRecipeVersions sorts the recipe semver versions
func sortRecipeVersions(versions RecipeVersions) {
sort.Slice(versions, func(i, j int) bool {
return safeCompareVersions(getVersionString(versions[i]), getVersionString(versions[j]))
version1, err := tagcmp.Parse(getVersionString(versions[i]))
if err != nil {
panic(err)
}
version2, err := tagcmp.Parse(getVersionString(versions[j]))
if err != nil {
panic(err)
}
return version1.IsLessThan(version2)
})
}
@@ -608,23 +617,18 @@ func getVersionString(versionMap map[string]map[string]ServiceMeta) string {
// sortVersionStrings sorts a list of semver version strings
func sortVersionStrings(versions []string) {
sort.Slice(versions, func(i, j int) bool {
return safeCompareVersions(versions[i], versions[j])
version1, err := tagcmp.Parse(versions[i])
if err != nil {
panic(err)
}
version2, err := tagcmp.Parse(versions[j])
if err != nil {
panic(err)
}
return version1.IsLessThan(version2)
})
}
// Invalid versions are sorted as less than valid ones.
func safeCompareVersions(v1, v2 string) bool {
version1, err1 := tagcmp.Parse(v1)
if err1 != nil {
return true
}
version2, err2 := tagcmp.Parse(v2)
if err2 != nil {
return false
}
return version1.IsLessThan(version2)
}
// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
var versions []string
-38
View File
@@ -123,41 +123,3 @@ func TestGetVersionLabelLocalDoesNotUseTimeoutLabel(t *testing.T) {
assert.NotEqual(t, label, timeout)
}
}
// This is not a full test of comparison, as that occurs in the external
// tagcmp package. This is a sunny-day integration test
func TestSortRecipeVersionsSortsTwoValidStrings(t *testing.T) {
versions := RecipeVersions{
{"2.0.0": {}}, // Index 0
{"1.0.0": {}}, // Index 1
}
sortRecipeVersions(versions)
assert.Equal(t, "1.0.0", getVersionString(versions[0]), "Version 1.0.0 should be first")
assert.Equal(t, "2.0.0", getVersionString(versions[1]), "Version 2.0.0 should be second")
}
func TestSortRecipeVersionsSortsInvalidVersionStringToTheBottom(t *testing.T) {
versions := RecipeVersions{
{"2.0.0": {}}, // Index 0
{"invalid": {}}, // Index 1
}
sortRecipeVersions(versions)
assert.Equal(t, "invalid", getVersionString(versions[0]), "The invalid version should be first")
assert.Equal(t, "2.0.0", getVersionString(versions[1]), "Version 2.0.0 should be second")
}
func TestSortRecipeVersionsSortsInvalidVersionStringToTheBottomInReverse(t *testing.T) {
versions := RecipeVersions{
{"invalid": {}}, // Index 0
{"2.0.0": {}}, // Index 1
}
sortRecipeVersions(versions)
assert.Equal(t, "invalid", getVersionString(versions[0]), "The invalid version should be first")
assert.Equal(t, "2.0.0", getVersionString(versions[1]), "Version 2.0.0 should be second")
}
+1 -1
View File
@@ -3,7 +3,7 @@ version: "3.8"
services:
app:
image: nginx:1.31.8
image: nginx:1.31.6
secrets:
- test_pass_one
- test_pass_two
+2 -1
View File
@@ -7,4 +7,5 @@
* Integration tests are in `./tests/integration`. Please see [these
docs](https://docs.coopcloud.tech/abra/hack/#integration-tests) for
instructions and tips on how to run them.
instructions and tips on how to run them. If you want to run it locally via nix,
read the [nix testing docs](../nix/hosts/README.md).
+73 -5
View File
@@ -598,8 +598,7 @@ teardown(){
run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
assert_output --partial "WITH_COMMENT=foo"
@@ -610,8 +609,7 @@ teardown(){
run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --force
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
refute_output --partial "WITH_COMMENT=foo"
assert_output --partial "WITH_COMMENT=bar"
@@ -629,8 +627,14 @@ teardown(){
run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
assert_success
run docker service inspect --format '{{ range .Endpoint.Ports }}{{ .Protocol }}={{ .PublishedPort }}{{ end }}' \
if _is_local;
then
run docker service inspect --format "{{ range .Endpoint.Ports }}{{ .Protocol }}={{ .PublishedPort }}{{ end }}" \
"${TEST_APP_DOMAIN//./_}_app"
else
run ssh "$TEST_SERVER" "docker service inspect --format '{{ range .Endpoint.Ports }}{{ .Protocol }}={{ .PublishedPort }}{{ end }}' ${TEST_APP_DOMAIN//./_}_app"
fi
assert_success
assert_output --partial "tcp=1312"
assert_output --partial "udp=1312"
@@ -702,3 +706,67 @@ teardown(){
assert_failure
assert_output --partial "unexpected character"
}
# Deploy pinned 1.21.5, upgrade to pinned 1.21.6, then roll back to 1.21.5.
# bats test_tags=slow
@test "deploy, upgrade and rollback digest-pinned images" {
recipeDir="$ABRA_DIR/recipes/$TEST_RECIPE"
service="${TEST_APP_DOMAIN//./_}_app"
# Manually upgrade the unpinned 0.3.5 recipe to 1.21.5 and pin it.
run git -C "$recipeDir" checkout "0.3.5+1.21.0"
assert_success
run docker pull nginx:1.21.5
assert_success
run docker image inspect --format '{{index .RepoDigests 0}}' nginx:1.21.5
assert_success
oldImage="nginx:1.21.5@${output#*@}"
run sed -i "s|image: nginx:1.21.0|image: $oldImage|" "$recipeDir/compose.yml"
assert_success
# Label and tag the local recipe release so app commands can track it.
run sed -i 's/version=0.3.5+1.21.0/version=0.4.0+1.21.5/' "$recipeDir/compose.yml"
assert_success
_git_commit
run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag \
-a "0.4.0+1.21.5" -m "fake: 0.4.0+1.21.5"
assert_success
# Manually upgrade and pin the next release to 1.21.6.
run docker pull nginx:1.21.6
assert_success
run docker image inspect --format '{{index .RepoDigests 0}}' nginx:1.21.6
assert_success
newImage="nginx:1.21.6@${output#*@}"
run sed -i "s|image: $oldImage|image: $newImage|" "$recipeDir/compose.yml"
assert_success
# Update the release label to match the next recipe tag.
run sed -i 's/version=0.4.0+1.21.5/version=0.4.1+1.21.6/' "$recipeDir/compose.yml"
assert_success
_git_commit
run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag \
-a "0.4.1+1.21.6" -m "fake: 0.4.1+1.21.6"
assert_success
# Deploy recipe 0.4.0 with the exact pinned 1.21.5 image.
run $ABRA app deploy "$TEST_APP_DOMAIN" "0.4.0+1.21.5" --offline --no-input --no-converge-checks
assert_success
run docker service inspect --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' "$service"
assert_success
assert_output "$oldImage"
# Upgrade to recipe 0.4.1 and the exact pinned 1.21.6 image.
run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.4.1+1.21.6" --offline --no-input --no-converge-checks
assert_success
run docker service inspect --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' "$service"
assert_success
assert_output "$newImage"
# Roll back to recipe 0.4.0 and its original pinned image.
run $ABRA app rollback "$TEST_APP_DOMAIN" "0.4.0+1.21.5" --offline --no-input --no-converge-checks
assert_success
run docker service inspect --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' "$service"
assert_success
assert_output "$oldImage"
}
@@ -133,8 +133,7 @@ teardown(){
run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
assert_output --partial "$TEST_RECIIPE:0.1.0+1.20.0"
}
+11 -4
View File
@@ -140,15 +140,22 @@ teardown(){
_undeploy_app
sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
run docker config create "${sanitisedDomainName}_test_conf_v99" "$ABRA_DIR/recipes/abra-test-recipe/abra.sh"
assert_success
remote_ssh_command=""
if _is_local;
then
run docker config create "${sanitisedDomainName}_test_conf_v99" "$ABRA_DIR/recipes/abra-test-recipe/abra.sh"
else
remote_ssh_command="ssh '$TEST_SERVER'"
run cat "$ABRA_DIR/recipes/abra-test-recipe/abra.sh" | ssh "$TEST_SERVER" docker config create "${sanitisedDomainName}_test_conf_v99 -"
assert_success
fi
assert bash -c "docker config ls | grep -q test_conf_v99"
assert bash -c "$remote_ssh_command docker config ls | grep -q test_conf_v99"
run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
assert_success
refute bash -c "docker config ls | grep -q test_conf_v99"
refute bash -c "$remote_ssh_command docker config ls | grep -q test_conf_v99"
}
@test "remove .env file" {
@@ -54,8 +54,7 @@ teardown(){
--no-input
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
assert_output --partial "$TEST_RECIIPE:0.1.0+1.20.0"
}
@@ -53,8 +53,7 @@ teardown(){
run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
assert_output --partial "$TEST_RECIIPE:0.2.0+1.21.0"
assert_output --partial "$TEST_RECIPE:0.2.0+1.21.0"
}
+11 -2
View File
@@ -31,9 +31,18 @@ _ensure_ssh_agent() {
exit 1
fi
export SSH_AUTH_SOCK="$HOME/.ssh/ssh_auth_sock"
if [ ! -S ~/.ssh/ssh_auth_sock ]; then
if [[ ! -v SSH_AUTH_SOCK ]]; then
export SSH_AUTH_SOCK="$HOME/.ssh/ssh_auth_sock"
eval `ssh-agent`
ln -sf "$SSH_AUTH_SOCK" ~/.ssh/ssh_auth_sock
fi
}
_is_local() {
if [[ "$TEST_SERVER" == "default" ]];
then
return 0
else
return 1
fi
}
+30 -6
View File
@@ -1,13 +1,37 @@
#!/usr/bin/env bash
_ensure_swarm() {
if [ "$(docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
run docker swarm init --advertise-addr 127.0.0.1:2377
assert_success
if [ "$(docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
run docker swarm init --advertise-addr 127.0.0.1:2377
assert_success
fi
if ! $(docker network ls | grep -q 'proxy'); then
run docker network create -d overlay proxy
assert_success
if ! docker network ls | grep -q 'proxy'; then
run docker network create -d overlay proxy
assert_success
fi
if ! _is_local;
then
if [ "$(ssh "$TEST_SERVER" docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
run ssh "$TEST_SERVER" docker swarm init --advertise-addr 127.0.0.1:2377
assert_success
fi
if ! ssh "$TEST_SERVER" docker network ls | grep -q 'proxy'; then
run ssh "$TEST_SERVER" docker network create -d overlay proxy
assert_success
fi
fi
}
_inspect_env_test_app() {
if _is_local;
then
containerId="$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)"
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' "$containerId"
else
containerId="$(ssh "$TEST_SERVER" docker ps -f name="$TEST_APP_DOMAIN" -q)"
run ssh "$TEST_SERVER" "docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' '$containerId'"
fi
}
-2
View File
@@ -38,8 +38,6 @@ teardown(){
run $ABRA recipe upgrade "custom-html" --no-input
assert_success
assert_output --partial 'can upgrade service: app'
assert_exists "$ABRA_DIR/recipes/custom-html"
}
+3 -3
View File
@@ -27,7 +27,7 @@ teardown(){
assert bash -c "docker context ls | grep -q $TEST_SERVER"
server_dir_perms=$(stat -c "%a" "$ABRA_DIR/servers/$TEST_SERVER")
assert_equal $server_dir_perms "700"
assert_equal "$server_dir_perms" "700"
}
@test "error if using name and --local together" {
@@ -43,8 +43,8 @@ teardown(){
assert bash -c "docker context ls | grep -q default"
assert_output --partial 'local server successfully added'
server_dir_perms=$(stat -c "%a" "$ABRA_DIR/servers/$TEST_SERVER")
assert_equal $server_dir_perms "700"
server_dir_perms=$(stat -c "%a" "$ABRA_DIR/servers/default")
assert_equal "$server_dir_perms" "700"
}
@test "create local server fails when no docker swarm" {
+1 -1
View File
@@ -3,7 +3,7 @@ version: "3.8"
services:
app:
image: nginx:1.31.8
image: nginx:1.31.6
networks:
- proxy
deploy:
+1
View File
@@ -1,2 +1,3 @@
* -text
*.bin -text -diff
*.md text eol=lf
+700 -700
View File
File diff suppressed because it is too large Load Diff
+78 -78
View File
@@ -1,79 +1,79 @@
# Finite State Entropy
This package provides Finite State Entropy encoding and decoding.
Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS))
encoding provides a fast near-optimal symbol encoding/decoding
for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
## News
* Feb 2018: First implementation released. Consider this beta software for now.
# Usage
This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
You must provide input and will receive the output and maybe an error.
These error values can be returned:
| Error | Description |
|---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `(error)` | An internal error occurred. |
As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
# Performance
A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.
All compression functions are currently only running on the calling goroutine so only one core will be used per block.
The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be
beneficial to transpose all your input values down by 64.
With moderate block sizes around 64k speed are typically 200MB/s per core for compression and
around 300MB/s decompression speed.
The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s.
# Plans
At one point, more internals will be exposed to facilitate more "expert" usage of the components.
A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).
# Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
# Finite State Entropy
This package provides Finite State Entropy encoding and decoding.
Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS))
encoding provides a fast near-optimal symbol encoding/decoding
for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
## News
* Feb 2018: First implementation released. Consider this beta software for now.
# Usage
This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
You must provide input and will receive the output and maybe an error.
These error values can be returned:
| Error | Description |
|---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `(error)` | An internal error occurred. |
As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
# Performance
A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.
All compression functions are currently only running on the calling goroutine so only one core will be used per block.
The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be
beneficial to transpose all your input values down by 64.
With moderate block sizes around 64k speed are typically 200MB/s per core for compression and
around 300MB/s decompression speed.
The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s.
# Plans
At one point, more internals will be exposed to facilitate more "expert" usage of the components.
A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).
# Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
changes will likely not be accepted. If in doubt open an issue before writing the PR.
+89 -89
View File
@@ -1,89 +1,89 @@
# Huff0 entropy compression
This package provides Huff0 encoding and decoding as used in zstd.
[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders),
a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU
(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
## News
This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
This ensures that most functionality is well tested.
# Usage
This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and
[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
You must provide input and will receive the output and maybe an error.
These error values can be returned:
| Error | Description |
|---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) |
| `(error)` | An internal error occurred. |
As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.
## Tables and re-use
Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results.
The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy)
that controls this behaviour. See the documentation for details. This can be altered between each block.
Do however note that this information is *not* stored in the output block and it is up to the users of the package to
record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
based on the boolean reported back from the CompressXX call.
If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the
[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
## Decompressing
The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
This will initialize the decoding tables.
You can supply the complete block to `ReadTable` and it will return the data part of the block
which can be given to the decompressor.
Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X)
or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
# Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
changes will likely not be accepted. If in doubt open an issue before writing the PR.
# Huff0 entropy compression
This package provides Huff0 encoding and decoding as used in zstd.
[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders),
a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU
(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
## News
This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
This ensures that most functionality is well tested.
# Usage
This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and
[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
You must provide input and will receive the output and maybe an error.
These error values can be returned:
| Error | Description |
|---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) |
| `(error)` | An internal error occurred. |
As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.
## Tables and re-use
Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results.
The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy)
that controls this behaviour. See the documentation for details. This can be altered between each block.
Do however note that this information is *not* stored in the output block and it is up to the users of the package to
record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
based on the boolean reported back from the CompressXX call.
If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the
[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
## Decompressing
The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
This will initialize the decoding tables.
You can supply the complete block to `ReadTable` and it will return the data part of the block
which can be given to the decompressor.
Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X)
or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
# Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
changes will likely not be accepted. If in doubt open an issue before writing the PR.
-64
View File
@@ -1,64 +0,0 @@
# AGENTS.md - gotext Project Guidelines
This document is the **source of truth** for the `gotext` project's architecture, conventions, and development workflows. All contributors — human or AI agent — must adhere to these guidelines.
Agent-specific configuration files (e.g., `GEMINI.md`, `CLAUDE.md`) should reference this document rather than duplicating its content.
## Project Overview
`gotext` is a GNU Gettext implementation for Go (Golang). It provides a thread-safe, hierarchical translation system that supports PO/MO files, plural forms, and context-aware translations.
## Core Architecture
The project follows a hierarchical structure for managing translations:
1. **Global Package API (`gotext.go`)**: Provides convenient, global functions (e.g., `Get`, `GetD`, `GetN`) that rely on a shared `globalConfig`.
2. **Locales (`locale.go`)**: Represent a specific language (e.g., "en_US", "es_ES"). A `Locale` container manages multiple `Domains`.
3. **Domains (`domain.go`)**: The core translation engine for a specific set of messages. It handles:
- Message retrieval (with support for plural forms and `msgctxt`).
- Header parsing (including plural-form rules).
- Thread-safety via `sync.RWMutex`.
4. **Translators (`translator.go`)**: An interface for translation sources.
- **PO (`po.go`)**: Parsers for human-readable Gettext Portable Object files.
- **MO (`mo.go`)**: Parsers for binary Gettext Machine Object files.
5. **Plural Forms (`plurals/`)**: A dedicated package for compiling and evaluating GNU Gettext plural form expressions.
## CLI Tools
- **xgotext (`cli/xgotext/`)**: A tool to scan Go source files, extract translatable strings (matching `Get`, `GetD`, etc.), and generate/update PO files.
## Key Files & Symbols
- `gotext.go`: `Configure`, `Get`, `GetD`, `GetN`.
- `locale.go`: `Locale`, `NewLocale`, `AddDomain`.
- `domain.go`: `Domain`, `Get`, `GetN`, `parseHeaders`.
- `translator.go`: `Translator` interface.
- `plurals/compiler.go`: `Compile` function for plural expressions.
## Development Workflows
### Branching & PRs
- **Feature Branches**: All development MUST happen in dedicated feature branches (e.g., `feature/description-of-change`, `fix/bug-description`, `refactor/topic`).
- **Target Branch**: All Pull Requests MUST be made against the `master` branch.
- **Review Requirement**: No direct commits to `master` are allowed. All changes must be reviewed via a PR.
- **GitHub Integration**: If the `gh` CLI tool is installed and authenticated, it should be used for managing Pull Requests, Issues, and Reviews. Otherwise, these tasks must be performed manually through the GitHub web interface.
### Coding Standards
- **Idiomatic Go**: Follow standard Go conventions (`gofmt`, `go vet`).
- **Thread Safety**: Ensure all shared state (like in `Domain`) is protected by appropriate synchronization primitives.
- **Testing**:
- Every new feature or bug fix must include corresponding tests.
- Use the `fixtures/` directory for sample PO/MO files.
- Run `go test ./...` to ensure no regressions.
### Pre-Submit Checklist
Before submitting a Pull Request, verify:
1. All existing tests pass: `go test ./...`
2. Code passes static analysis: `go vet ./...`
3. Code is properly formatted: `gofmt`
4. New features or bug fixes include corresponding tests.
5. Commit messages are clear and descriptive.
+45 -2
View File
@@ -1,3 +1,46 @@
# CODE OF CONDUCT
# Contributor Covenant Code of Conduct
The canonical text of this project's Code of Conduct lives at [docs/CODE_OF_CONDUCT.md](docs/CODE_OF_CONDUCT.md).
## Our Pledge
In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at leonel.quinteros@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
+14 -4
View File
@@ -1,9 +1,19 @@
# CONTRIBUTING
Thank you for considering contributing to `gotext`! All kinds of contributions — features, bug fixes, tests, documentation, or discussions — are welcome.
This open source project welcomes everybody that wants to contribute to it by implementing new features, fixing bugs, testing, creating documentation or simply talk about it.
The canonical contribution guidelines live at [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md).
Most contributions will start by creating a new Issue to discuss what is the contribution about and to agree on the steps to move forward.
Before contributing, please also read [AGENTS.md](AGENTS.md): it is the source of truth for project architecture, conventions, and development workflows (branching, PR process, coding standards, and the pre-submit checklist).
## Issues
All issues reports are welcome. Open a new Issue whenever you want to report a bug, request a change or make a proposal.
This should be your start point of contribution.
## Pull Requests
If you have any changes that can be merged, feel free to send a Pull Request.
Usually, you'd want to create a new Issue to discuss about the change you want to merge and why it's needed or what it solves.
Start point: open an [Issue](https://github.com/leonelquinteros/gotext/issues) to discuss your idea, then send a Pull Request against `master`.
-9
View File
@@ -1,9 +0,0 @@
# GEMINI.md - Gemini CLI Instructions for gotext
> **Important**: Read and follow [`AGENTS.md`](./AGENTS.md) first. It is the source of truth for project architecture, conventions, and development workflows. The instructions below are specific to Gemini CLI and supplement — not replace — those guidelines.
## Gemini-Specific Instructions
- Always work in a feature branch. Never commit directly to `master`.
- Before submitting, verify changes with existing and new tests (`go test ./...`).
- Propose a PR to `master` after confirmation, using the `gh` CLI when available.
+206 -99
View File
@@ -9,151 +9,258 @@
[GNU gettext utilities](https://www.gnu.org/software/gettext) for Go.
`gotext` is a native Go implementation of the GNU Gettext utilities. It provides a thread-safe, flexible, and powerful way to handle internationalization (i18n) and localization (l10n) in your Go applications.
---
# Features
**📖 [Read the Full Documentation](https://leonelquinteros.github.io/gotext/)**
- Implements GNU gettext support in native Go.
- Complete support for [PO files](https://www.gnu.org/software/gettext/manual/html_node/PO-Files.html) including:
- Support for multiline strings and headers.
- Support for variables inside translation strings using Go's [fmt syntax](https://golang.org/pkg/fmt/).
- Support for [pluralization rules](https://www.gnu.org/software/gettext/manual/html_node/Translating-plural-forms.html).
- Support for [message contexts](https://www.gnu.org/software/gettext/manual/html_node/Contexts.html).
- Support for MO files.
- Thread-safe: This package is safe for concurrent use across multiple goroutines.
- It works with UTF-8 encoding as it's the default for Go language.
- Unit tests available.
- Language codes are automatically simplified from the form `en_UK` to `en` if the first isn't available.
- Ready to use inside Go templates.
- Objects are serializable to []byte to store them in cache.
- Support for Go Modules.
---
### Table of Contents
- [Features](#features)
- [Installation](#installation)
- [Getting Started](#getting-started)
- [CLI Tool (xgotext)](#cli-tool-xgotext)
- [Advanced Usage](#advanced-usage)
- [Locale Object](#using-locale-object)
- [Plural Forms](#use-plural-forms-of-translations)
- [Dynamic Variables](#using-dynamic-variables-on-translations)
- [Directory Structure](#locales-directories-structure)
- [Contributing](#contributing)
- [License](#license)
# License
---
[MIT license](LICENSE)
## Features
- Native Go implementation of Gettext (the core library has no external dependencies).
- Full support for **PO and MO files**.
- **Pluralization rules** support via GNU Gettext plural-form expressions.
- **Message context** support (`msgctxt`).
- Thread-safe for concurrent use.
- Works with UTF-8 by default.
- Integrated CLI tool (`xgotext`) for string extraction.
- Serializable objects for caching.
- Plain-string results that plug directly into Go's `text/template` and `html/template` pipelines.
---
# Documentation
## Installation
```bash
Refer to package documentation at (https://pkg.go.dev/github.com/leonelquinteros/gotext)
# Installation
```
go get github.com/leonelquinteros/gotext
```
---
- There are no requirements or dependencies to use this package.
- No need to install GNU gettext utilities (unless specific needs of CLI tools).
- No need for environment variables. Some naming conventions are applied but not needed.
## Getting Started
For a quick start, use the package-level API:
# Usage examples
## Using package for single language/domain settings
For quick/simple translations you can use the package level functions directly.
```go
package main
import (
"fmt"
"github.com/leonelquinteros/gotext"
)
func main() {
// Configure package: locales path, language, and domain
gotext.Configure("/path/to/locales", "en_US", "default")
// Configure package
gotext.Configure("/path/to/locales/root/dir", "en_UK", "domain-name")
// Simple translation
fmt.Println(gotext.Get("Hello, world!"))
// Translate text from default domain
fmt.Println(gotext.Get("My text on 'domain-name' domain"))
// Translation with variables
fmt.Println(gotext.Get("Hello, %s!", "Gopher"))
// Translate text from a different domain without reconfigure
fmt.Println(gotext.GetD("domain2", "Another text on a different domain"))
}
```
## Using dynamic variables on translations
All translation strings support dynamic variables to be inserted without translate.
Use the fmt.Printf syntax (from Go's "fmt" package) to specify how to print the non-translated variable inside the translation string.
```go
import (
"fmt"
"github.com/leonelquinteros/gotext"
)
func main() {
// Configure package
gotext.Configure("/path/to/locales/root/dir", "en_UK", "domain-name")
// Set variables
name := "John"
// Translate text with variables
fmt.Println(gotext.Get("Hi, my name is %s", name))
}
```
## Using Locale object
When having multiple languages/domains/libraries at the same time, you can create Locale objects for each variation
so you can handle each settings on their own.
```go
import (
"fmt"
"github.com/leonelquinteros/gotext"
)
func main() {
// Create Locale with library path and language code
l := gotext.NewLocale("/path/to/locales/root/dir", "es_UY")
// Load domain '/path/to/locales/root/dir/es_UY/default.po'
l.AddDomain("default")
// Translate text from default domain
fmt.Println(l.Get("Translate this"))
// Load different domain
l.AddDomain("translations")
// Translate text from domain
fmt.Println(l.GetD("translations", "Translate this"))
}
```
For more details, see the [Getting Started Guide](docs/GETTING_STARTED.md).
This is also helpful for using inside templates (from the "text/template" package), where you can pass the Locale object to the template.
If you set the Locale object as "Loc" in the template, then the template code would look like:
---
## CLI Tool (xgotext)
`gotext` includes a command-line tool to extract translatable strings from your Go source code.
**Install xgotext:**
```bash
go install github.com/leonelquinteros/gotext/cli/xgotext@latest
```
{{ .Loc.Get "Translate this" }}
```
**Extract strings:**
```bash
xgotext -in . -out locales/templates
```
The command generates `.pot` templates under `locales/templates`; merge those templates into language-specific `.po` files.
See the [xgotext Documentation](docs/xgotext.md) for full usage details.
## Using the Po object to handle .po files and PO-formatted strings
---
## Advanced Usage
### Using Locale object
For managing multiple languages or domains independently:
For when you need to work with PO files and strings,
you can directly use the Po object to parse it and access the translations in there in the same way.
```go
l := gotext.NewLocale("/path/to/locales", "es_UY")
l.AddDomain("default")
fmt.Println(l.Get("Translate this"))
import (
"fmt"
"github.com/leonelquinteros/gotext"
)
func main() {
// Set PO content
str := `
msgid "Translate this"
msgstr "Translated text"
msgid "Another string"
msgstr ""
msgid "One with var: %s"
msgstr "This one sets the var: %s"
`
// Create Po object
po := gotext.NewPo()
po.Parse(str)
fmt.Println(po.Get("Translate this"))
}
```
### Use plural forms of translations
`gotext` handles complex pluralization rules defined in PO headers:
## Use plural forms of translations
PO format supports defining one or more plural forms for the same translation.
Relying on the PO file headers, a Plural-Forms formula can be set on the translation file
as defined in (https://www.gnu.org/savannah-checkouts/gnu/gettext/manual/html_node/Plural-forms.html)
```go
// GetN(singular, plural, quantity, args...)
fmt.Println(gotext.GetN("I have one apple.", "I have %d apples.", 5, 5))
```
See the [Plural Forms Guide](docs/PLURALS.md) for more examples.
import (
"fmt"
"github.com/leonelquinteros/gotext"
)
### Using dynamic variables on translations
Supports standard `fmt` package syntax:
```go
name := "John"
fmt.Println(gotext.Get("Hi, my name is %s", name))
func main() {
// Set PO content
str := `
msgid ""
msgstr ""
# Header below
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
msgid "Translate this"
msgstr "Translated text"
msgid "Another string"
msgstr ""
msgid "One with var: %s"
msgid_plural "Several with vars: %s"
msgstr[0] "This one is the singular: %s"
msgstr[1] "This one is the plural: %s"
`
// Create Po object
po := new(gotext.Po)
po.Parse(str)
fmt.Println(po.GetN("One with var: %s", "Several with vars: %s", 54, v))
// "This one is the plural: Variable"
}
```
### More advanced scenarios
- **Embedded translation files** (`embed.FS`) and named `%(name)s` formatting: see the [Advanced Usage Guide](docs/ADVANCED.md).
- **Languages fallback**: pass multiple languages to `Configure` as a colon-separated list (`gotext.Configure("/path/to/locales", "es_UY:es:en_US", "default")`).
- **Serialization/caching** of parsed locales: `Locale.MarshalBinary` / `UnmarshalBinary`.
---
# Locales directories structure
The package will assume a directories structure starting with a base path that will be provided to the package configuration
or to object constructors depending on the use, but either will use the same convention to lookup inside the base path.
Inside the base directory where will be the language directories named using the language and country 2-letter codes (en_US, es_AR, ...).
All package functions can lookup after the simplified version for each language in case the full code isn't present but the more general language code exists.
So if the language set is `en_UK`, but there is no directory named after that code and there is a directory named `en`,
all package functions will be able to resolve this generalization and provide translations for the more general library.
The language codes are assumed to be [ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) codes (2-letter codes).
That said, most functions will work with any coding standard as long the directory name matches the language code set on the configuration.
Then, there can be a `LC_MESSAGES` containing all PO files or the PO files themselves.
A library directory structure can look like:
## Locales directories structure
The package expects a standard Gettext directory structure:
```
/path/to/locales
/en_US
/LC_MESSAGES
default.po
/es_ES
/LC_MESSAGES
default.po
/path/to/locales/en_US
/path/to/locales/en_US/LC_MESSAGES
/path/to/locales/en_US/LC_MESSAGES/default.po
/path/to/locales/en_US/LC_MESSAGES/extras.po
/path/to/locales/en_UK
/path/to/locales/en_UK/LC_MESSAGES
/path/to/locales/en_UK/LC_MESSAGES/default.po
/path/to/locales/en_UK/LC_MESSAGES/extras.po
/path/to/locales/en_AU
/path/to/locales/en_AU/LC_MESSAGES
/path/to/locales/en_AU/LC_MESSAGES/default.po
/path/to/locales/en_AU/LC_MESSAGES/extras.po
/path/to/locales/es
/path/to/locales/es/default.po
/path/to/locales/es/extras.po
/path/to/locales/es_ES
/path/to/locales/es_ES/default.po
/path/to/locales/es_ES/extras.po
/path/to/locales/fr
/path/to/locales/fr/default.po
/path/to/locales/fr/extras.po
```
Translation files can live either inside an `LC_MESSAGES` subdirectory or directly under the language code (`/es_ES/default.po`); both layouts are supported.
It supports automatic language simplification (e.g., falling back from `en_UK` to `en`).
And so on...
---
## Contributing
We welcome contributions of all kinds!
- [Contributing Guidelines](CONTRIBUTING.md)
- [Code of Conduct](CODE_OF_CONDUCT.md)
# Contribute
---
- Please, contribute.
- Use the package on your projects.
- Report issues on Github.
- Send pull requests for bugfixes and improvements.
- Send proposals on Github issues.
## License
`gotext` is released under the [MIT License](LICENSE).
+157 -316
View File
@@ -4,7 +4,7 @@ import (
"bytes"
"encoding/gob"
"fmt"
"maps"
"regexp"
"sort"
"strconv"
"strings"
@@ -64,6 +64,9 @@ func (m HeaderMap) Del(key string) {
// Get value for key from HeaderMap
func (m HeaderMap) Get(key string) string {
if m == nil {
return ""
}
v := m[key]
if len(v) == 0 {
return ""
@@ -78,6 +81,9 @@ func (m HeaderMap) Set(key, value string) {
// Values returns all values for a given key from HeaderMap
func (m HeaderMap) Values(key string) []string {
if m == nil {
return nil
}
return m[key]
}
@@ -94,111 +100,29 @@ func NewDomain() *Domain {
return domain
}
func cloneRefs(refs []string) []string {
if refs == nil {
return nil
}
owned := make([]string, len(refs))
copy(owned, refs)
return owned
}
func cloneHeaderMap(headers HeaderMap) HeaderMap {
if headers == nil {
return nil
}
owned := make(HeaderMap, len(headers))
for key, values := range headers {
owned[key] = cloneRefs(values)
}
return owned
}
func cloneTranslation(trans *Translation) *Translation {
if trans == nil {
return nil
}
owned := NewTranslation()
owned.ID = trans.ID
owned.PluralID = trans.PluralID
owned.dirty = trans.dirty
owned.Refs = cloneRefs(trans.Refs)
maps.Copy(owned.Trs, trans.Trs)
return owned
}
func cloneTranslations(translations map[string]*Translation) map[string]*Translation {
if translations == nil {
return nil
}
owned := make(map[string]*Translation, len(translations))
for id, trans := range translations {
owned[id] = cloneTranslation(trans)
}
return owned
}
func cloneContextTranslations(contexts map[string]map[string]*Translation) map[string]map[string]*Translation {
if contexts == nil {
return nil
}
owned := make(map[string]map[string]*Translation, len(contexts))
for context, translations := range contexts {
owned[context] = cloneTranslations(translations)
}
return owned
}
func (do *Domain) hasTranslation(id string) bool {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
return do.translations[id] != nil
}
func (do *Domain) hasContextTranslation(id, context string) bool {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
return do.contextTranslations[context][id] != nil
}
// SetPluralResolver sets a custom plural resolver function
func (do *Domain) SetPluralResolver(f func(int) int) {
do.pluralMutex.Lock()
defer do.pluralMutex.Unlock()
do.customPluralResolver = f
}
func (do *Domain) pluralForm(n int) int {
// Snapshot plural state before calling an expression or custom resolver.
// do we really need locking here? not sure how this plurals.Expression works, so sticking with it for now
do.pluralMutex.RLock()
pluralforms := do.pluralforms
customPluralResolver := do.customPluralResolver
do.pluralMutex.RUnlock()
if pluralforms != nil {
return pluralforms.Eval(uint32(n))
}
defer do.pluralMutex.RUnlock()
// Failure fallback
if customPluralResolver != nil {
return customPluralResolver(n)
}
if do.pluralforms == nil {
if do.customPluralResolver != nil {
return do.customPluralResolver(n)
}
/* Use the Germanic plural rule. */
if n == 1 {
return 0
/* Use the Germanic plural rule. */
if n == 1 {
return 0
}
return 1
}
return 1
return do.pluralforms.Eval(uint32(n))
}
// parseHeaders retrieves data from previously parsed headers. it's called by both Mo and Po when parsing
@@ -212,16 +136,18 @@ func (do *Domain) parseHeaders() {
languageKey := "Language"
pluralFormsKey := "Plural-Forms"
for line := range strings.SplitSeq(raw, "\n") {
rawLines := strings.Split(raw, "\n")
for _, line := range rawLines {
if len(line) == 0 {
continue
}
key, value, ok := strings.Cut(line, ":")
if !ok {
colonIdx := strings.Index(line, ":")
if colonIdx < 0 {
continue
}
key := line[:colonIdx]
lowerKey := strings.ToLower(key)
if lowerKey == strings.ToLower(languageKey) {
languageKey = key
@@ -229,7 +155,7 @@ func (do *Domain) parseHeaders() {
pluralFormsKey = key
}
value = strings.TrimSpace(value)
value := strings.TrimSpace(line[colonIdx+1:])
do.Headers.Add(key, value)
}
@@ -242,18 +168,22 @@ func (do *Domain) parseHeaders() {
return
}
for i := range strings.SplitSeq(do.PluralForms, ";") {
key, value, ok := strings.Cut(i, "=")
if !ok {
// Split plural form header value
pfs := strings.Split(do.PluralForms, ";")
// Parse values
for _, i := range pfs {
vs := strings.SplitN(i, "=", 2)
if len(vs) != 2 {
continue
}
switch strings.TrimSpace(key) {
switch strings.TrimSpace(vs[0]) {
case "nplurals":
do.nplurals, _ = strconv.Atoi(value)
do.nplurals, _ = strconv.Atoi(vs[1])
case "plural":
do.plural = value
do.plural = vs[1]
if expr, err := plurals.Compile(do.plural); err == nil {
do.pluralforms = expr
@@ -291,19 +221,17 @@ func (do *Domain) DropStaleTranslations() {
// SetRefs set source references for a given translation
func (do *Domain) SetRefs(str string, refs []string) {
ownedRefs := cloneRefs(refs)
do.trMutex.Lock()
do.pluralMutex.Lock()
defer do.trMutex.Unlock()
defer do.pluralMutex.Unlock()
if trans, ok := do.translations[str]; ok && trans != nil {
trans.SetRefs(ownedRefs)
if trans, ok := do.translations[str]; ok {
trans.Refs = refs
} else {
trans := NewTranslation()
trans = NewTranslation()
trans.ID = str
trans.SetRefs(ownedRefs)
trans.SetRefs(refs)
do.translations[str] = trans
}
}
@@ -315,8 +243,8 @@ func (do *Domain) GetRefs(str string) []string {
defer do.trMutex.RUnlock()
if do.translations != nil {
if trans, ok := do.translations[str]; ok && trans != nil {
return cloneRefs(trans.Refs)
if trans, ok := do.translations[str]; ok {
return trans.Refs
}
}
return nil
@@ -340,7 +268,7 @@ func (do *Domain) Set(id, str string) {
}
// Get retrieves the Translation for the given string.
func (do *Domain) Get(str string, vars ...any) string {
func (do *Domain) Get(str string, vars ...interface{}) string {
// Sync read
do.trMutex.RLock()
defer do.trMutex.RUnlock()
@@ -356,8 +284,8 @@ func (do *Domain) Get(str string, vars ...any) string {
}
// Append retrieves the Translation for the given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) Append(b []byte, str string, vars ...any) []byte {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) Append(b []byte, str string, vars ...interface{}) []byte {
// Sync read
do.trMutex.RLock()
defer do.trMutex.RUnlock()
@@ -394,48 +322,40 @@ func (do *Domain) SetN(id, plural string, n int, str string) {
}
// GetN retrieves the (N)th plural form of Translation for the given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) GetN(str, plural string, n int, vars ...any) string {
// Get plural form before acquiring the translation lock so a custom
// resolver is never called while a project lock is held.
pluralForm := do.pluralForm(n)
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) GetN(str, plural string, n int, vars ...interface{}) string {
// Sync read
do.trMutex.RLock()
defer do.trMutex.RUnlock()
if do.translations != nil {
if _, ok := do.translations[str]; ok {
return FormatString(do.translations[str].GetN(pluralForm), vars...)
return FormatString(do.translations[str].GetN(do.pluralForm(n)), vars...)
}
}
// Parse plural forms to distinguish between plural and singular
if pluralForm == 0 {
if do.pluralForm(n) == 0 {
return FormatString(str, vars...)
}
return FormatString(plural, vars...)
}
// AppendN adds the (N)th plural form of Translation for the given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) AppendN(b []byte, str, plural string, n int, vars ...any) []byte {
// Get plural form before acquiring the translation lock so a custom
// resolver is never called while a project lock is held.
pluralForm := do.pluralForm(n)
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) AppendN(b []byte, str, plural string, n int, vars ...interface{}) []byte {
// Sync read
do.trMutex.RLock()
defer do.trMutex.RUnlock()
if do.translations != nil {
if _, ok := do.translations[str]; ok {
return Appendf(b, do.translations[str].GetN(pluralForm), vars...)
return Appendf(b, do.translations[str].GetN(do.pluralForm(n)), vars...)
}
}
// Parse plural forms to distinguish between plural and singular
if pluralForm == 0 {
if do.pluralForm(n) == 0 {
return Appendf(b, str, vars...)
}
return Appendf(b, plural, vars...)
@@ -468,8 +388,8 @@ func (do *Domain) SetC(id, ctx, str string) {
}
// GetC retrieves the corresponding Translation for a given string in the given context.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) GetC(str, ctx string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) GetC(str, ctx string, vars ...interface{}) string {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
@@ -488,8 +408,8 @@ func (do *Domain) GetC(str, ctx string, vars ...any) string {
}
// AppendC retrieves the corresponding Translation for a given string in the given context.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) AppendC(b []byte, str, ctx string, vars ...any) []byte {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) AppendC(b []byte, str, ctx string, vars ...interface{}) []byte {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
@@ -523,14 +443,12 @@ func (do *Domain) SetNC(id, plural, ctx string, n int, str string) {
} else {
trans = NewTranslation()
trans.ID = id
trans.PluralID = plural
trans.SetN(pluralForm, str)
context[id] = trans
}
} else {
trans := NewTranslation()
trans.ID = id
trans.PluralID = plural
trans.SetN(pluralForm, str)
do.contextTranslations[ctx] = map[string]*Translation{
id: trans,
@@ -539,22 +457,19 @@ func (do *Domain) SetNC(id, plural, ctx string, n int, str string) {
}
// GetNC retrieves the (N)th plural form of Translation for the given string in the given context.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) GetNC(str, plural string, n int, ctx string, vars ...any) string {
if do.hasContextTranslation(str, ctx) {
// Get plural form before acquiring the translation lock so a custom
// resolver is never called while a project lock is held.
pluralForm := do.pluralForm(n)
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) GetNC(str, plural string, n int, ctx string, vars ...interface{}) string {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
do.trMutex.RLock()
if translations, ok := do.contextTranslations[ctx]; ok && translations != nil {
if trans, ok := translations[str]; ok && trans != nil {
translated := trans.GetN(pluralForm)
do.trMutex.RUnlock()
return FormatString(translated, vars...)
if do.contextTranslations != nil {
if _, ok := do.contextTranslations[ctx]; ok {
if do.contextTranslations[ctx] != nil {
if _, ok := do.contextTranslations[ctx][str]; ok {
return FormatString(do.contextTranslations[ctx][str].GetN(do.pluralForm(n)), vars...)
}
}
}
do.trMutex.RUnlock()
}
if n == 1 {
@@ -564,22 +479,19 @@ func (do *Domain) GetNC(str, plural string, n int, ctx string, vars ...any) stri
}
// AppendNC retrieves the (N)th plural form of Translation for the given string in the given context.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) AppendNC(b []byte, str, plural string, n int, ctx string, vars ...any) []byte {
if do.hasContextTranslation(str, ctx) {
// Get plural form before acquiring the translation lock so a custom
// resolver is never called while a project lock is held.
pluralForm := do.pluralForm(n)
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) AppendNC(b []byte, str, plural string, n int, ctx string, vars ...interface{}) []byte {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
do.trMutex.RLock()
if translations, ok := do.contextTranslations[ctx]; ok && translations != nil {
if trans, ok := translations[str]; ok && trans != nil {
translated := trans.GetN(pluralForm)
do.trMutex.RUnlock()
return Appendf(b, translated, vars...)
if do.contextTranslations != nil {
if _, ok := do.contextTranslations[ctx]; ok {
if do.contextTranslations[ctx] != nil {
if _, ok := do.contextTranslations[ctx][str]; ok {
return Appendf(b, do.contextTranslations[ctx][str].GetN(do.pluralForm(n)), vars...)
}
}
}
do.trMutex.RUnlock()
}
if n == 1 {
@@ -595,14 +507,6 @@ func (do *Domain) IsTranslated(str string) bool {
// IsTranslatedN reports whether a plural string is translated
func (do *Domain) IsTranslatedN(str string, n int) bool {
if !do.hasTranslation(str) {
return false
}
// Get plural form before acquiring the translation lock so a custom
// resolver is never called while a project lock is held.
pluralForm := do.pluralForm(n)
do.trMutex.RLock()
defer do.trMutex.RUnlock()
@@ -610,10 +514,10 @@ func (do *Domain) IsTranslatedN(str string, n int) bool {
return false
}
tr, ok := do.translations[str]
if !ok || tr == nil {
if !ok {
return false
}
return tr.IsTranslatedN(pluralForm)
return tr.IsTranslatedN(do.pluralForm(n))
}
// IsTranslatedC reports whether a context string is translated
@@ -623,14 +527,6 @@ func (do *Domain) IsTranslatedC(str, ctx string) bool {
// IsTranslatedNC reports whether a plural context string is translated
func (do *Domain) IsTranslatedNC(str string, n int, ctx string) bool {
if !do.hasContextTranslation(str, ctx) {
return false
}
// Get plural form before acquiring the translation lock so a custom
// resolver is never called while a project lock is held.
pluralForm := do.pluralForm(n)
do.trMutex.RLock()
defer do.trMutex.RUnlock()
@@ -638,24 +534,36 @@ func (do *Domain) IsTranslatedNC(str string, n int, ctx string) bool {
return false
}
translations, ok := do.contextTranslations[ctx]
if !ok || translations == nil {
if !ok {
return false
}
tr, ok := translations[str]
if !ok || tr == nil {
if !ok {
return false
}
return tr.IsTranslatedN(pluralForm)
return tr.IsTranslatedN(do.pluralForm(n))
}
// GetTranslations returns a copy of every translation in the domain. It does not support contexts.
func (do *Domain) GetTranslations() map[string]*Translation {
all := make(map[string]*Translation, len(do.translations))
do.trMutex.RLock()
defer do.trMutex.RUnlock()
all := make(map[string]*Translation, len(do.translations))
for msgID, trans := range do.translations {
all[msgID] = cloneTranslation(trans)
newTrans := NewTranslation()
newTrans.ID = trans.ID
newTrans.PluralID = trans.PluralID
newTrans.dirty = trans.dirty
if len(trans.Refs) > 0 {
newTrans.Refs = make([]string, len(trans.Refs))
copy(newTrans.Refs, trans.Refs)
}
for k, v := range trans.Trs {
newTrans.Trs[k] = v
}
all[msgID] = newTrans
}
return all
@@ -663,18 +571,32 @@ func (do *Domain) GetTranslations() map[string]*Translation {
// GetCtxTranslations returns a copy of every translation in the domain with context
func (do *Domain) GetCtxTranslations() map[string]map[string]*Translation {
all := make(map[string]map[string]*Translation, len(do.contextTranslations))
do.trMutex.RLock()
defer do.trMutex.RUnlock()
all := make(map[string]map[string]*Translation, len(do.contextTranslations))
for ctx, translations := range do.contextTranslations {
for msgID, trans := range translations {
newTrans := NewTranslation()
newTrans.ID = trans.ID
newTrans.PluralID = trans.PluralID
newTrans.dirty = trans.dirty
if len(trans.Refs) > 0 {
newTrans.Refs = make([]string, len(trans.Refs))
copy(newTrans.Refs, trans.Refs)
}
for k, v := range trans.Trs {
newTrans.Trs[k] = v
}
if all[ctx] == nil {
all[ctx] = make(map[string]*Translation)
}
all[ctx][msgID] = cloneTranslation(trans)
all[ctx][msgID] = newTrans
}
}
return all
@@ -682,11 +604,10 @@ func (do *Domain) GetCtxTranslations() map[string]map[string]*Translation {
// SourceReference is a struct to hold source reference information
type SourceReference struct {
path string
line int
context string
hasContext bool
trans *Translation
path string
line int
context string
trans *Translation
}
func extractPathAndLine(ref string) (string, int) {
@@ -706,11 +627,6 @@ func extractPathAndLine(ref string) (string, int) {
// MarshalText implements encoding.TextMarshaler interface
// Assists round-trip of POT/PO content
func (do *Domain) MarshalText() ([]byte, error) {
do.trMutex.RLock()
do.pluralMutex.RLock()
defer do.trMutex.RUnlock()
defer do.pluralMutex.RUnlock()
var buf bytes.Buffer
if len(do.headerComments) > 0 {
buf.WriteString(strings.Join(do.headerComments, "\n"))
@@ -765,29 +681,31 @@ func (do *Domain) MarshalText() ([]byte, error) {
v := do.Headers[k]
for _, value := range v {
encoded := EscapeSpecialCharacters(k + ": " + value + "\n")
buf.WriteString("\n\"" + encoded + "\"")
buf.WriteString("\n\"" + k + ": " + value + "\\n\"")
}
}
// Just as with headers, output translations in consistent order (to minimise diffs between round-trips), with (first) source reference taking priority, followed by context and finally ID
references := make([]SourceReference, 0)
for name, ctx := range do.contextTranslations {
for _, trans := range ctx {
for id, trans := range ctx {
if id == "" {
continue
}
if len(trans.Refs) > 0 {
path, line := extractPathAndLine(trans.Refs[0])
references = append(references, SourceReference{
path: path,
line: line,
context: name,
hasContext: true,
trans: trans,
path,
line,
name,
trans,
})
} else {
references = append(references, SourceReference{
context: name,
hasContext: true,
trans: trans,
"",
0,
name,
trans,
})
}
}
@@ -801,13 +719,17 @@ func (do *Domain) MarshalText() ([]byte, error) {
if len(trans.Refs) > 0 {
path, line := extractPathAndLine(trans.Refs[0])
references = append(references, SourceReference{
path: path,
line: line,
trans: trans,
path,
line,
"",
trans,
})
} else {
references = append(references, SourceReference{
trans: trans,
"",
0,
"",
trans,
})
}
}
@@ -832,9 +754,6 @@ func (do *Domain) MarshalText() ([]byte, error) {
if references[i].context > references[j].context {
return false
}
if references[i].hasContext != references[j].hasContext {
return !references[i].hasContext
}
return references[i].trans.ID < references[j].trans.ID
})
@@ -846,7 +765,7 @@ func (do *Domain) MarshalText() ([]byte, error) {
buf.WriteByte(byte('\n'))
}
if !ref.hasContext {
if ref.context == "" {
buf.WriteString("\nmsgid \"" + EscapeSpecialCharacters(trans.ID) + "\"")
} else {
buf.WriteString("\nmsgctxt \"" + EscapeSpecialCharacters(ref.context) + "\"\nmsgid \"" + EscapeSpecialCharacters(trans.ID) + "\"")
@@ -855,16 +774,9 @@ func (do *Domain) MarshalText() ([]byte, error) {
if trans.PluralID == "" {
buf.WriteString("\nmsgstr \"" + EscapeSpecialCharacters(trans.Trs[0]) + "\"")
} else {
buf.WriteString("\nmsgid_plural \"" + EscapeSpecialCharacters(trans.PluralID) + "\"")
pluralIndexes := make([]int, 0, len(trans.Trs))
for i := range trans.Trs {
pluralIndexes = append(pluralIndexes, i)
}
sort.Ints(pluralIndexes)
for _, i := range pluralIndexes {
buf.WriteString("\nmsgstr[" + strconv.Itoa(i) + "] \"" + EscapeSpecialCharacters(trans.Trs[i]) + "\"")
buf.WriteString("\nmsgid_plural \"" + trans.PluralID + "\"")
for i, tr := range trans.Trs {
buf.WriteString("\nmsgstr[" + EscapeSpecialCharacters(strconv.Itoa(i)) + "] \"" + tr + "\"")
}
}
}
@@ -874,23 +786,8 @@ func (do *Domain) MarshalText() ([]byte, error) {
// EscapeSpecialCharacters escapes special characters in a string
func EscapeSpecialCharacters(s string) string {
var escaped strings.Builder
escaped.Grow(len(s))
s = regexp.MustCompile(`([^\\])(")`).ReplaceAllString(s, "$1\\\"") // Escape non-escaped double quotation marks
for i := range len(s) {
switch s[i] {
case '\\':
escaped.WriteByte('\\')
escaped.WriteByte('\\')
case '"':
escaped.WriteByte('\\')
escaped.WriteByte('"')
default:
escaped.WriteByte(s[i])
}
}
s = escaped.String()
if strings.Count(s, "\n") == 0 {
return s
}
@@ -924,26 +821,16 @@ func EscapeSpecialCharacters(s string) string {
return strings.Join(data, "\n")
}
func (do *Domain) translatorEncodingSnapshot() *TranslatorEncoding {
do.trMutex.RLock()
do.pluralMutex.RLock()
defer do.trMutex.RUnlock()
defer do.pluralMutex.RUnlock()
return &TranslatorEncoding{
Headers: cloneHeaderMap(do.Headers),
Language: do.Language,
PluralForms: do.PluralForms,
Nplurals: do.nplurals,
Plural: do.plural,
Translations: cloneTranslations(do.translations),
Contexts: cloneContextTranslations(do.contextTranslations),
}
}
// MarshalBinary implements encoding.BinaryMarshaler interface
func (do *Domain) MarshalBinary() ([]byte, error) {
obj := do.translatorEncodingSnapshot()
obj := new(TranslatorEncoding)
obj.Headers = do.Headers
obj.Language = do.Language
obj.PluralForms = do.PluralForms
obj.Nplurals = do.nplurals
obj.Plural = do.plural
obj.Translations = do.translations
obj.Contexts = do.contextTranslations
var buff bytes.Buffer
encoder := gob.NewEncoder(&buff)
@@ -958,68 +845,22 @@ func (do *Domain) UnmarshalBinary(data []byte) error {
obj := new(TranslatorEncoding)
decoder := gob.NewDecoder(buff)
if err := decoder.Decode(obj); err != nil {
err := decoder.Decode(obj)
if err != nil {
return err
}
headers := cloneHeaderMap(obj.Headers)
if headers == nil {
headers = make(HeaderMap)
}
translations := cloneTranslations(obj.Translations)
if translations == nil {
translations = make(map[string]*Translation)
}
contextTranslations := cloneContextTranslations(obj.Contexts)
if contextTranslations == nil {
contextTranslations = make(map[string]map[string]*Translation)
}
for id, trans := range translations {
if trans == nil {
trans = NewTranslation()
trans.ID = id
translations[id] = trans
}
}
for context, translationsForContext := range contextTranslations {
if translationsForContext == nil {
contextTranslations[context] = make(map[string]*Translation)
continue
}
for id, trans := range translationsForContext {
if trans == nil {
trans = NewTranslation()
trans.ID = id
translationsForContext[id] = trans
}
}
}
var pluralforms plurals.Expression
if expr, err := plurals.Compile(obj.Plural); err == nil {
pluralforms = expr
}
do.trMutex.Lock()
do.pluralMutex.Lock()
do.Headers = headers
do.Headers = obj.Headers
do.Language = obj.Language
do.PluralForms = obj.PluralForms
do.nplurals = obj.Nplurals
do.plural = obj.Plural
do.pluralforms = pluralforms
do.translations = translations
do.contextTranslations = contextTranslations
do.pluralTranslations = make(map[string]*Translation)
do.headerComments = make([]string, 0)
do.trBuffer = nil
do.ctxBuffer = ""
do.refBuffer = ""
do.translations = obj.Translations
do.contextTranslations = obj.Contexts
do.pluralMutex.Unlock()
do.trMutex.Unlock()
if expr, err := plurals.Compile(do.plural); err == nil {
do.pluralforms = expr
}
return nil
}
+108 -170
View File
@@ -55,59 +55,51 @@ func init() {
domain: "default",
languages: []string{FallbackLocale},
library: "/usr/local/share/locale",
locales: make([]*Locale, 0),
locales: nil,
}
// Register Translator types for gob encoding
gob.Register(TranslatorEncoding{})
}
// loadLocales ensures package-level locales and their configured domain are loaded.
// loadLocales creates a new Locale object for every language (specified using Configure)
// at package level based on the configuration of global configuration .
// It is called when trying to use Get or GetD methods.
func loadLocales(rebuildCache bool) {
globalConfig.Lock()
if globalConfig.locales == nil || rebuildCache {
locales := make([]*Locale, 0, len(globalConfig.languages))
var locales []*Locale
for _, language := range globalConfig.languages {
locales = append(locales, NewLocale(globalConfig.library, language))
}
globalConfig.locales = locales
}
locales := append([]*Locale(nil), globalConfig.locales...)
domain := globalConfig.domain
globalConfig.Unlock()
for _, locale := range locales {
if locale == nil {
continue
for _, locale := range globalConfig.locales {
if _, ok := locale.Domains[globalConfig.domain]; !ok || rebuildCache {
locale.AddDomain(globalConfig.domain)
}
if rebuildCache || !locale.hasDomain(domain) {
locale.AddDomain(domain)
}
locale.SetDomain(domain)
locale.SetDomain(globalConfig.domain)
}
globalConfig.Unlock()
}
// GetDomain is the domain getter for the package configuration
func GetDomain() string {
var dom string
globalConfig.RLock()
domain := globalConfig.domain
var locale *Locale
for _, candidate := range globalConfig.locales {
if candidate != nil {
locale = candidate
break
}
if globalConfig.locales != nil {
// All locales have the same domain
dom = globalConfig.locales[0].GetDomain()
}
if dom == "" {
dom = globalConfig.domain
}
globalConfig.RUnlock()
if locale != nil {
if localeDomain := locale.GetDomain(); localeDomain != "" {
return localeDomain
}
}
return domain
return dom
}
// SetDomain sets the name for the domain to be used at package level.
@@ -115,8 +107,8 @@ func GetDomain() string {
func SetDomain(dom string) {
globalConfig.Lock()
globalConfig.domain = dom
for _, locale := range globalConfig.locales {
if locale != nil {
if globalConfig.locales != nil {
for _, locale := range globalConfig.locales {
locale.SetDomain(dom)
}
}
@@ -139,10 +131,8 @@ func GetLanguage() string {
// GetLanguages returns all languages that have been supplied.
func GetLanguages() []string {
globalConfig.RLock()
languages := make([]string, len(globalConfig.languages))
copy(languages, globalConfig.languages)
globalConfig.RUnlock()
return languages
defer globalConfig.RUnlock()
return globalConfig.languages
}
// SetLanguage sets the language code (or colon separated language codes) to be used at package level.
@@ -150,7 +140,7 @@ func GetLanguages() []string {
func SetLanguage(lang string) {
globalConfig.Lock()
var languages []string
for language := range strings.SplitSeq(lang, ":") {
for _, language := range strings.Split(lang, ":") {
languages = append(languages, SimplifiedLocale(language))
}
globalConfig.languages = languages
@@ -181,20 +171,17 @@ func SetLibrary(lib string) {
// GetLocales returns the locales that have been set for the package configuration.
func GetLocales() []*Locale {
globalConfig.RLock()
locales := make([]*Locale, len(globalConfig.locales))
copy(locales, globalConfig.locales)
globalConfig.RUnlock()
return locales
defer globalConfig.RUnlock()
return globalConfig.locales
}
// GetStorage is the locale storage getter for the package configuration.
//
// Deprecated: Storage has been renamed to Locale for consistency, use GetLocales instead.
func GetStorage() *Locale {
for _, locale := range GetLocales() {
if locale != nil {
return locale
}
locales := GetLocales()
if len(locales) > 0 {
return locales[0]
}
return nil
}
@@ -203,40 +190,24 @@ func GetStorage() *Locale {
// NewLocale(). This makes it possible to attach custom Domain objects from in-memory po/mo.
// The library, language and domain of the first Locale will set the default global configuration.
func SetLocales(locales []*Locale) {
ownedLocales := make([]*Locale, len(locales))
copy(ownedLocales, locales)
globalConfig.Lock()
globalConfig.locales = ownedLocales
defer globalConfig.Unlock()
languages := make([]string, 0, len(ownedLocales))
var first *Locale
for _, locale := range ownedLocales {
if locale == nil {
continue
}
globalConfig.locales = locales
globalConfig.library = locales[0].path
globalConfig.domain = locales[0].defaultDomain
locale.RLock()
if first == nil {
first = locale
globalConfig.library = locale.path
globalConfig.domain = locale.defaultDomain
}
var languages []string
for _, locale := range locales {
languages = append(languages, locale.lang)
locale.RUnlock()
}
globalConfig.languages = languages
globalConfig.Unlock()
}
// SetStorage allows overriding the global Locale object with one built manually with NewLocale().
//
// Deprecated: Storage has been renamed to Locale for consistency, use SetLocales instead.
func SetStorage(locale *Locale) {
if locale == nil {
SetLocales(nil)
return
}
SetLocales([]*Locale{locale})
}
@@ -248,7 +219,7 @@ func Configure(lib, lang, dom string) {
globalConfig.Lock()
globalConfig.library = lib
var languages []string
for language := range strings.SplitSeq(lang, ":") {
for _, language := range strings.Split(lang, ":") {
languages = append(languages, SimplifiedLocale(language))
}
globalConfig.languages = languages
@@ -259,137 +230,114 @@ func Configure(lib, lang, dom string) {
}
// Get uses the default domain globally set to return the corresponding Translation of a given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func Get(str string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func Get(str string, vars ...interface{}) string {
return GetD(GetDomain(), str, vars...)
}
// GetN retrieves the (N)th plural form of Translation for the given string in the default domain.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func GetN(str, plural string, n int, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func GetN(str, plural string, n int, vars ...interface{}) string {
return GetND(GetDomain(), str, plural, n, vars...)
}
// GetD returns the corresponding Translation in the given domain for a given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func GetD(dom, str string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func GetD(dom, str string, vars ...interface{}) string {
// Try to load default package Locales
loadLocales(false)
locales := GetLocales()
var fallback *Locale
for _, locale := range locales {
if locale == nil {
continue
}
fallback = locale
if !locale.hasDomain(dom) {
globalConfig.RLock()
defer globalConfig.RUnlock()
var tr string
for i, locale := range globalConfig.locales {
if _, ok := locale.Domains[dom]; !ok {
locale.AddDomain(dom)
}
if locale.IsTranslatedD(dom, str) {
return locale.GetD(dom, str, vars...)
if !locale.IsTranslatedD(dom, str) && i < (len(globalConfig.locales)-1) {
continue
}
tr = locale.GetD(dom, str, vars...)
break
}
if fallback != nil {
return fallback.GetD(dom, str, vars...)
}
return FormatString(str, vars...)
return tr
}
// GetND retrieves the (N)th plural form of Translation in the given domain for a given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func GetND(dom, str, plural string, n int, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func GetND(dom, str, plural string, n int, vars ...interface{}) string {
// Try to load default package Locales
loadLocales(false)
locales := GetLocales()
var fallback *Locale
for _, locale := range locales {
if locale == nil {
continue
}
fallback = locale
if !locale.hasDomain(dom) {
globalConfig.RLock()
defer globalConfig.RUnlock()
var tr string
for i, locale := range globalConfig.locales {
if _, ok := locale.Domains[dom]; !ok {
locale.AddDomain(dom)
}
if locale.IsTranslatedND(dom, str, n) {
return locale.GetND(dom, str, plural, n, vars...)
if !locale.IsTranslatedND(dom, str, n) && i < (len(globalConfig.locales)-1) {
continue
}
tr = locale.GetND(dom, str, plural, n, vars...)
break
}
if fallback != nil {
return fallback.GetND(dom, str, plural, n, vars...)
}
if n == 1 {
return FormatString(str, vars...)
}
return FormatString(plural, vars...)
return tr
}
// GetC uses the default domain globally set to return the corresponding Translation of the given string in the given context.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func GetC(str, ctx string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func GetC(str, ctx string, vars ...interface{}) string {
return GetDC(GetDomain(), str, ctx, vars...)
}
// GetNC retrieves the (N)th plural form of Translation for the given string in the given context in the default domain.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func GetNC(str, plural string, n int, ctx string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func GetNC(str, plural string, n int, ctx string, vars ...interface{}) string {
return GetNDC(GetDomain(), str, plural, n, ctx, vars...)
}
// GetDC returns the corresponding Translation in the given domain for the given string in the given context.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func GetDC(dom, str, ctx string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func GetDC(dom, str, ctx string, vars ...interface{}) string {
// Try to load default package Locales
loadLocales(false)
locales := GetLocales()
var fallback *Locale
for _, locale := range locales {
if locale == nil {
globalConfig.RLock()
defer globalConfig.RUnlock()
var tr string
for i, locale := range globalConfig.locales {
if !locale.IsTranslatedDC(dom, str, ctx) && i < (len(globalConfig.locales)-1) {
continue
}
fallback = locale
if !locale.hasDomain(dom) {
locale.AddDomain(dom)
}
if locale.IsTranslatedDC(dom, str, ctx) {
return locale.GetDC(dom, str, ctx, vars...)
}
tr = locale.GetDC(dom, str, ctx, vars...)
break
}
if fallback != nil {
return fallback.GetDC(dom, str, ctx, vars...)
}
return FormatString(str, vars...)
return tr
}
// GetNDC retrieves the (N)th plural form of Translation in the given domain for a given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func GetNDC(dom, str, plural string, n int, ctx string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func GetNDC(dom, str, plural string, n int, ctx string, vars ...interface{}) string {
// Try to load default package Locales
loadLocales(false)
locales := GetLocales()
var fallback *Locale
for _, locale := range locales {
if locale == nil {
// Return Translation
globalConfig.RLock()
defer globalConfig.RUnlock()
var tr string
for i, locale := range globalConfig.locales {
if !locale.IsTranslatedNDC(dom, str, n, ctx) && i < (len(globalConfig.locales)-1) {
continue
}
fallback = locale
if !locale.hasDomain(dom) {
locale.AddDomain(dom)
}
if locale.IsTranslatedNDC(dom, str, n, ctx) {
return locale.GetNDC(dom, str, plural, n, ctx, vars...)
}
tr = locale.GetNDC(dom, str, plural, n, ctx, vars...)
break
}
if fallback != nil {
return fallback.GetNDC(dom, str, plural, n, ctx, vars...)
}
if n == 1 {
return FormatString(str, vars...)
}
return FormatString(plural, vars...)
return tr
}
// IsTranslated reports whether a string is translated in given languages.
@@ -418,23 +366,18 @@ func IsTranslatedND(dom, str string, n int, langs ...string) bool {
}
loadLocales(false)
locales := GetLocales()
globalConfig.RLock()
defer globalConfig.RUnlock()
for _, lang := range langs {
lang = SimplifiedLocale(lang)
for _, supportedLocale := range locales {
if supportedLocale == nil {
for _, supportedLocale := range globalConfig.locales {
if lang != supportedLocale.GetActualLanguage(dom) {
continue
}
if lang != supportedLocale.GetLanguage() && lang != supportedLocale.GetActualLanguage(dom) {
continue
}
if !supportedLocale.hasDomain(dom) {
supportedLocale.AddDomain(dom)
}
if supportedLocale.IsTranslatedND(dom, str, n) {
return true
}
return supportedLocale.IsTranslatedND(dom, str, n)
}
}
return false
@@ -455,7 +398,7 @@ func IsTranslatedNC(str string, n int, ctx string, langs ...string) bool {
// IsTranslatedDC reports whether a domain context string is translated in given languages.
// When the langs argument is omitted, the output of GetLanguages is used.
func IsTranslatedDC(dom, str, ctx string, langs ...string) bool {
return IsTranslatedNDC(dom, str, 1, ctx, langs...)
return IsTranslatedNDC(dom, str, 0, ctx, langs...)
}
// IsTranslatedNDC reports whether a plural domain context string is translated in any of given languages.
@@ -466,23 +409,18 @@ func IsTranslatedNDC(dom, str string, n int, ctx string, langs ...string) bool {
}
loadLocales(false)
locales := GetLocales()
globalConfig.RLock()
defer globalConfig.RUnlock()
for _, lang := range langs {
lang = SimplifiedLocale(lang)
for _, supportedLocale := range locales {
if supportedLocale == nil {
for _, locale := range globalConfig.locales {
if lang != locale.GetActualLanguage(dom) {
continue
}
if lang != supportedLocale.GetLanguage() && lang != supportedLocale.GetActualLanguage(dom) {
continue
}
if !supportedLocale.hasDomain(dom) {
supportedLocale.AddDomain(dom)
}
if supportedLocale.IsTranslatedNDC(dom, str, n, ctx) {
return true
}
return locale.IsTranslatedNDC(dom, str, n, ctx)
}
}
return false
+27 -31
View File
@@ -29,21 +29,16 @@ func SimplifiedLocale(lang string) string {
}
// FormatString applies text formatting only when needed to parse variables.
func FormatString(str string, vars ...any) string {
func FormatString(str string, vars ...interface{}) string {
if len(vars) > 0 {
return FormatStringWithArgs(str, vars)
return fmt.Sprintf(str, vars...)
}
return str
}
// FormatStringWithArgs applies text formatting as a proxy for fmt.Sprintf.
func FormatStringWithArgs(str string, vars []any) string {
return fmt.Sprintf(str, vars...)
}
// Appendf applies text formatting only when needed to parse variables.
func Appendf(b []byte, str string, vars ...any) []byte {
func Appendf(b []byte, str string, vars ...interface{}) []byte {
if len(vars) > 0 {
return fmt.Appendf(b, str, vars...)
}
@@ -52,49 +47,50 @@ func Appendf(b []byte, str string, vars ...any) []byte {
}
// NPrintf support named format
// NPrintf("%(name)s is Type %(type)s", map[string]any{"name": "Gotext", "type": "struct"})
func NPrintf(format string, params map[string]any) {
// NPrintf("%(name)s is Type %(type)s", map[string]interface{}{"name": "Gotext", "type": "struct"})
func NPrintf(format string, params map[string]interface{}) {
f, p := parseSprintf(format, params)
fmt.Printf(f, p...)
}
// Sprintf support named format
//
// Sprintf("%(name)s is Type %(type)s", map[string]any{"name": "Gotext", "type": "struct"})
func Sprintf(format string, params map[string]any) string {
// Sprintf("%(name)s is Type %(type)s", map[string]interface{}{"name": "Gotext", "type": "struct"})
func Sprintf(format string, params map[string]interface{}) string {
f, p := parseSprintf(format, params)
return fmt.Sprintf(f, p...)
}
func parseSprintf(format string, params map[string]any) (string, []any) {
func parseSprintf(format string, params map[string]interface{}) (string, []interface{}) {
f, n := reformatSprintf(format)
var p []any
if len(n) > 0 {
p = make([]any, len(n))
for i, v := range n {
p[i] = params[v]
}
var p []interface{}
for _, v := range n {
p = append(p, params[v])
}
return f, p
}
func reformatSprintf(f string) (string, []string) {
matches := re.FindAllStringSubmatchIndex(f, -1)
ord := make([]string, 0, len(matches))
pair := make([]int, 1, 1+2*len(matches))
pair[0] = 0
m := re.FindAllStringSubmatch(f, -1)
i := re.FindAllStringSubmatchIndex(f, -1)
for _, match := range matches {
ord = append(ord, f[match[2]:match[3]])
pair = append(pair, match[2]-1, match[3]+1)
ord := []string{}
for _, v := range m {
ord = append(ord, v[1])
}
pair := []int{0}
for _, v := range i {
pair = append(pair, v[2]-1)
pair = append(pair, v[3]+1)
}
pair = append(pair, len(f))
plen := len(pair)
var out strings.Builder
out.Grow(len(f))
for n := 0; n < len(pair); n += 2 {
out.WriteString(f[pair[n]:pair[n+1]])
out := ""
for n := 0; n < plen; n += 2 {
out += f[pair[n]:pair[n+1]]
}
return out.String(), ord
return out, ord
}
+142 -193
View File
@@ -9,7 +9,6 @@ import (
"bytes"
"encoding/gob"
"io/fs"
"maps"
"os"
"path"
"sync"
@@ -23,6 +22,8 @@ multiple languages at the same time by working with this object.
Example:
import (
"encoding/gob"
"bytes"
"fmt"
"github.com/leonelquinteros/gotext"
)
@@ -89,114 +90,95 @@ func NewLocaleFSWithPath(l string, filesystem fs.FS, p string) *Locale {
return loc
}
func (l *Locale) snapshotPathLanguageFS() (string, string, fs.FS) {
if l == nil {
return "", "", nil
func (l *Locale) findExt(dom, ext string) string {
filename := path.Join(l.path, l.lang, "LC_MESSAGES", dom+"."+ext)
if l.fileExists(filename) {
return filename
}
l.RLock()
p := l.path
lang := l.lang
filesystem := l.fs
l.RUnlock()
return p, lang, filesystem
}
func fileExistsIn(filesystem fs.FS, filename string) bool {
if filesystem != nil {
_, err := fs.Stat(filesystem, filename)
return err == nil
}
_, err := os.Stat(filename)
return err == nil
}
func (l *Locale) findExtWithFS(dom, ext string) (fs.FS, string) {
p, lang, filesystem := l.snapshotPathLanguageFS()
filename := path.Join(p, lang, "LC_MESSAGES", dom+"."+ext)
if fileExistsIn(filesystem, filename) {
return filesystem, filename
}
if len(lang) > 2 {
filename = path.Join(p, lang[:2], "LC_MESSAGES", dom+"."+ext)
if fileExistsIn(filesystem, filename) {
return filesystem, filename
if len(l.lang) > 2 {
filename = path.Join(l.path, l.lang[:2], "LC_MESSAGES", dom+"."+ext)
if l.fileExists(filename) {
return filename
}
}
filename = path.Join(p, lang, dom+"."+ext)
if fileExistsIn(filesystem, filename) {
return filesystem, filename
filename = path.Join(l.path, l.lang, dom+"."+ext)
if l.fileExists(filename) {
return filename
}
if len(lang) > 2 {
filename = path.Join(p, lang[:2], dom+"."+ext)
if fileExistsIn(filesystem, filename) {
return filesystem, filename
if len(l.lang) > 2 {
filename = path.Join(l.path, l.lang[:2], dom+"."+ext)
if l.fileExists(filename) {
return filename
}
}
return filesystem, ""
return ""
}
// GetActualLanguage inspects the filesystem and decides whether to strip
// a CC part of the ll_CC locale string.
func (l *Locale) GetActualLanguage(dom string) string {
p, lang, filesystem := l.snapshotPathLanguageFS()
extensions := []string{"po", "mo"}
extensions := []string{"mo", "po"}
var fp string
for _, ext := range extensions {
// 'll' (or 'll_CC') exists, and it was specified as-is
fp = path.Join(p, lang, "LC_MESSAGES", dom+"."+ext)
if fileExistsIn(filesystem, fp) {
return lang
fp = path.Join(l.path, l.lang, "LC_MESSAGES", dom+"."+ext)
if l.fileExists(fp) {
return l.lang
}
// 'll' exists, but 'll_CC' was specified
if len(lang) > 2 {
fp = path.Join(p, lang[:2], "LC_MESSAGES", dom+"."+ext)
if fileExistsIn(filesystem, fp) {
return lang[:2]
if len(l.lang) > 2 {
fp = path.Join(l.path, l.lang[:2], "LC_MESSAGES", dom+"."+ext)
if l.fileExists(fp) {
return l.lang[:2]
}
}
// 'll' (or 'll_CC') exists outside of LC_category, and it was specified as-is
fp = path.Join(p, lang, dom+"."+ext)
if fileExistsIn(filesystem, fp) {
return lang
fp = path.Join(l.path, l.lang, dom+"."+ext)
if l.fileExists(fp) {
return l.lang
}
// 'll' exists outside of LC_category, but 'll_CC' was specified
if len(lang) > 2 {
fp = path.Join(p, lang[:2], dom+"."+ext)
if fileExistsIn(filesystem, fp) {
return lang[:2]
if len(l.lang) > 2 {
fp = path.Join(l.path, l.lang[:2], dom+"."+ext)
if l.fileExists(fp) {
return l.lang[:2]
}
}
}
return ""
}
func (l *Locale) fileExists(filename string) bool {
if l.fs != nil {
_, err := fs.Stat(l.fs, filename)
return err == nil
}
_, err := os.Stat(filename)
return err == nil
}
// AddDomain creates a new domain for a given locale object and initializes the Po object.
// If the domain exists, it gets reloaded.
func (l *Locale) AddDomain(dom string) {
if l == nil {
return
}
var poObj Translator
filesystem, file := l.findExtWithFS(dom, "po")
file := l.findExt(dom, "po")
if file != "" {
poObj = NewPoFS(filesystem)
poObj = NewPoFS(l.fs)
// Parse file.
poObj.ParseFile(file)
} else {
filesystem, file = l.findExtWithFS(dom, "mo")
file = l.findExt(dom, "mo")
if file != "" {
poObj = NewMoFS(filesystem)
poObj = NewMoFS(l.fs)
// Parse file.
poObj.ParseFile(file)
} else {
// fallback return if no file found with
return
}
}
@@ -230,34 +212,9 @@ func (l *Locale) AddTranslator(dom string, tr Translator) {
l.Unlock()
}
func (l *Locale) translatorFor(dom string) Translator {
if l == nil {
return nil
}
l.RLock()
translator := l.Domains[dom]
l.RUnlock()
return translator
}
func (l *Locale) hasDomain(dom string) bool {
if l == nil {
return false
}
l.RLock()
_, ok := l.Domains[dom]
l.RUnlock()
return ok
}
// GetDomain is the domain getter for Locale configuration
func (l *Locale) GetDomain() string {
if l == nil {
return ""
}
l.RLock()
dom := l.defaultDomain
l.RUnlock()
@@ -266,10 +223,6 @@ func (l *Locale) GetDomain() string {
// SetDomain sets the name for the domain to be used.
func (l *Locale) SetDomain(dom string) {
if l == nil {
return
}
l.Lock()
l.defaultDomain = dom
l.Unlock()
@@ -277,10 +230,6 @@ func (l *Locale) SetDomain(dom string) {
// GetLanguage is the lang getter for Locale configuration
func (l *Locale) GetLanguage() string {
if l == nil {
return ""
}
l.RLock()
lang := l.lang
l.RUnlock()
@@ -288,37 +237,51 @@ func (l *Locale) GetLanguage() string {
}
// Get uses a domain "default" to return the corresponding Translation of a given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) Get(str string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) Get(str string, vars ...interface{}) string {
return l.GetD(l.GetDomain(), str, vars...)
}
// GetN retrieves the (N)th plural form of Translation for the given string in the "default" domain.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetN(str, plural string, n int, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetN(str, plural string, n int, vars ...interface{}) string {
return l.GetND(l.GetDomain(), str, plural, n, vars...)
}
// GetD returns the corresponding Translation in the given domain for the given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetD(dom, str string, vars ...any) string {
translator := l.translatorFor(dom)
if translator != nil {
return translator.Get(str, vars...)
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetD(dom, str string, vars ...interface{}) string {
// Sync read
l.RLock()
defer l.RUnlock()
if l.Domains != nil {
if _, ok := l.Domains[dom]; ok {
if l.Domains[dom] != nil {
return l.Domains[dom].Get(str, vars...)
}
}
}
return FormatString(str, vars...)
}
// GetND retrieves the (N)th plural form of Translation in the given domain for the given string.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetND(dom, str, plural string, n int, vars ...any) string {
translator := l.translatorFor(dom)
if translator != nil {
return translator.GetN(str, plural, n, vars...)
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetND(dom, str, plural string, n int, vars ...interface{}) string {
// Sync read
l.RLock()
defer l.RUnlock()
if l.Domains != nil {
if _, ok := l.Domains[dom]; ok {
if l.Domains[dom] != nil {
return l.Domains[dom].GetN(str, plural, n, vars...)
}
}
}
// Use western default rule (n != 1) to handle missing domain default result.
// Use western default rule (plural > 1) to handle missing domain default result.
if n == 1 {
return FormatString(str, vars...)
}
@@ -326,37 +289,51 @@ func (l *Locale) GetND(dom, str, plural string, n int, vars ...any) string {
}
// GetC uses a domain "default" to return the corresponding Translation of the given string in the given context.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetC(str, ctx string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetC(str, ctx string, vars ...interface{}) string {
return l.GetDC(l.GetDomain(), str, ctx, vars...)
}
// GetNC retrieves the (N)th plural form of Translation for the given string in the given context in the "default" domain.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetNC(str, plural string, n int, ctx string, vars ...any) string {
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetNC(str, plural string, n int, ctx string, vars ...interface{}) string {
return l.GetNDC(l.GetDomain(), str, plural, n, ctx, vars...)
}
// GetDC returns the corresponding Translation in the given domain for the given string in the given context.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetDC(dom, str, ctx string, vars ...any) string {
translator := l.translatorFor(dom)
if translator != nil {
return translator.GetC(str, ctx, vars...)
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetDC(dom, str, ctx string, vars ...interface{}) string {
// Sync read
l.RLock()
defer l.RUnlock()
if l.Domains != nil {
if _, ok := l.Domains[dom]; ok {
if l.Domains[dom] != nil {
return l.Domains[dom].GetC(str, ctx, vars...)
}
}
}
return FormatString(str, vars...)
}
// GetNDC retrieves the (N)th plural form of Translation in the given domain for the given string in the given context.
// Supports optional parameters (vars... any) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetNDC(dom, str, plural string, n int, ctx string, vars ...any) string {
translator := l.translatorFor(dom)
if translator != nil {
return translator.GetNC(str, plural, n, ctx, vars...)
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (l *Locale) GetNDC(dom, str, plural string, n int, ctx string, vars ...interface{}) string {
// Sync read
l.RLock()
defer l.RUnlock()
if l.Domains != nil {
if _, ok := l.Domains[dom]; ok {
if l.Domains[dom] != nil {
return l.Domains[dom].GetNC(str, plural, n, ctx, vars...)
}
}
}
// Use western default rule (n != 1) to handle missing domain default result.
// Use western default rule (plural > 1) to handle missing domain default result.
if n == 1 {
return FormatString(str, vars...)
}
@@ -366,26 +343,13 @@ func (l *Locale) GetNDC(dom, str, plural string, n int, ctx string, vars ...any)
// GetTranslations returns a copy of all translations in all domains of this locale. It does not support contexts.
func (l *Locale) GetTranslations() map[string]*Translation {
all := make(map[string]*Translation)
if l == nil {
return all
}
l.RLock()
translators := make([]Translator, 0, len(l.Domains))
defer l.RUnlock()
for _, translator := range l.Domains {
translators = append(translators, translator)
}
l.RUnlock()
for _, translator := range translators {
if translator == nil {
continue
for msgID, trans := range translator.GetDomain().GetTranslations() {
all[msgID] = trans
}
domain := translator.GetDomain()
if domain == nil {
continue
}
maps.Copy(all, domain.GetTranslations())
}
return all
@@ -393,7 +357,7 @@ func (l *Locale) GetTranslations() map[string]*Translation {
// IsTranslated reports whether a string is translated
func (l *Locale) IsTranslated(str string) bool {
return l.IsTranslatedND(l.GetDomain(), str, 1)
return l.IsTranslatedND(l.GetDomain(), str, 0)
}
// IsTranslatedN reports whether a plural string is translated
@@ -403,25 +367,27 @@ func (l *Locale) IsTranslatedN(str string, n int) bool {
// IsTranslatedD reports whether a domain string is translated
func (l *Locale) IsTranslatedD(dom, str string) bool {
return l.IsTranslatedND(dom, str, 1)
return l.IsTranslatedND(dom, str, 0)
}
// IsTranslatedND reports whether a plural domain string is translated
func (l *Locale) IsTranslatedND(dom, str string, n int) bool {
translator := l.translatorFor(dom)
if translator == nil {
l.RLock()
defer l.RUnlock()
if l.Domains == nil {
return false
}
domain := translator.GetDomain()
if domain == nil {
translator, ok := l.Domains[dom]
if !ok {
return false
}
return domain.IsTranslatedN(str, n)
return translator.GetDomain().IsTranslatedN(str, n)
}
// IsTranslatedC reports whether a context string is translated
func (l *Locale) IsTranslatedC(str, ctx string) bool {
return l.IsTranslatedNDC(l.GetDomain(), str, 1, ctx)
return l.IsTranslatedNDC(l.GetDomain(), str, 0, ctx)
}
// IsTranslatedNC reports whether a plural context string is translated
@@ -431,20 +397,22 @@ func (l *Locale) IsTranslatedNC(str string, n int, ctx string) bool {
// IsTranslatedDC reports whether a domain context string is translated
func (l *Locale) IsTranslatedDC(dom, str, ctx string) bool {
return l.IsTranslatedNDC(dom, str, 1, ctx)
return l.IsTranslatedNDC(dom, str, 0, ctx)
}
// IsTranslatedNDC reports whether a plural domain context string is translated
func (l *Locale) IsTranslatedNDC(dom string, str string, n int, ctx string) bool {
translator := l.translatorFor(dom)
if translator == nil {
l.RLock()
defer l.RUnlock()
if l.Domains == nil {
return false
}
domain := translator.GetDomain()
if domain == nil {
translator, ok := l.Domains[dom]
if !ok {
return false
}
return domain.IsTranslatedNC(str, n, ctx)
return translator.GetDomain().IsTranslatedNC(str, n, ctx)
}
// LocaleEncoding is used as intermediary storage to encode Locale objects to Gob.
@@ -457,35 +425,18 @@ type LocaleEncoding struct {
// MarshalBinary implements encoding BinaryMarshaler interface
func (l *Locale) MarshalBinary() ([]byte, error) {
var localePath, lang, defaultDomain string
domains := make(map[string]Translator)
if l != nil {
l.RLock()
localePath = l.path
lang = l.lang
defaultDomain = l.defaultDomain
domains = make(map[string]Translator, len(l.Domains))
for name, translator := range l.Domains {
if translator != nil {
domains[name] = translator
}
}
l.RUnlock()
}
obj := &LocaleEncoding{
Path: localePath,
Lang: lang,
Domains: make(map[string][]byte, len(domains)),
DefaultDomain: defaultDomain,
}
for name, translator := range domains {
data, err := translator.MarshalBinary()
obj := new(LocaleEncoding)
obj.DefaultDomain = l.defaultDomain
obj.Domains = make(map[string][]byte)
for k, v := range l.Domains {
var err error
obj.Domains[k], err = v.MarshalBinary()
if err != nil {
return nil, err
}
obj.Domains[name] = data
}
obj.Lang = l.lang
obj.Path = l.path
var buff bytes.Buffer
encoder := gob.NewEncoder(&buff)
@@ -494,7 +445,7 @@ func (l *Locale) MarshalBinary() ([]byte, error) {
return buff.Bytes(), err
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler interface
// UnmarshalBinary implements encoding BinaryUnmarshaler interface
func (l *Locale) UnmarshalBinary(data []byte) error {
buff := bytes.NewBuffer(data)
obj := new(LocaleEncoding)
@@ -505,7 +456,12 @@ func (l *Locale) UnmarshalBinary(data []byte) error {
return err
}
domains := make(map[string]Translator, len(obj.Domains))
l.defaultDomain = obj.DefaultDomain
l.lang = obj.Lang
l.path = obj.Path
// Decode Domains
l.Domains = make(map[string]Translator)
for k, v := range obj.Domains {
var tr TranslatorEncoding
buff := bytes.NewBuffer(v)
@@ -515,15 +471,8 @@ func (l *Locale) UnmarshalBinary(data []byte) error {
return err
}
domains[k] = tr.GetTranslator()
l.Domains[k] = tr.GetTranslator()
}
l.Lock()
l.defaultDomain = obj.DefaultDomain
l.lang = obj.Lang
l.path = obj.Path
l.Domains = domains
l.Unlock()
return nil
}
-40
View File
@@ -1,40 +0,0 @@
site_name: gotext
site_description: GNU gettext utilities for Go
site_author: Leonel Quinteros
repo_url: https://github.com/leonelquinteros/gotext
edit_uri: edit/master/docs/
theme:
name: material
palette:
primary: indigo
accent: indigo
features:
- navigation.tabs
- navigation.sections
- navigation.top
- search.suggest
- search.highlight
nav:
- Home: index.md
- Getting Started: GETTING_STARTED.md
- User Guides:
- Plural Forms: PLURALS.md
- xgotext CLI: xgotext.md
- Character Encoding: ENCODING.md
- Advanced Usage: ADVANCED.md
- Best Practices: BEST_PRACTICES.md
- API Reference: https://pkg.go.dev/github.com/leonelquinteros/gotext
- Contributing:
- Guidelines: CONTRIBUTING.md
- Code of Conduct: CODE_OF_CONDUCT.md
- License: LICENSE.md
markdown_extensions:
- admonition
- pymdownx.highlight:
anchor_linenums: true
- pymdownx.inlinehilite
- pymdownx.snippets
- pymdownx.superfences
+94 -62
View File
@@ -76,42 +76,42 @@ func (mo *Mo) GetDomain() *Domain {
}
// Get returns the translation for the given string
func (mo *Mo) Get(str string, vars ...any) string {
func (mo *Mo) Get(str string, vars ...interface{}) string {
return mo.domain.Get(str, vars...)
}
// Append a translation string into the domain
func (mo *Mo) Append(b []byte, str string, vars ...any) []byte {
func (mo *Mo) Append(b []byte, str string, vars ...interface{}) []byte {
return mo.domain.Append(b, str, vars...)
}
// GetN returns the translation for the given string and plural form
func (mo *Mo) GetN(str, plural string, n int, vars ...any) string {
func (mo *Mo) GetN(str, plural string, n int, vars ...interface{}) string {
return mo.domain.GetN(str, plural, n, vars...)
}
// AppendN appends a translation string for the given plural form into the domain
func (mo *Mo) AppendN(b []byte, str, plural string, n int, vars ...any) []byte {
func (mo *Mo) AppendN(b []byte, str, plural string, n int, vars ...interface{}) []byte {
return mo.domain.AppendN(b, str, plural, n, vars...)
}
// GetC returns the translation for the given string and context
func (mo *Mo) GetC(str, ctx string, vars ...any) string {
func (mo *Mo) GetC(str, ctx string, vars ...interface{}) string {
return mo.domain.GetC(str, ctx, vars...)
}
// AppendC appends a translation string for the given context into the domain
func (mo *Mo) AppendC(b []byte, str, ctx string, vars ...any) []byte {
func (mo *Mo) AppendC(b []byte, str, ctx string, vars ...interface{}) []byte {
return mo.domain.AppendC(b, str, ctx, vars...)
}
// GetNC returns the translation for the given string, plural form and context
func (mo *Mo) GetNC(str, plural string, n int, ctx string, vars ...any) string {
func (mo *Mo) GetNC(str, plural string, n int, ctx string, vars ...interface{}) string {
return mo.domain.GetNC(str, plural, n, ctx, vars...)
}
// AppendNC appends a translation string for the given plural form and context into the domain
func (mo *Mo) AppendNC(b []byte, str, plural string, n int, ctx string, vars ...any) []byte {
func (mo *Mo) AppendNC(b []byte, str, plural string, n int, ctx string, vars ...interface{}) []byte {
return mo.domain.AppendNC(b, str, plural, n, ctx, vars...)
}
@@ -163,18 +163,22 @@ func (mo *Mo) Parse(buf []byte) {
defer mo.domain.trMutex.Unlock()
defer mo.domain.pluralMutex.Unlock()
if len(buf) < 28 {
return
}
r := bytes.NewReader(buf)
var magicNumber uint32
if err := binary.Read(r, binary.LittleEndian, &magicNumber); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
var bo binary.ByteOrder
switch binary.LittleEndian.Uint32(buf[:4]) {
switch magicNumber {
case MoMagicLittleEndian:
bo = binary.LittleEndian
case MoMagicBigEndian:
bo = binary.BigEndian
default:
return
// return fmt.Errorf("gettext: %v", "invalid magic number")
}
var header struct {
@@ -183,58 +187,82 @@ func (mo *Mo) Parse(buf []byte) {
MsgIDCount uint32
MsgIDOffset uint32
MsgStrOffset uint32
HashSize uint32
HashOffset uint32
}
bufLen := uint64(len(buf))
header.MajorVersion = bo.Uint16(buf[4:6])
header.MinorVersion = bo.Uint16(buf[6:8])
header.MsgIDCount = bo.Uint32(buf[8:12])
header.MsgIDOffset = bo.Uint32(buf[12:16])
header.MsgStrOffset = bo.Uint32(buf[16:20])
if header.MajorVersion != 0 && header.MajorVersion != 1 {
if err := binary.Read(r, bo, &header); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
if header.MinorVersion != 0 && header.MinorVersion != 1 {
if v := header.MajorVersion; v != 0 && v != 1 {
return
// return fmt.Errorf("gettext: %v", "invalid version number")
}
if v := header.MinorVersion; v != 0 && v != 1 {
return
// return fmt.Errorf("gettext: %v", "invalid version number")
}
directorySize := uint64(header.MsgIDCount) * 8
if uint64(header.MsgIDOffset)+directorySize > bufLen ||
uint64(header.MsgStrOffset)+directorySize > bufLen {
msgIDStart := make([]uint32, header.MsgIDCount)
msgIDLen := make([]uint32, header.MsgIDCount)
if _, err := r.Seek(int64(header.MsgIDOffset), 0); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
for i := uint32(0); i < header.MsgIDCount; i++ {
msgIDEntry := int(uint64(header.MsgIDOffset) + uint64(i)*8)
msgIDLen := bo.Uint32(buf[msgIDEntry:])
msgIDStart := bo.Uint32(buf[msgIDEntry+4:])
if uint64(msgIDStart) > bufLen ||
uint64(msgIDLen) > bufLen-uint64(msgIDStart) {
for i := 0; i < int(header.MsgIDCount); i++ {
if err := binary.Read(r, bo, &msgIDLen[i]); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
msgStrEntry := int(uint64(header.MsgStrOffset) + uint64(i)*8)
msgStrLen := bo.Uint32(buf[msgStrEntry:])
msgStrStart := bo.Uint32(buf[msgStrEntry+4:])
if uint64(msgStrStart) > bufLen ||
uint64(msgStrLen) > bufLen-uint64(msgStrStart) {
if err := binary.Read(r, bo, &msgIDStart[i]); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
}
for i := uint32(0); i < header.MsgIDCount; i++ {
msgIDEntry := int(uint64(header.MsgIDOffset) + uint64(i)*8)
msgIDLen := bo.Uint32(buf[msgIDEntry:])
msgIDStart := bo.Uint32(buf[msgIDEntry+4:])
msgIDEnd := int(uint64(msgIDStart) + uint64(msgIDLen))
msgStrStart := make([]int32, header.MsgIDCount)
msgStrLen := make([]int32, header.MsgIDCount)
if _, err := r.Seek(int64(header.MsgStrOffset), 0); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
for i := 0; i < int(header.MsgIDCount); i++ {
if err := binary.Read(r, bo, &msgStrLen[i]); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
if err := binary.Read(r, bo, &msgStrStart[i]); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
}
msgStrEntry := int(uint64(header.MsgStrOffset) + uint64(i)*8)
msgStrLen := bo.Uint32(buf[msgStrEntry:])
msgStrStart := bo.Uint32(buf[msgStrEntry+4:])
msgStrEnd := int(uint64(msgStrStart) + uint64(msgStrLen))
for i := 0; i < int(header.MsgIDCount); i++ {
if _, err := r.Seek(int64(msgIDStart[i]), 0); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
msgIDData := make([]byte, msgIDLen[i])
if _, err := r.Read(msgIDData); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
mo.addTranslation(buf[int(msgIDStart):msgIDEnd], buf[int(msgStrStart):msgStrEnd])
if _, err := r.Seek(int64(msgStrStart[i]), 0); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
msgStrData := make([]byte, msgStrLen[i])
if _, err := r.Read(msgStrData); err != nil {
return
// return fmt.Errorf("gettext: %v", err)
}
if len(msgIDData) == 0 {
mo.addTranslation(msgIDData, msgStrData)
} else {
mo.addTranslation(msgIDData, msgStrData)
}
}
// Parse headers
@@ -251,32 +279,36 @@ func (mo *Mo) addTranslation(msgid, msgstr []byte) {
translation := NewTranslation()
var msgctxt []byte
var msgidPlural []byte
hasContext := false
if eotIndex := bytes.IndexByte(msgid, EotSeparator[0]); eotIndex >= 0 {
hasContext = true
msgctxt = msgid[:eotIndex]
msgid = msgid[eotIndex+len(EotSeparator):]
d := bytes.Split(msgid, []byte(EotSeparator))
if len(d) == 1 {
msgid = d[0]
} else {
msgid, msgctxt = d[1], d[0]
}
if nulIndex := bytes.IndexByte(msgid, NulSeparator[0]); nulIndex >= 0 {
msgidPlural = msgid[nulIndex+1:]
msgid = msgid[:nulIndex]
dd := bytes.Split(msgid, []byte(NulSeparator))
if len(dd) > 1 {
msgid = dd[0]
dd = dd[1:]
}
translation.ID = string(msgid)
msgidPlural = bytes.Join(dd, []byte(NulSeparator))
if len(msgidPlural) > 0 {
translation.PluralID = string(msgidPlural)
}
i := 0
for s := range bytes.SplitSeq(msgstr, []byte(NulSeparator)) {
translation.Trs[i] = string(s)
i++
ddd := bytes.Split(msgstr, []byte(NulSeparator))
if len(ddd) > 0 {
for i, s := range ddd {
translation.Trs[i] = string(s)
}
}
if hasContext {
// With context, including an explicitly empty context...
if len(msgctxt) > 0 {
// With context...
if _, ok := mo.domain.contextTranslations[string(msgctxt)]; !ok {
mo.domain.contextTranslations[string(msgctxt)] = make(map[string]*Translation)
}
+182 -366
View File
@@ -12,12 +12,18 @@ package plurals
import (
"errors"
"fmt"
"slices"
"regexp"
"strconv"
"strings"
"unicode"
)
type match struct {
openPos int
closePos int
}
var pat = regexp.MustCompile(`(\?|:|\|\||&&|==|!=|>=|>|<=|<|%|\d+|n)`)
type testToken interface {
compile(tokens []string) (test test, err error)
}
@@ -25,30 +31,30 @@ type testToken interface {
type cmpTestBuilder func(val uint32, flipped bool) test
type logicTestBuild func(left test, right test) test
func compileTernary(tokens []string, depth int) (expr Expression, err error) {
if depth > maxParseDepth {
return nil, errors.New("expression nesting is too deep")
}
var ternaryToken ternaryStruct
type ternaryStruct struct{}
func (ternaryStruct) compile(tokens []string) (expr Expression, err error) {
main, err := splitTokens(tokens, "?")
if err != nil {
return nil, err
return expr, err
}
test, err := compileTestDepth(strings.Join(main.Left, ""), depth+1)
test, err := compileTest(strings.Join(main.Left, ""))
if err != nil {
return nil, err
return expr, err
}
actions, err := splitTernaryTokens(main.Right)
actions, err := splitTokens(main.Right, ":")
if err != nil {
return nil, err
return expr, err
}
trueAction, err := compileExpressionDepth(strings.Join(actions.Left, ""), depth+1)
trueAction, err := compileExpression(strings.Join(actions.Left, ""))
if err != nil {
return nil, err
return expr, err
}
falseAction, err := compileExpressionDepth(strings.Join(actions.Right, ""), depth+1)
falseAction, err := compileExpression(strings.Join(actions.Right, ""))
if err != nil {
return nil, err
return expr, nil
}
return ternary{
test: test,
@@ -57,145 +63,93 @@ func compileTernary(tokens []string, depth int) (expr Expression, err error) {
}, nil
}
const maxParseDepth = 1024
func splitTernaryTokens(tokens []string) (s splitted, err error) {
depth := 0
for index, token := range tokens {
switch token {
case "?":
depth++
case ":":
if depth == 0 {
return splitted{
Left: tokens[:index],
Right: tokens[index+1:],
}, nil
}
depth--
}
}
return s, errors.New("':' not found for ternary expression")
}
var constToken constValStruct
type constValStruct struct{}
func (constValStruct) compile(tokens []string) (expr Expression, err error) {
if len(tokens) == 0 {
return nil, errors.New("got nothing instead of constant")
return expr, errors.New("got nothing instead of constant")
}
if len(tokens) != 1 {
return nil, fmt.Errorf("invalid constant: %s", strings.Join(tokens, ""))
return expr, fmt.Errorf("invalid constant: %s", strings.Join(tokens, ""))
}
i, err := strconv.Atoi(tokens[0])
if err != nil {
return nil, err
return expr, err
}
return constValue{value: i}, nil
}
func compileLogicTestDepth(tokens []string, sep string, builder logicTestBuild, depth int) (test test, err error) {
if depth > maxParseDepth {
return nil, errors.New("expression nesting is too deep")
}
func compileLogicTest(tokens []string, sep string, builder logicTestBuild) (test test, err error) {
split, err := splitTokens(tokens, sep)
if err != nil {
return nil, err
return test, err
}
left, err := compileTestDepth(strings.Join(split.Left, ""), depth+1)
left, err := compileTest(strings.Join(split.Left, ""))
if err != nil {
return nil, err
return test, err
}
right, err := compileTestDepth(strings.Join(split.Right, ""), depth+1)
right, err := compileTest(strings.Join(split.Right, ""))
if err != nil {
return nil, err
return test, err
}
return builder(left, right), nil
}
var orToken orStruct
type orStruct struct{}
func (orStruct) compile(tokens []string) (test test, err error) {
return compileLogicTest(tokens, "||", buildOr)
}
func buildOr(left test, right test) test {
return or{left: left, right: right}
}
var andToken andStruct
type andStruct struct{}
func (andStruct) compile(tokens []string) (test test, err error) {
return compileLogicTest(tokens, "&&", buildAnd)
}
func buildAnd(left test, right test) test {
return and{left: left, right: right}
}
func unwrapSingleGroup(tokens []string) ([]string, error) {
for len(tokens) == 1 && strings.HasPrefix(tokens[0], "(") {
inner, err := tokenize(tokens[0])
if err != nil {
return nil, err
}
if len(inner) == 1 && inner[0] == tokens[0] {
break
}
tokens = inner
}
return tokens, nil
}
func isSimpleN(tokens []string) bool {
tokens, err := unwrapSingleGroup(tokens)
if err != nil || len(tokens) != 1 {
return false
}
return tokens[0] == "n"
}
func parseLiteralTokens(tokens []string) (uint32, error) {
tokens, err := unwrapSingleGroup(tokens)
if err != nil {
return 0, err
}
if len(tokens) != 1 {
return 0, errors.New("expected a single integer")
}
return parseUint32(tokens[0])
}
func containsSimpleN(tokens []string) bool {
for _, token := range tokens {
if isSimpleN([]string{token}) {
return true
}
}
return false
}
func compileMod(tokens []string) (math math, err error) {
split, err := splitTokens(tokens, "%")
if err != nil {
return math, err
}
if !isSimpleN(split.Left) {
if len(split.Left) != 1 || split.Left[0] != "n" {
return math, errors.New("modulus operation requires 'n' as left operand")
}
i, err := parseLiteralTokens(split.Right)
if len(split.Right) != 1 {
return math, errors.New("modulus operation requires simple integer as right operand")
}
i, err := parseUint32(split.Right[0])
if err != nil {
return math, err
}
if i == 0 {
return math, errors.New("modulus operation requires a non-zero integer as right operand")
}
return mod{value: i}, nil
return mod{value: uint32(i)}, nil
}
func subPipe(modTokens []string, actionTokens []string, builder cmpTestBuilder, flipped bool) (test test, err error) {
modifier, err := compileMod(modTokens)
if err != nil {
return nil, err
return test, err
}
if len(actionTokens) == 0 {
return nil, errors.New("can only get modulus of integer")
if len(actionTokens) != 1 {
return test, errors.New("can only get modulus of integer")
}
i, err := parseLiteralTokens(actionTokens)
i, err := parseUint32(actionTokens[0])
if err != nil {
return nil, err
return test, err
}
action := builder(i, flipped)
action := builder(uint32(i), flipped)
return pipe{
modifier: modifier,
action: action,
@@ -205,37 +159,31 @@ func subPipe(modTokens []string, actionTokens []string, builder cmpTestBuilder,
func compileEquality(tokens []string, sep string, builder cmpTestBuilder) (test test, err error) {
split, err := splitTokens(tokens, sep)
if err != nil {
return nil, err
return test, err
}
left, err := unwrapSingleGroup(split.Left)
if err != nil {
return nil, err
}
right, err := unwrapSingleGroup(split.Right)
if err != nil {
return nil, err
}
if isSimpleN(left) {
i, err := parseLiteralTokens(right)
if len(split.Left) == 1 && split.Left[0] == "n" {
if len(split.Right) != 1 {
return test, errors.New("test can only compare n to integers")
}
i, err := parseUint32(split.Right[0])
if err != nil {
return nil, err
return test, err
}
return builder(i, false), nil
}
if isSimpleN(right) {
i, err := parseLiteralTokens(left)
} else if len(split.Right) == 1 && split.Right[0] == "n" {
if len(split.Left) != 1 {
return test, errors.New("test can only compare n to integers")
}
i, err := parseUint32(split.Left[0])
if err != nil {
return nil, err
return test, err
}
return builder(i, true), nil
} else if contains(split.Left, "n") && contains(split.Left, "%") {
return subPipe(split.Left, split.Right, builder, false)
}
if containsSimpleN(left) && slices.Contains(left, "%") {
return subPipe(left, right, builder, false)
}
if containsSimpleN(right) && slices.Contains(right, "%") {
return subPipe(right, left, builder, true)
}
return nil, errors.New("equality test must have 'n' as one of the two tests")
return test, errors.New("equality test must have 'n' as one of the two tests")
}
var eqToken eqStruct
@@ -310,8 +258,8 @@ type testTokenDef struct {
}
var precedence = []testTokenDef{
{op: "||"},
{op: "&&"},
{op: "||", token: orToken},
{op: "&&", token: andToken},
{op: "==", token: eqToken},
{op: "!=", token: neqToken},
{op: ">=", token: gteToken},
@@ -325,10 +273,20 @@ type splitted struct {
Right []string
}
// Find index of token in list of tokens
func index(tokens []string, sep string) int {
for index, token := range tokens {
if token == sep {
return index
}
}
return -1
}
// Split a list of tokens by a token into a splitted struct holding the tokens
// before and after the token to be split by.
func splitTokens(tokens []string, sep string) (s splitted, err error) {
index := slices.Index(tokens, sep)
index := index(tokens, sep)
if index == -1 {
return s, fmt.Errorf("'%s' not found in ['%s']", sep, strings.Join(tokens, "','"))
}
@@ -338,277 +296,135 @@ func splitTokens(tokens []string, sep string) (s splitted, err error) {
}, nil
}
func trimLexicalWhitespace(s string) string {
return strings.Map(func(r rune) rune {
if unicode.IsSpace(r) {
return -1
// Scan a string for parenthesis
func scan(s string) <-chan match {
ch := make(chan match)
go func() {
depth := 0
opener := 0
for index, char := range s {
switch char {
case '(':
if depth == 0 {
opener = index
}
depth++
case ')':
depth--
if depth == 0 {
ch <- match{
openPos: opener,
closePos: index + 1,
}
}
}
}
return r
}, s)
close(ch)
}()
return ch
}
func lexicalToken(s string, index int) (token string, next int, err error) {
if index >= len(s) {
return "", index, errors.New("unexpected end of expression")
}
switch s[index] {
case '?', ':', '%', '(', ')':
return s[index : index+1], index + 1, nil
case '|':
if index+1 >= len(s) || s[index+1] != '|' {
return "", index, errors.New("logical OR must use '||'")
}
return "||", index + 2, nil
case '&':
if index+1 >= len(s) || s[index+1] != '&' {
return "", index, errors.New("logical AND must use '&&'")
}
return "&&", index + 2, nil
case '=':
if index+1 >= len(s) || s[index+1] != '=' {
return "", index, errors.New("equality must use '=='")
}
return "==", index + 2, nil
case '!':
if index+1 >= len(s) || s[index+1] != '=' {
return "", index, errors.New("inequality must use '!='")
}
return "!=", index + 2, nil
case '>':
if index+1 < len(s) && s[index+1] == '=' {
return ">=", index + 2, nil
}
return ">", index + 1, nil
case '<':
if index+1 < len(s) && s[index+1] == '=' {
return "<=", index + 2, nil
}
return "<", index + 1, nil
case 'n':
return "n", index + 1, nil
default:
if s[index] >= '0' && s[index] <= '9' {
next = index + 1
for next < len(s) && s[next] >= '0' && s[next] <= '9' {
next++
// Split the string into tokens
func split(s string) <-chan string {
ch := make(chan string)
go func() {
s = strings.ReplaceAll(s, " ", "")
if !strings.Contains(s, "(") {
ch <- s
} else {
last := 0
end := len(s)
for info := range scan(s) {
if last != info.openPos {
ch <- s[last:info.openPos]
}
ch <- s[info.openPos:info.closePos]
last = info.closePos
}
return s[index:next], next, nil
}
return "", index, fmt.Errorf("invalid character %q in expression", s[index])
}
}
func validateParentheses(s string) error {
depth := 0
for index := 0; index < len(s); {
token, next, err := lexicalToken(s, index)
if err != nil {
return err
}
switch token {
case "(":
depth++
case ")":
depth--
if depth < 0 {
return errors.New("unmatched closing parenthesis")
if last != end {
ch <- s[last:]
}
}
index = next
}
if depth != 0 {
return errors.New("unmatched opening parenthesis")
}
return nil
}
func stripRedundantOuterParens(s string) string {
if len(s) < 2 || s[0] != '(' {
return s
}
leading := 0
for leading < len(s) && s[leading] == '(' {
leading++
}
openers := make([]int, 0, leading)
outer := leading
for index := range s {
switch s[index] {
case '(':
openers = append(openers, index)
case ')':
if len(openers) == 0 {
return s
}
opener := openers[len(openers)-1]
openers = openers[:len(openers)-1]
if opener < outer && index != len(s)-1-opener {
outer = opener
}
}
}
if len(openers) != 0 {
return s
}
return s[outer : len(s)-outer]
close(ch)
}()
return ch
}
// Tokenizes a string into a list of strings, tokens grouped by parenthesis are
// not split! If the string starts with ( and ends in ), those are stripped.
func tokenize(s string) ([]string, error) {
s = trimLexicalWhitespace(s)
if s == "" {
return []string{}, nil
}
if err := validateParentheses(s); err != nil {
return nil, err
}
s = stripRedundantOuterParens(s)
if s == "" {
return []string{}, nil
}
func tokenize(s string) []string {
/*
TODO: Properly detect if the string starts with a ( and ends with a )
and that those two form a matching pair.
ret := []string{}
for index := 0; index < len(s); {
if s[index] == '(' {
start := index
depth := 0
groupEnd := false
for index < len(s) {
switch s[index] {
case '(':
depth++
case ')':
depth--
if depth == 0 {
index++
ret = append(ret, s[start:index])
groupEnd = true
}
}
if groupEnd {
break
}
index++
}
if depth != 0 {
return nil, errors.New("unmatched opening parenthesis")
}
continue
}
token, next, err := lexicalToken(s, index)
if err != nil {
return nil, err
}
if token == ")" {
return nil, errors.New("unmatched closing parenthesis")
}
ret = append(ret, token)
index = next
Eg: (foo) -> true; (foo)(bar) -> false;
*/
if len(s) == 0 {
return []string{}
}
return ret, nil
if s[0] == '(' && s[len(s)-1] == ')' {
s = s[1 : len(s)-1]
}
ret := []string{}
for chunk := range split(s) {
if len(chunk) != 0 {
if chunk[0] == '(' && chunk[len(chunk)-1] == ')' {
ret = append(ret, chunk)
} else {
for _, token := range pat.FindAllStringSubmatch(chunk, -1) {
ret = append(ret, token[0])
}
}
} else {
fmt.Printf("Empty chunk in string '%s'\n", s)
}
}
return ret
}
// Compile a string containing a plural form expression to a Expression object.
func Compile(s string) (expr Expression, err error) {
s = trimLexicalWhitespace(s)
if s == "" {
return nil, errors.New("empty expression")
if s == "0" {
return constValue{value: 0}, nil
}
return compileExpressionDepth(s, 0)
if !strings.Contains(s, "?") {
s += "?1:0"
}
return compileExpression(s)
}
// Compiles an expression (ternary, numeric leaf, or test).
func compileExpressionDepth(s string, depth int) (expr Expression, err error) {
if depth > maxParseDepth {
return nil, errors.New("expression nesting is too deep")
}
tokens, err := tokenize(s)
if err != nil {
return nil, err
}
if slices.Contains(tokens, "?") {
return compileTernary(tokens, depth)
}
for _, tokenDef := range precedence {
if !slices.Contains(tokens, tokenDef.op) {
continue
// Check if a token is in a slice of strings
func contains(haystack []string, needle string) bool {
for _, s := range haystack {
if s == needle {
return true
}
condition, err := compileTestDepth(s, depth+1)
if err != nil {
return nil, err
}
return testValue{condition: condition}, nil
}
if isSimpleN(tokens) {
return variableValue{}, nil
}
if slices.Contains(tokens, "%") {
modifier, err := compileMod(tokens)
if err != nil {
return nil, err
}
return mathValue{value: modifier}, nil
return false
}
// Compiles an expression (ternary or constant)
func compileExpression(s string) (expr Expression, err error) {
tokens := tokenize(s)
if contains(tokens, "?") {
return ternaryToken.compile(tokens)
}
return constToken.compile(tokens)
}
// Compiles a test (comparison)
func compileTestDepth(s string, depth int) (test test, err error) {
if depth > maxParseDepth {
return nil, errors.New("expression nesting is too deep")
}
tokens, err := tokenize(s)
if err != nil {
return nil, err
}
func compileTest(s string) (test test, err error) {
tokens := tokenize(s)
for _, tokenDef := range precedence {
if !slices.Contains(tokens, tokenDef.op) {
continue
}
switch tokenDef.op {
case "||":
return compileLogicTestDepth(tokens, "||", buildOr, depth)
case "&&":
return compileLogicTestDepth(tokens, "&&", buildAnd, depth)
default:
if contains(tokens, tokenDef.op) {
return tokenDef.token.compile(tokens)
}
}
if slices.Contains(tokens, "%") {
m, err := compileMod(tokens)
if err != nil {
return nil, err
}
return pipe{
modifier: m,
action: notequal{value: 0},
}, nil
}
if len(tokens) == 1 && strings.HasPrefix(tokens[0], "(") {
inner, err := tokenize(tokens[0])
if err != nil {
return nil, err
}
if len(inner) == 1 && inner[0] == tokens[0] {
return nil, errors.New("cannot compile parenthesized test")
}
return compileTestDepth(tokens[0], depth+1)
}
if len(tokens) == 1 && tokens[0] == "n" {
return notequal{value: 0}, nil
}
if value, err := parseLiteralTokens(tokens); err == nil {
return literalTest{value: value}, nil
}
return nil, errors.New("cannot compile")
return test, errors.New("cannot compile")
}
func parseUint32(s string) (ui uint32, err error) {
s = trimLexicalWhitespace(s)
i, err := strconv.ParseUint(s, 10, 32)
if err != nil {
return ui, err
-34
View File
@@ -20,37 +20,6 @@ func (c constValue) Eval(n uint32) int {
return c.value
}
type variableValue struct{}
func (variableValue) Eval(n uint32) int {
return int(n)
}
type mathValue struct {
value math
}
func (m mathValue) Eval(n uint32) int {
if m.value == nil {
return -1
}
return int(m.value.calc(n))
}
type testValue struct {
condition test
}
func (t testValue) Eval(n uint32) int {
if t.condition == nil {
return -1
}
if t.condition.test(n) {
return 1
}
return 0
}
type test interface {
test(n uint32) bool
}
@@ -62,9 +31,6 @@ type ternary struct {
}
func (t ternary) Eval(n uint32) int {
if t.test == nil {
return -1
}
if t.test.test(n) {
if t.trueExpr == nil {
return -1
-3
View File
@@ -15,8 +15,5 @@ type mod struct {
}
func (m mod) calc(n uint32) uint32 {
if m.value == 0 {
return 0
}
return n % m.value
}
-17
View File
@@ -22,14 +22,6 @@ func (e notequal) test(n uint32) bool {
return n != e.value
}
type literalTest struct {
value uint32
}
func (l literalTest) test(n uint32) bool {
return l.value != 0
}
type gt struct {
value uint32
flipped bool
@@ -85,9 +77,6 @@ type and struct {
}
func (e and) test(n uint32) bool {
if e.left == nil || e.right == nil {
return false
}
if !e.left.test(n) {
return false
}
@@ -100,9 +89,6 @@ type or struct {
}
func (e or) test(n uint32) bool {
if e.left == nil || e.right == nil {
return false
}
if e.left.test(n) {
return true
}
@@ -115,8 +101,5 @@ type pipe struct {
}
func (e pipe) test(n uint32) bool {
if e.modifier == nil || e.action == nil {
return false
}
return e.action.test(e.modifier.calc(n))
}
+80 -307
View File
@@ -42,10 +42,6 @@ type Po struct {
domain *Domain
fs fs.FS
parseBufferHasID bool
parseBufferHasPlural bool
parseBufferHasContext bool
}
type parseState int
@@ -56,7 +52,6 @@ const (
msgID
msgIDPlural
msgStr
msgStrInvalid
)
// NewPo should always be used to instantiate a new Po object
@@ -99,7 +94,7 @@ func (po *Po) GetRefs(str string) []string {
// SetPluralResolver sets the plural resolver function
func (po *Po) SetPluralResolver(f func(int) int) {
po.domain.SetPluralResolver(f)
po.domain.customPluralResolver = f
}
// Set translation
@@ -108,12 +103,12 @@ func (po *Po) Set(id, str string) {
}
// Get translation
func (po *Po) Get(str string, vars ...any) string {
func (po *Po) Get(str string, vars ...interface{}) string {
return po.domain.Get(str, vars...)
}
// Append translation
func (po *Po) Append(b []byte, str string, vars ...any) []byte {
func (po *Po) Append(b []byte, str string, vars ...interface{}) []byte {
return po.domain.Append(b, str, vars...)
}
@@ -123,12 +118,12 @@ func (po *Po) SetN(id, plural string, n int, str string) {
}
// GetN gets the plural translation
func (po *Po) GetN(str, plural string, n int, vars ...any) string {
func (po *Po) GetN(str, plural string, n int, vars ...interface{}) string {
return po.domain.GetN(str, plural, n, vars...)
}
// AppendN appends the plural translation
func (po *Po) AppendN(b []byte, str, plural string, n int, vars ...any) []byte {
func (po *Po) AppendN(b []byte, str, plural string, n int, vars ...interface{}) []byte {
return po.domain.AppendN(b, str, plural, n, vars...)
}
@@ -138,12 +133,12 @@ func (po *Po) SetC(id, ctx, str string) {
}
// GetC gets the translation for a given context
func (po *Po) GetC(str, ctx string, vars ...any) string {
func (po *Po) GetC(str, ctx string, vars ...interface{}) string {
return po.domain.GetC(str, ctx, vars...)
}
// AppendC appends the translation for a given context
func (po *Po) AppendC(b []byte, str, ctx string, vars ...any) []byte {
func (po *Po) AppendC(b []byte, str, ctx string, vars ...interface{}) []byte {
return po.domain.AppendC(b, str, ctx, vars...)
}
@@ -153,12 +148,12 @@ func (po *Po) SetNC(id, plural, ctx string, n int, str string) {
}
// GetNC gets the plural translation for a given context
func (po *Po) GetNC(str, plural string, n int, ctx string, vars ...any) string {
func (po *Po) GetNC(str, plural string, n int, ctx string, vars ...interface{}) string {
return po.domain.GetNC(str, plural, n, ctx, vars...)
}
// AppendNC appends the plural translation for a given context
func (po *Po) AppendNC(b []byte, str, plural string, n int, ctx string, vars ...any) []byte {
func (po *Po) AppendNC(b []byte, str, plural string, n int, ctx string, vars ...interface{}) []byte {
return po.domain.AppendNC(b, str, plural, n, ctx, vars...)
}
@@ -219,17 +214,16 @@ func (po *Po) Parse(buf []byte) {
defer po.domain.trMutex.Unlock()
defer po.domain.pluralMutex.Unlock()
// Get lines
lines := strings.Split(string(buf), "\n")
// Init buffer
po.domain.trBuffer = NewTranslation()
po.domain.ctxBuffer = ""
po.domain.refBuffer = ""
po.parseBufferHasID = false
po.parseBufferHasPlural = false
po.parseBufferHasContext = false
state := head
activeMsgStrIndex := 0
for l := range strings.Lines(string(buf)) {
for _, l := range lines {
// Trim spaces
l = strings.TrimSpace(l)
@@ -239,71 +233,44 @@ func (po *Po) Parse(buf []byte) {
continue
}
// Multi line strings and headers
if strings.HasPrefix(l, "\"") {
if !po.parseString(l, state, activeMsgStrIndex) && state != head {
state = msgStrInvalid
}
continue
}
// Buffer context and continue
if po.hasKeywordBoundary(l, "msgctxt") {
if po.parseContext(l) {
state = msgCtxt
} else {
state = msgStrInvalid
}
continue
}
// Check for plural form
if po.hasKeywordBoundary(l, "msgid_plural") {
if state != msgID || !po.parsePluralID(l) {
state = msgStrInvalid
} else {
state = msgIDPlural
}
if strings.HasPrefix(l, "msgctxt") {
po.parseContext(l)
state = msgCtxt
continue
}
// Buffer msgid and continue
if po.hasKeywordBoundary(l, "msgid") {
if po.parseID(l) {
state = msgID
} else {
state = msgStrInvalid
}
if strings.HasPrefix(l, "msgid") && !strings.HasPrefix(l, "msgid_plural") {
po.parseID(l)
state = msgID
continue
}
// Check for plural form
if strings.HasPrefix(l, "msgid_plural") {
po.parsePluralID(l)
po.domain.pluralTranslations[po.domain.trBuffer.PluralID] = po.domain.trBuffer
state = msgIDPlural
continue
}
// Save Translation
if po.hasKeywordBoundary(l, "msgstr") {
if state != msgID && state != msgIDPlural && state != msgStr {
activeMsgStrIndex = 0
state = msgStrInvalid
continue
}
if strings.HasPrefix(l, "msgstr") {
po.parseMessage(l)
state = msgStr
continue
}
var ok bool
activeMsgStrIndex, ok = po.parseMessage(l)
if ok {
state = msgStr
} else {
activeMsgStrIndex = 0
state = msgStrInvalid
}
// Multi line strings and headers
if strings.HasPrefix(l, "\"") && strings.HasSuffix(l, "\"") {
po.parseString(l, state)
continue
}
}
// Save last Translation buffer, but do not synthesize an entry after an
// invalid directive. Preserve the historical empty-input entry.
if po.parseBufferHasID {
po.saveBuffer()
} else if len(buf) == 0 {
po.domain.translations[""] = po.domain.trBuffer
}
// Save last Translation buffer.
po.saveBuffer()
// Parse headers
po.domain.parseHeaders()
@@ -318,30 +285,22 @@ func (po *Po) Parse(buf []byte) {
// saveBuffer takes the context and Translation buffers
// and saves it on the translations collection
func (po *Po) saveBuffer() {
if !po.parseBufferHasID {
return
}
// Store a completed plural ID after all of its continuation lines have
// been decoded.
if po.parseBufferHasPlural {
po.domain.pluralTranslations[po.domain.trBuffer.PluralID] = po.domain.trBuffer
}
if po.parseBufferHasContext {
// With no context...
if po.domain.ctxBuffer == "" {
po.domain.translations[po.domain.trBuffer.ID] = po.domain.trBuffer
} else {
// With context...
if _, ok := po.domain.contextTranslations[po.domain.ctxBuffer]; !ok {
po.domain.contextTranslations[po.domain.ctxBuffer] = make(map[string]*Translation)
}
po.domain.contextTranslations[po.domain.ctxBuffer][po.domain.trBuffer.ID] = po.domain.trBuffer
// Context applies only to the record it prefixes, even when its ID is empty.
po.domain.ctxBuffer = ""
po.parseBufferHasContext = false
} else {
po.domain.translations[po.domain.trBuffer.ID] = po.domain.trBuffer
// Cleanup current context buffer if needed
if po.domain.trBuffer.ID != "" {
po.domain.ctxBuffer = ""
}
}
po.parseBufferHasID = false
po.parseBufferHasPlural = false
// Flush Translation buffer
if po.domain.refBuffer == "" {
po.domain.trBuffer = NewTranslation()
@@ -369,106 +328,68 @@ func (po *Po) parseComment(l string, state parseState) {
// parseContext takes a line starting with "msgctxt",
// saves the current Translation buffer and creates a new context.
func (po *Po) parseContext(l string) bool {
value, ok := po.parseQuotedDirective(l, "msgctxt")
if !ok {
return false
}
func (po *Po) parseContext(l string) {
// Save current Translation buffer.
po.saveBuffer()
// Buffer context, including an explicitly empty context.
po.domain.ctxBuffer = value
po.parseBufferHasContext = true
return true
// Buffer context
po.domain.ctxBuffer, _ = strconv.Unquote(strings.TrimSpace(strings.TrimPrefix(l, "msgctxt")))
}
// parseID takes a line starting with "msgid",
// saves the current Translation and creates a new msgid buffer.
func (po *Po) parseID(l string) bool {
value, ok := po.parseQuotedDirective(l, "msgid")
if !ok {
return false
}
func (po *Po) parseID(l string) {
// Save current Translation buffer.
po.saveBuffer()
// Set id
po.domain.trBuffer.ID = value
po.parseBufferHasID = true
return true
po.domain.trBuffer.ID, _ = strconv.Unquote(strings.TrimSpace(strings.TrimPrefix(l, "msgid")))
}
// parsePluralID saves the plural id buffer from a line starting with "msgid_plural"
func (po *Po) parsePluralID(l string) bool {
value, ok := po.parseQuotedDirective(l, "msgid_plural")
if !ok {
return false
}
po.domain.trBuffer.PluralID = value
po.parseBufferHasPlural = true
return true
func (po *Po) parsePluralID(l string) {
po.domain.trBuffer.PluralID, _ = strconv.Unquote(strings.TrimSpace(strings.TrimPrefix(l, "msgid_plural")))
}
// parseMessage takes a line starting with "msgstr" and saves it into the current buffer.
func (po *Po) parseMessage(l string) (int, bool) {
if !po.hasKeywordBoundary(l, "msgstr") {
return 0, false
}
l = strings.TrimSpace(l[len("msgstr"):])
func (po *Po) parseMessage(l string) {
l = strings.TrimSpace(strings.TrimPrefix(l, "msgstr"))
// Check for indexed Translation forms
if strings.HasPrefix(l, "[") {
idx := strings.IndexByte(l, ']')
if idx <= 1 || !isASCIIPluralIndex(l[1:idx]) {
idx := strings.Index(l, "]")
if idx == -1 {
// Skip wrong index formatting
return 0, false
return
}
// Parse index
i, err := strconv.Atoi(l[1:idx])
if err != nil {
// Skip wrong index formatting
return 0, false
return
}
// Parse Translation string
clean, ok := decodePOQuotedString(strings.TrimSpace(l[idx+1:]))
if !ok {
return 0, false
}
po.domain.trBuffer.Trs[i] = clean
po.domain.trBuffer.Trs[i], _ = strconv.Unquote(strings.TrimSpace(l[idx+1:]))
return i, true
// Loop
return
}
// Save single Translation form under 0 index
clean, ok := decodePOQuotedString(l)
if !ok {
return 0, false
}
po.domain.trBuffer.Trs[0] = clean
return 0, true
po.domain.trBuffer.Trs[0], _ = strconv.Unquote(l)
}
// parseString takes a well formatted string without prefix
// and creates headers or attach multi-line strings when corresponding
func (po *Po) parseString(l string, state parseState, activeMsgStrIndex int) bool {
if state == msgStrInvalid {
return false
}
clean, ok := decodePOQuotedString(l)
if !ok {
return false
}
func (po *Po) parseString(l string, state parseState) {
clean, _ := strconv.Unquote(l)
switch state {
case msgStr:
// Append to active Translation form
po.domain.trBuffer.Trs[activeMsgStrIndex] += clean
// Append to last Translation found
po.domain.trBuffer.Trs[len(po.domain.trBuffer.Trs)-1] += clean
case msgID:
// Multiline msgid - Append to current id
@@ -481,174 +402,26 @@ func (po *Po) parseString(l string, state parseState, activeMsgStrIndex int) boo
case msgCtxt:
// Multiline context - Append to current context
po.domain.ctxBuffer += clean
}
return true
}
}
// isValidLine checks for line prefixes to detect valid syntax.
func (po *Po) isValidLine(l string) bool {
if strings.HasPrefix(l, "\"") {
return true
// Check prefix
valid := []string{
"\"",
"msgctxt",
"msgid",
"msgid_plural",
"msgstr",
}
for _, keyword := range []string{
"msgctxt",
"msgid_plural",
"msgid",
"msgstr",
} {
if po.hasKeywordBoundary(l, keyword) {
for _, v := range valid {
if strings.HasPrefix(l, v) {
return true
}
}
return false
}
func (po *Po) hasKeywordBoundary(l, keyword string) bool {
if !strings.HasPrefix(l, keyword) {
return false
}
if len(l) == len(keyword) {
return true
}
next := l[len(keyword)]
if keyword == "msgstr" && next == '[' {
return true
}
return next == ' ' || next == '\t'
}
func (po *Po) parseQuotedDirective(l, keyword string) (string, bool) {
if !po.hasKeywordBoundary(l, keyword) {
return "", false
}
value, ok := decodePOQuotedString(strings.TrimSpace(l[len(keyword):]))
if !ok {
return "", false
}
return value, true
}
// decodePOQuotedString decodes a GNU PO double-quoted string.
func decodePOQuotedString(source string) (string, bool) {
if len(source) < 2 || source[0] != '"' {
return "", false
}
var decoded []byte
for i := 1; i < len(source); {
switch source[i] {
case '"':
if i != len(source)-1 {
return "", false
}
if decoded == nil {
return source[1:i], true
}
return string(decoded), true
case '\\':
if decoded == nil {
decoded = make([]byte, 0, len(source)-2)
decoded = append(decoded, source[1:i]...)
}
i++
if i >= len(source) {
return "", false
}
switch source[i] {
case 'a':
decoded = append(decoded, '\a')
i++
case 'b':
decoded = append(decoded, '\b')
i++
case 'f':
decoded = append(decoded, '\f')
i++
case 'n':
decoded = append(decoded, '\n')
i++
case 'r':
decoded = append(decoded, '\r')
i++
case 't':
decoded = append(decoded, '\t')
i++
case 'v':
decoded = append(decoded, '\v')
i++
case '\\':
decoded = append(decoded, '\\')
i++
case '"':
decoded = append(decoded, '"')
i++
case '0', '1', '2', '3', '4', '5', '6', '7':
var value byte
for digits := 0; i < len(source) && digits < 3; digits++ {
if source[i] < '0' || source[i] > '7' {
break
}
value = value<<3 | (source[i] - '0')
i++
}
decoded = append(decoded, value)
case 'x':
i++
if i >= len(source) || !isPOHexDigit(source[i]) {
return "", false
}
var value byte
for i < len(source) && isPOHexDigit(source[i]) {
value = value<<4 | poHexValue(source[i])
i++
}
decoded = append(decoded, value)
default:
return "", false
}
default:
if decoded != nil {
decoded = append(decoded, source[i])
}
i++
}
}
return "", false
}
func isPOHexDigit(c byte) bool {
return c >= '0' && c <= '9' ||
c >= 'a' && c <= 'f' ||
c >= 'A' && c <= 'F'
}
func poHexValue(c byte) byte {
switch {
case c >= '0' && c <= '9':
return c - '0'
case c >= 'a' && c <= 'f':
return c - 'a' + 10
default:
return c - 'A' + 10
}
}
func isASCIIPluralIndex(index string) bool {
if index == "" {
return false
}
for i := 0; i < len(index); i++ {
if index[i] < '0' || index[i] > '9' {
return false
}
}
return true
}
+14 -6
View File
@@ -49,8 +49,11 @@ func (t *Translation) Set(str string) {
// Get returns the string of the translation
func (t *Translation) Get() string {
if tr := t.Trs[0]; tr != "" {
return tr
// Look for Translation index 0
if _, ok := t.Trs[0]; ok {
if t.Trs[0] != "" {
return t.Trs[0]
}
}
// Return untranslated id by default
@@ -65,8 +68,11 @@ func (t *Translation) SetN(n int, str string) {
// GetN returns the string of the plural translation
func (t *Translation) GetN(n int) string {
if tr := t.Trs[n]; tr != "" {
return tr
// Look for Translation index
if _, ok := t.Trs[n]; ok {
if t.Trs[n] != "" {
return t.Trs[n]
}
}
// Return untranslated singular if corresponding
@@ -80,10 +86,12 @@ func (t *Translation) GetN(n int) string {
// IsTranslated reports whether a string is translated
func (t *Translation) IsTranslated() bool {
return t.Trs[0] != ""
tr, ok := t.Trs[0]
return tr != "" && ok
}
// IsTranslatedN reports whether a plural string is translated
func (t *Translation) IsTranslatedN(n int) bool {
return t.Trs[n] != ""
tr, ok := t.Trs[n]
return tr != "" && ok
}
+14 -60
View File
@@ -9,20 +9,18 @@ import (
"errors"
"io/fs"
"os"
"github.com/leonelquinteros/gotext/plurals"
)
// Translator interface is used by Locale and Po objects.
// Translator interface is used by Locale and Po objects.Translator
// It contains all methods needed to parse translation sources and obtain corresponding translations.
// It includes encoding.BinaryMarshaler and encoding.BinaryUnmarshaler methods for Locale serialization.
// Also implements gob.GobEncoder/gob.DobDecoder interfaces to allow serialization of Locale objects.
type Translator interface {
ParseFile(f string)
Parse(buf []byte)
Get(str string, vars ...any) string
GetN(str, plural string, n int, vars ...any) string
GetC(str, ctx string, vars ...any) string
GetNC(str, plural string, n int, ctx string, vars ...any) string
Get(str string, vars ...interface{}) string
GetN(str, plural string, n int, vars ...interface{}) string
GetC(str, ctx string, vars ...interface{}) string
GetNC(str, plural string, n int, ctx string, vars ...interface{}) string
MarshalBinary() ([]byte, error)
UnmarshalBinary([]byte) error
@@ -33,10 +31,10 @@ type Translator interface {
// It contains all methods needed to parse translation sources and to append entries to the object.
type AppendTranslator interface {
Translator
Append(b []byte, str string, vars ...any) []byte
AppendN(b []byte, str, plural string, n int, vars ...any) []byte
AppendC(b []byte, str, ctx string, vars ...any) []byte
AppendNC(b []byte, str, plural string, n int, ctx string, vars ...any) []byte
Append(b []byte, str string, vars ...interface{}) []byte
AppendN(b []byte, str, plural string, n int, vars ...interface{}) []byte
AppendC(b []byte, str, ctx string, vars ...interface{}) []byte
AppendNC(b []byte, str, plural string, n int, ctx string, vars ...interface{}) []byte
}
// TranslatorEncoding is used as intermediary storage to encode Translator objects to Gob.
@@ -65,58 +63,14 @@ type TranslatorEncoding struct {
// deserialize into a Po-compatible object.
func (te *TranslatorEncoding) GetTranslator() Translator {
po := NewPo()
if te == nil {
return po
}
headers := te.Headers
if headers == nil {
headers = make(HeaderMap)
}
translations := te.Translations
if translations == nil {
translations = make(map[string]*Translation)
}
for id, translation := range translations {
if translation == nil {
translation = NewTranslation()
translation.ID = id
translations[id] = translation
}
}
contexts := te.Contexts
if contexts == nil {
contexts = make(map[string]map[string]*Translation)
}
for context, translationsForContext := range contexts {
if translationsForContext == nil {
contexts[context] = make(map[string]*Translation)
continue
}
for id, translation := range translationsForContext {
if translation == nil {
translation = NewTranslation()
translation.ID = id
translationsForContext[id] = translation
}
}
}
po.domain.Headers = headers
po.domain = NewDomain()
po.domain.Headers = te.Headers
po.domain.Language = te.Language
po.domain.PluralForms = te.PluralForms
po.domain.nplurals = te.Nplurals
po.domain.plural = te.Plural
po.domain.translations = translations
po.domain.contextTranslations = contexts
if expr, err := plurals.Compile(te.Plural); err == nil {
po.domain.pluralforms = expr
}
po.Headers = po.domain.Headers
po.Language = po.domain.Language
po.PluralForms = po.domain.PluralForms
po.domain.translations = te.Translations
po.domain.contextTranslations = te.Contexts
return po
}
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.27@sha256:820153096fda4de6ac1b7194d12e416b7dd8f4e9a33d17e0345b111b945162d8
FROM golang:1.27@sha256:e432b43af23a9328d56a7c499be0476810aa344acbcf65fc7c455d4ff5a40602
ENV GOOS=linux
ENV GOARCH=arm
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.27@sha256:820153096fda4de6ac1b7194d12e416b7dd8f4e9a33d17e0345b111b945162d8
FROM golang:1.27@sha256:e432b43af23a9328d56a7c499be0476810aa344acbcf65fc7c455d4ff5a40602
ENV GOOS=linux
ENV GOARCH=arm64
+15
View File
@@ -0,0 +1,15 @@
*
!.git/
!build/root.tgz
!cmd/
!config/
!internal/
!mod/
!pkg/
!regclient/
!scheme/
!types/
!vendor/
!go.*
!*.go
!Makefile
+5
View File
@@ -0,0 +1,5 @@
artifacts/
bin/
output/
vendor/
.regctl_conf_ci.json
+19
View File
@@ -0,0 +1,19 @@
# all lists use a `-`
MD004:
style: dash
# allow tabs in code blocks (for Go)
MD010:
code_blocks: false
# disable line length, prefer one sentence per line for PRs
MD013: false
# emphasis with underscore (`_emphasis_`)
MD049:
style: "underscore"
# bold with asterisk (`**bold**`)
MD050:
style: "asterisk"
+1
View File
@@ -0,0 +1 @@
GoVersionOverride = "1.26.3"
+53
View File
@@ -0,0 +1,53 @@
{"name":"docker-arg-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
{"name":"docker-arg-alpine-tag","key":"docker.io/library/alpine","version":"3.23.4"}
{"name":"docker-arg-ecr","key":"https://github.com/awslabs/amazon-ecr-credential-helper.git","version":"v0.12.0"}
{"name":"docker-arg-gcr","key":"https://github.com/GoogleCloudPlatform/docker-credential-gcr.git","version":"v2.1.32"}
{"name":"docker-arg-go-digest","key":"docker.io/library/golang:1.26.3-alpine","version":"sha256:91eda9776261207ea25fd06b5b7fed8d397dd2c0a283e77f2ab6e91bfa71079d"}
{"name":"docker-arg-go-tag","key":"docker.io/library/golang","version":"1.26.3"}
{"name":"docker-arg-lunajson","key":"https://github.com/grafi-tt/lunajson.git:master","version":"e3a9666eb1275741e887e29926b144f8daee3bef"}
{"name":"docker-arg-semver","key":"https://github.com/kikito/semver.lua.git:master","version":"a4b708ba243208d46e575da870af969dca46a94d"}
{"name":"gha-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
{"name":"gha-alpine-tag-base","key":"docker.io/library/alpine","version":"3"}
{"name":"gha-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.23.4"}
{"name":"gha-cosign-version","key":"https://github.com/sigstore/cosign.git","version":"v3.0.6"}
{"name":"gha-golang-matrix","key":"golang-matrix","version":"[\"1.25\", \"1.26\"]"}
{"name":"gha-golang-release","key":"golang-latest","version":"1.26"}
{"name":"gha-syft-version","key":"docker.io/anchore/syft","version":"v1.44.0"}
{"name":"gha-uses-commit","key":"https://github.com/actions/checkout.git:v6.0.2","version":"de0fac2e4500dabe0009e67214ff5f5447ce83dd"}
{"name":"gha-uses-commit","key":"https://github.com/actions/setup-go.git:v6.4.0","version":"4a3601121dd01d1626a1e23e37211e3254c1c06c"}
{"name":"gha-uses-commit","key":"https://github.com/actions/stale.git:v10.3.0","version":"eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899"}
{"name":"gha-uses-commit","key":"https://github.com/actions/upload-artifact.git:v7.0.1","version":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"}
{"name":"gha-uses-commit","key":"https://github.com/anchore/sbom-action.git:v0.24.0","version":"e22c389904149dbc22b58101806040fa8d37a610"}
{"name":"gha-uses-commit","key":"https://github.com/docker/build-push-action.git:v7.2.0","version":"f9f3042f7e2789586610d6e8b85c8f03e5195baf"}
{"name":"gha-uses-commit","key":"https://github.com/docker/login-action.git:v4.2.0","version":"650006c6eb7dba73a995cc03b0b2d7f5ca915bee"}
{"name":"gha-uses-commit","key":"https://github.com/docker/setup-buildx-action.git:v4.1.0","version":"d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5"}
{"name":"gha-uses-commit","key":"https://github.com/regclient/actions.git:main","version":"c70ad64367908075211b10dcd2ab9fad4bfa1816"}
{"name":"gha-uses-commit","key":"https://github.com/sigstore/cosign-installer.git:v4.1.2","version":"6f9f17788090df1f26f669e9d70d6ae9567deba6"}
{"name":"gha-uses-commit","key":"https://github.com/softprops/action-gh-release.git:v3.0.0","version":"b4309332981a82ec1c5618f44dd2e27cc8bfbfda"}
{"name":"gha-uses-semver","key":"https://github.com/actions/checkout.git","version":"v6.0.2"}
{"name":"gha-uses-semver","key":"https://github.com/actions/setup-go.git","version":"v6.4.0"}
{"name":"gha-uses-semver","key":"https://github.com/actions/stale.git","version":"v10.3.0"}
{"name":"gha-uses-semver","key":"https://github.com/actions/upload-artifact.git","version":"v7.0.1"}
{"name":"gha-uses-semver","key":"https://github.com/anchore/sbom-action.git","version":"v0.24.0"}
{"name":"gha-uses-semver","key":"https://github.com/docker/build-push-action.git","version":"v7.2.0"}
{"name":"gha-uses-semver","key":"https://github.com/docker/login-action.git","version":"v4.2.0"}
{"name":"gha-uses-semver","key":"https://github.com/docker/setup-buildx-action.git","version":"v4.1.0"}
{"name":"gha-uses-semver","key":"https://github.com/sigstore/cosign-installer.git","version":"v4.1.2"}
{"name":"gha-uses-semver","key":"https://github.com/softprops/action-gh-release.git","version":"v3.0.0"}
{"name":"go-mod-golang-release","key":"golang-oldest","version":"1.25.0"}
{"name":"makefile-ci-distribution","key":"docker.io/library/registry","version":"3.1.1"}
{"name":"makefile-ci-zot","key":"ghcr.io/project-zot/zot-linux-amd64","version":"v2.1.17"}
{"name":"makefile-go-vulncheck","key":"https://go.googlesource.com/vuln.git","version":"v1.3.0"}
{"name":"makefile-gofumpt","key":"https://github.com/mvdan/gofumpt.git","version":"v0.10.0"}
{"name":"makefile-gomajor","key":"https://github.com/icholy/gomajor.git","version":"v0.15.0"}
{"name":"makefile-gosec","key":"https://github.com/securego/gosec.git","version":"v2.26.1"}
{"name":"makefile-markdown-lint","key":"docker.io/davidanson/markdownlint-cli2","version":"v0.22.1"}
{"name":"makefile-osv-scanner","key":"https://github.com/google/osv-scanner.git","version":"v2.3.8"}
{"name":"makefile-staticcheck","key":"https://github.com/dominikh/go-tools.git","version":"v0.7.0"}
{"name":"makefile-syft-container-digest","key":"anchore/syft:v1.44.0","version":"sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6"}
{"name":"makefile-syft-container-tag","key":"anchore/syft","version":"v1.44.0"}
{"name":"makefile-syft-version","key":"docker.io/anchore/syft","version":"v1.44.0"}
{"name":"osv-golang-release","key":"docker.io/library/golang","version":"1.26.3"}
{"name":"shell-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
{"name":"shell-alpine-tag-base","key":"docker.io/library/alpine","version":"3"}
{"name":"shell-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.23.4"}
+346
View File
@@ -0,0 +1,346 @@
files:
"build/Dockerfile*":
processors:
- docker-arg-alpine-tag
- docker-arg-alpine-digest
- docker-arg-go-tag
- docker-arg-go-digest
- docker-arg-ecr
- docker-arg-gcr
- docker-arg-lunajson
- docker-arg-semver
"build/oci-image.sh":
processors:
- shell-alpine-tag-base
- shell-alpine-tag-comment
- shell-alpine-digest
".github/workflows/*.yml":
processors:
- gha-golang-matrix
- gha-golang-release
- gha-uses-vx
- gha-uses-semver
- gha-uses-commit
- gha-syft-version
- gha-cosign-version
- gha-alpine-tag-base
- gha-alpine-tag-comment
- gha-alpine-digest
"Makefile":
processors:
- makefile-gofumpt
- makefile-gomajor
- makefile-go-vulncheck
- makefile-markdown-lint
- makefile-gosec
- makefile-osv-scanner
- makefile-staticcheck
- makefile-syft-version
- makefile-syft-container-tag
- makefile-syft-container-digest
- makefile-ci-distribution
- makefile-ci-zot
"go.mod":
processors:
- go-mod-golang-release
".osv-scanner.toml":
processors:
- osv-golang-release
x-processor-tmpl:
git-commit: &git-commit
key: "{{ .SourceArgs.url }}:{{ .SourceArgs.ref }}"
scan: "regexp"
source: "git-commit"
filter:
expr: "^{{ .SourceArgs.ref }}$"
git-tag-semver: &git-tag-semver
key: "{{ .SourceArgs.url }}"
scan: "regexp"
source: "git-tag"
filter:
expr: '^v?\d+\.\d+\.\d+$'
sort:
method: "semver"
registry-digest: &registry-digest
key: "{{ .SourceArgs.image }}"
scan: "regexp"
source: "registry-digest"
registry-tag-semver: &registry-tag-semver
key: "{{ .SourceArgs.repo }}"
scan: "regexp"
source: "registry-tag"
filter:
expr: '^v?\d+\.\d+\.\d+$'
sort:
method: "semver"
processors:
docker-arg-alpine-tag:
<<: *registry-tag-semver
scanArgs:
regexp: '^ARG ALPINE_VER=(?P<Version>v?\d+\.\d+\.\d+)@(?P<SHA>sha256:[0-9a-f]+)\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
docker-arg-alpine-digest:
<<: *registry-digest
scanArgs:
regexp: '^ARG ALPINE_VER=(?P<Tag>v?\d+\.\d+\.\d+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
sourceArgs:
image: "docker.io/library/alpine:{{.ScanMatch.Tag}}"
docker-arg-go-tag:
<<: *registry-tag-semver
scanArgs:
regexp: '^ARG GO_VER=(?P<Version>[a-z0-9\-\.]+)-alpine@(?P<SHA>sha256:[0-9a-f]+)\s*$'
sourceArgs:
repo: "docker.io/library/golang"
docker-arg-go-digest:
<<: *registry-digest
scanArgs:
regexp: '^ARG GO_VER=(?P<Tag>[a-z0-9\-\.]+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
sourceArgs:
image: "docker.io/library/golang:{{.ScanMatch.Tag}}"
docker-arg-ecr:
<<: *git-tag-semver
scanArgs:
regexp: '^ARG ECR_HELPER_VER=(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
url: "https://github.com/awslabs/amazon-ecr-credential-helper.git"
# get the version for the ecr-login nested package in the repo
filter:
expr: '^ecr-login/v?\d+\.\d+\.\d+$'
# sort and output only the version number without the package name prefix
sort:
method: "semver"
template: '{{ index (split . "/") 1 }}'
template: '{{ index (split .Version "/") 1 }}'
docker-arg-gcr:
<<: *git-tag-semver
scanArgs:
regexp: '^ARG GCR_HELPER_VER=(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
url: "https://github.com/GoogleCloudPlatform/docker-credential-gcr.git"
docker-arg-lunajson:
<<: *git-commit
scanArgs:
regexp: '^ARG LUNAJSON_COMMIT=(?P<Version>[0-9a-f]+)\s*$'
sourceArgs:
url: "https://github.com/grafi-tt/lunajson.git"
ref: master
docker-arg-semver:
<<: *git-commit
scanArgs:
regexp: '^ARG SEMVER_COMMIT=(?P<Version>[0-9a-f]+)\s*$'
sourceArgs:
url: "https://github.com/kikito/semver.lua.git"
ref: master
gha-alpine-digest:
<<: *registry-digest
scanArgs:
regexp: '^\s*ALPINE_DIGEST: "(?P<Version>sha256:[0-9a-f]+)"\s*#\s*(?P<Tag>\d+\.\d+\.\d+)\s*$'
sourceArgs:
image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}"
gha-alpine-tag-base:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_NAME: "alpine:(?P<Version>v?\d+)"\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
# only return the major version number in the tag to support detecting a change in the base image
template: '{{ index ( split .Version "." ) 0 }}'
gha-alpine-tag-comment:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_DIGEST: "(?P<Digest>sha256:[0-9a-f]+)"\s*#\s*(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
gha-cosign-version:
<<: *git-tag-semver
scanArgs:
regexp: '^\s*cosign-release: "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
url: "https://github.com/sigstore/cosign.git"
filter:
expr: '^v?3\.\d+\.\d+$' # pin to v3, v4 will remove support for older clients
gha-golang-matrix:
<<: *registry-tag-semver
key: "golang-matrix"
scanArgs:
regexp: '^\s*gover: (?P<Version>\[["0-9, \.]+\])\s*$'
sourceArgs:
repo: "docker.io/library/golang"
filter:
expr: '^v?\d+\.\d+$'
template: '["{{ index .VerMap ( index .VerList 1 ) }}", "{{ index .VerMap ( index .VerList 0 ) }}"]'
gha-golang-release:
<<: *registry-tag-semver
key: "golang-latest"
scanArgs:
regexp: '^\s*RELEASE_GO_VER: "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
repo: "docker.io/library/golang"
filter:
expr: '^v?\d+\.\d+$'
gha-syft-version:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*syft-version: "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
repo: "docker.io/anchore/syft"
gha-uses-vx:
<<: *git-tag-semver
scanArgs:
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Commit>[0-9a-f]+)\s+#\s+(?P<Version>v?\d+)\s*$'
sourceArgs:
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
filter:
expr: '^v?\d+$'
gha-uses-semver:
<<: *git-tag-semver
scanArgs:
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Commit>[0-9a-f]+)\s+#\s+(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
gha-uses-commit:
<<: *git-commit
scanArgs:
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Version>[0-9a-f]+)\s+#\s+(?P<Ref>[\w\d\.]+)\s*$'
sourceArgs:
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
ref: "{{ .ScanMatch.Ref }}"
go-mod-golang-release:
<<: *registry-tag-semver
key: "golang-oldest"
scanArgs:
regexp: '^go (?P<Version>[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/library/golang"
filter:
expr: '^\d+\.\d+$'
template: '{{ index .VerMap ( index .VerList 1 ) }}.0'
makefile-ci-distribution:
<<: *registry-tag-semver
scanArgs:
regexp: '^CI_DISTRIBUTION_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/library/registry"
makefile-ci-zot:
<<: *registry-tag-semver
scanArgs:
regexp: '^CI_ZOT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
repo: "ghcr.io/project-zot/zot-linux-amd64"
makefile-gofumpt:
<<: *git-tag-semver
scanArgs:
regexp: '^GOFUMPT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/mvdan/gofumpt.git"
makefile-gomajor:
<<: *git-tag-semver
scanArgs:
regexp: '^GOMAJOR_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/icholy/gomajor.git"
makefile-gosec:
<<: *git-tag-semver
scanArgs:
regexp: '^GOSEC_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/securego/gosec.git"
makefile-go-vulncheck:
<<: *git-tag-semver
scanArgs:
regexp: '^GO_VULNCHECK_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://go.googlesource.com/vuln.git"
makefile-markdown-lint:
<<: *registry-tag-semver
scanArgs:
regexp: '^MARKDOWN_LINT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/davidanson/markdownlint-cli2"
makefile-osv-scanner:
<<: *git-tag-semver
scanArgs:
regexp: '^OSV_SCANNER_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/google/osv-scanner.git"
makefile-staticcheck:
<<: *git-tag-semver
scanArgs:
regexp: '^STATICCHECK_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/dominikh/go-tools.git"
filter:
# repo also has dated tags, ignore versions without a preceding "v"
expr: '^v\d+\.\d+\.\d+$'
makefile-syft-container-tag:
<<: *registry-tag-semver
scanArgs:
regexp: '^SYFT_CONTAINER\?=(?P<Repo>[^:]*):(?P<Version>v?[0-9\.]+)@(?P<Digest>sha256:[0-9a-f]+)\s*$'
sourceArgs:
repo: "{{ .ScanMatch.Repo }}"
makefile-syft-container-digest:
<<: *registry-digest
scanArgs:
regexp: '^SYFT_CONTAINER\?=(?P<Image>[^:]*):(?P<Tag>v?[0-9\.]+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
sourceArgs:
image: "{{ .ScanMatch.Image }}:{{.ScanMatch.Tag}}"
makefile-syft-version:
<<: *registry-tag-semver
scanArgs:
regexp: '^SYFT_VERSION\?=(?P<Version>v[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/anchore/syft"
osv-golang-release:
<<: *registry-tag-semver
scanArgs:
regexp: '^GoVersionOverride = "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
repo: "docker.io/library/golang"
shell-alpine-tag-base:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_NAME="alpine:(?P<Version>v?\d+)"\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
# only return the major version number in the tag to support detecting a change in the base image
template: '{{ index ( split .Version "." ) 0 }}'
shell-alpine-tag-comment:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_DIGEST="(?P<Digest>sha256:[0-9a-f]+)"\s*#\s*(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
shell-alpine-digest:
<<: *registry-digest
scanArgs:
regexp: '^\s*ALPINE_DIGEST="(?P<Version>sha256:[0-9a-f]+)"\s*#\s*(?P<Tag>\d+\.\d+\.\d+)\s*$'
sourceArgs:
image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}"
scans:
regexp:
type: "regexp"
sources:
git-commit:
type: "git"
args:
type: "commit"
git-tag:
type: "git"
args:
type: "tag"
registry-digest:
type: "registry"
registry-tag:
type: "registry"
args:
type: "tag"
+134
View File
@@ -0,0 +1,134 @@
# Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
- Demonstrating empathy and kindness toward other people
- Being respectful of differing opinions, viewpoints, and experiences
- Giving and gracefully accepting constructive feedback
- Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
- Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior include:
- The use of sexualized language or imagery, and sexual attention or advances of
any kind
- Trolling, insulting or derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or email address,
without their explicit permission
- Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
<git@bmitch.net> or slack (I'm found on the CNCF, Docker, OCI, and OpenSSF
slacks).
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
[homepage]: https://www.contributor-covenant.org
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[Mozilla CoC]: https://github.com/mozilla/diversity
[FAQ]: https://www.contributor-covenant.org/faq
[translations]: https://www.contributor-covenant.org/translations
+80
View File
@@ -0,0 +1,80 @@
# Contributing
## Reporting security issues
Please see [SECURITY.md](security.md) for the process to report security issues.
## Reporting other issues
Please search for similar issues and if none are seen, report an issue at [github.com/regclient/regclient/issues](https://github.com/regclient/regclient/issues).
## Code style
This project attempts to follow these principles:
- Code is canonical Go, following styles and patterns commonly used by the Go community.
- Dependencies outside of the Go standard library should be minimized.
- Dependencies should be pinned to a specific digest and tracked by Go or version-check.
- Unit tests are strongly encouraged with a focus on test coverage of the successful path and common errors.
- Linters and other style formatting tools are used, please run `make all` before committing any changes.
## LLM Policy
This project expects all contributions to be developed by a human or created with a reproducible tool.
Developers using an AI/LLM tool to generate their contribution are expected to fully understand the entire contribution and the logic behind its design.
Contributions that appear to have been generated by an AI/LLM without a human review may result in a ban from future contributions to the project.
## Pull requests
PRs are welcome following the below guides:
- For anything beyond a minor fix, opening an issue is suggested to discuss possible solutions.
- Changes should be rebased on the main branch.
- Changes should be squashed to a single commit per logical change.
All changes must be signed (`git commit -s`) to indicate you agree to the [Developer Certificate or Origin](https://developercertificate.org/):
```text
Developer Certificate of Origin
Version 1.1
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
Everyone is permitted to copy and distribute verbatim copies of this
license document, but changing it is not allowed.
Developer's Certificate of Origin 1.1
By making a contribution to this project, I certify that:
(a) The contribution was created in whole or in part by me and I
have the right to submit it under the open source license
indicated in the file; or
(b) The contribution is based upon previous work that, to the best
of my knowledge, is covered under an appropriate open source
license and I have the right under that license to submit that
work with modifications, whether created in whole or in part
by me, under the same open source license (unless I am
permitted to submit under a different license), as indicated
in the file; or
(c) The contribution was provided directly to me by some other
person who certified (a), (b) or (c) and I have not modified
it.
(d) I understand and agree that this project and the contribution
are public and that a record of the contribution (including all
personal information I submit with it, including my sign-off) is
maintained indefinitely and may be redistributed consistent with
this project or the open source license(s) involved.
```
The sign-off will include the following message in your commit:
```text
Signed-off-by: Your Name <your-email@example.org>
```
This needs to be your real name, no aliases please.
+191
View File
@@ -0,0 +1,191 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
Copyright 2020 The regclient Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+285
View File
@@ -0,0 +1,285 @@
COMMANDS?=regctl regsync regbot
BINARIES?=$(addprefix bin/,$(COMMANDS))
IMAGES?=$(addprefix docker-,$(COMMANDS))
ARTIFACT_PLATFORMS?=linux-amd64 linux-arm64 linux-ppc64le linux-s390x linux-riscv64 darwin-amd64 darwin-arm64 windows-amd64.exe freebsd-amd64
ARTIFACTS?=$(foreach cmd,$(addprefix artifacts/,$(COMMANDS)),$(addprefix $(cmd)-,$(ARTIFACT_PLATFORMS)))
IMAGE_PLATFORMS?=linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/riscv64
VCS_REPO?="https://github.com/regclient/regclient.git"
VCS_REF?=$(shell git rev-list -1 HEAD)
ifneq ($(shell git status --porcelain 2>/dev/null),)
VCS_REF := $(VCS_REF)-dirty
endif
VCS_VERSION?=$(shell vcs_describe="$$(git describe --all)"; \
vcs_version="(devel)"; \
if [ "$${vcs_describe}" != "$${vcs_describe#tags/}" ]; then \
vcs_version="$${vcs_describe#tags/}"; \
elif [ "$${vcs_describe}" != "$${vcs_describe#heads/}" ]; then \
vcs_version="$${vcs_describe#heads/}"; \
if [ "main" = "$${vcs_version}" ]; then vcs_version=edge; fi; \
fi; \
echo "$${vcs_version}" | sed -r 's#/+#-#g')
VCS_TAG?=$(shell git describe --tags --abbrev=0 2>/dev/null || true)
VCS_SEC?=$(shell git log -1 --format=%ct)
VCS_DATE?=$(shell date -d "@$(VCS_SEC)" +%Y-%m-%dT%H:%M:%SZ --utc)
LD_FLAGS?=-s -w -extldflags -static -buildid= -X \"github.com/regclient/regclient/internal/version.vcsTag=$(VCS_TAG)\"
GO_BUILD_FLAGS?=-trimpath -ldflags "$(LD_FLAGS)"
DOCKERFILE_EXT?=$(shell if docker build --help 2>/dev/null | grep -q -- '--progress'; then echo ".buildkit"; fi)
DOCKER_ARGS?=--build-arg "VCS_REF=$(VCS_REF)" --build-arg "VCS_VERSION=$(VCS_VERSION)" --build-arg "SOURCE_DATE_EPOCH=$(VCS_SEC)" --build-arg "BUILD_DATE=$(VCS_DATE)"
GOPATH?=$(shell go env GOPATH)
PWD:=$(shell pwd)
VER_BUMP?=$(shell command -v version-bump 2>/dev/null)
VER_BUMP_CONTAINER?=sudobmitch/version-bump:edge
ifeq "$(strip $(VER_BUMP))" ''
VER_BUMP=docker run --rm \
-v "$(shell pwd)/:$(shell pwd)/" -w "$(shell pwd)" \
-u "$(shell id -u):$(shell id -g)" \
$(VER_BUMP_CONTAINER)
endif
MARKDOWN_LINT_VER?=v0.22.1
GOFUMPT_VER?=v0.10.0
GOMAJOR_VER?=v0.15.0
GOSEC_VER?=v2.26.1
GO_VULNCHECK_VER?=v1.3.0
OSV_SCANNER_VER?=v2.3.8
SYFT?=$(shell command -v syft 2>/dev/null)
SYFT_CMD_VER:=$(shell [ -x "$(SYFT)" ] && echo "v$$($(SYFT) version | awk '/^Version: / {print $$2}')" || echo "0")
SYFT_VERSION?=v1.44.0
SYFT_CONTAINER?=anchore/syft:v1.44.0@sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6
ifneq "$(SYFT_CMD_VER)" "$(SYFT_VERSION)"
SYFT=docker run --rm \
-v "$(shell pwd)/:$(shell pwd)/" -w "$(shell pwd)" \
-u "$(shell id -u):$(shell id -g)" \
$(SYFT_CONTAINER)
endif
STATICCHECK_VER?=v0.7.0
CI_DISTRIBUTION_VER?=3.1.1
CI_ZOT_VER?=v2.1.17
.PHONY: .FORCE
.FORCE:
.PHONY: all
all: fmt gofumpt gofix goimports vet test lint binaries ## Full build of Go binaries (including fmt, vet, test, and lint)
.PHONY: fmt
fmt: ## go fmt
go fmt ./...
.PHONY: gofumpt
gofumpt: $(GOPATH)/bin/gofumpt ## gofumpt is a stricter alternative to go fmt
gofumpt -l -w .
.PHONY: gofix
gofix: ## go fix
go fix ./...
goimports: $(GOPATH)/bin/goimports
$(GOPATH)/bin/goimports -w -format-only -local github.com/regclient .
.PHONY: vet
vet: ## go vet
go vet ./...
.PHONY: test
test: ## go test
go test -cover -race ./...
.PHONY: lint
lint: lint-go lint-goimports lint-md lint-gosec ## Run all linting
.PHONY: lint-go
lint-go: $(GOPATH)/bin/gofumpt $(GOPATH)/bin/staticcheck .FORCE ## Run linting for Go
$(GOPATH)/bin/staticcheck -checks all ./...
$(GOPATH)/bin/gofumpt -l -d .
errors=$$(go fix -diff ./...); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi
lint-goimports: $(GOPATH)/bin/goimports
@if [ -n "$$($(GOPATH)/bin/goimports -l -format-only -local github.com/regclient .)" ]; then \
echo $(GOPATH)/bin/goimports -d -format-only -local github.com/regclient .; \
$(GOPATH)/bin/goimports -d -format-only -local github.com/regclient .; \
exit 1; \
fi
# excluding types/platform pending resultion to https://github.com/securego/gosec/issues/1116
.PHONY: lint-gosec
lint-gosec: $(GOPATH)/bin/gosec .FORCE ## Run gosec
$(GOPATH)/bin/gosec -terse -exclude-dir types/platform ./...
.PHONY: lint-md
lint-md: .FORCE ## Run linting for markdown
docker run --rm -v "$(PWD):/workdir:ro" davidanson/markdownlint-cli2:$(MARKDOWN_LINT_VER) \
"**/*.md" "#vendor"
.PHONY: vulnerability-scan
vulnerability-scan: osv-scanner vulncheck-go ## Run all vulnerability scanners
.PHONY: osv-scanner
osv-scanner: $(GOPATH)/bin/osv-scanner .FORCE ## Run OSV Scanner
$(GOPATH)/bin/osv-scanner scan --config .osv-scanner.toml -r --licenses="Apache-2.0,BSD-3-Clause,MIT,CC-BY-SA-4.0,UNKNOWN" .
.PHONY: vulncheck-go
vulncheck-go: $(GOPATH)/bin/govulncheck .FORCE ## Run govulncheck
$(GOPATH)/bin/govulncheck ./...
.PHONY: vendor
vendor: ## Vendor Go modules
go mod vendor
.PHONY: binaries
binaries: $(BINARIES) ## Build Go binaries
bin/%: .FORCE
CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o bin/$* ./cmd/$*
.PHONY: docker
docker: $(IMAGES) ## Build Docker images
docker-%: .FORCE
docker build -t regclient/$* -f build/Dockerfile.$*$(DOCKERFILE_EXT) $(DOCKER_ARGS) .
docker build -t regclient/$*:alpine -f build/Dockerfile.$*$(DOCKERFILE_EXT) --target release-alpine $(DOCKER_ARGS) .
.PHONY: oci-image
oci-image: $(addprefix oci-image-,$(COMMANDS)) ## Build reproducible images to an OCI Layout
oci-image-%: bin/regctl .FORCE
PATH="$(PWD)/bin:$(PATH)" build/oci-image.sh -r scratch -i "$*" -p "$(IMAGE_PLATFORMS)"
PATH="$(PWD)/bin:$(PATH)" build/oci-image.sh -r alpine -i "$*" -p "$(IMAGE_PLATFORMS)" -b "alpine:3"
.PHONY: test-docker
test-docker: $(addprefix test-docker-,$(COMMANDS)) ## Build multi-platform docker images (but do not tag)
test-docker-%:
docker buildx build --platform="$(IMAGE_PLATFORMS)" -f build/Dockerfile.$*.buildkit .
docker buildx build --platform="$(IMAGE_PLATFORMS)" -f build/Dockerfile.$*.buildkit --target release-alpine .
.PHONY: ci
ci: ci-distribution ci-zot ## Run CI tests against self hosted registries
.PHONY: ci-distribution
ci-distribution:
docker run --rm -d -p 5000 \
--label regclient-ci=true --name regclient-ci-distribution \
-e "REGISTRY_STORAGE_DELETE_ENABLED=true" \
docker.io/library/registry:$(CI_DISTRIBUTION_VER)
./build/ci-test.sh -t localhost:$$(docker port regclient-ci-distribution 5000 | head -1 | cut -f2 -d:)/test-ci
docker stop regclient-ci-distribution
.PHONY: ci-zot
ci-zot:
docker run --rm -d -p 5000 \
--label regclient-ci=true --name regclient-ci-zot \
-v "$$(pwd)/build/zot-config.json:/etc/zot/config.json:ro" \
ghcr.io/project-zot/zot-linux-amd64:$(CI_ZOT_VER)
./build/ci-test.sh -t localhost:$$(docker port regclient-ci-zot 5000 | head -1 | cut -f2 -d:)/test-ci
docker stop regclient-ci-zot
.PHONY: artifacts
artifacts: $(ARTIFACTS) ## Generate artifacts
.PHONY: artifact-pre
artifact-pre:
mkdir -p artifacts
artifacts/%: artifact-pre .FORCE
@set -e; \
target="$*"; \
command="$${target%%-*}"; \
platform_ext="$${target#*-}"; \
platform="$${platform_ext%.*}"; \
export GOOS="$${platform%%-*}"; \
export GOARCH="$${platform#*-}"; \
echo export GOOS=$${GOOS}; \
echo export GOARCH=$${GOARCH}; \
echo go build ${GO_BUILD_FLAGS} -o "$@" ./cmd/$${command}/; \
CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o "$@" ./cmd/$${command}/; \
$(SYFT) scan -q "file:$@" --source-name "$${command}" -o cyclonedx-json >"artifacts/$${command}-$${platform}.cyclonedx.json"; \
$(SYFT) scan -q "file:$@" --source-name "$${command}" -o spdx-json >"artifacts/$${command}-$${platform}.spdx.json"
.PHONY: clean
clean: ## delete generated content
[ ! -d artifacts ] || rm -r artifacts
[ ! -d bin ] || rm -r bin
[ ! -d output ] || rm -r output
[ ! -d vendor ] || rm -r vendor
.PHONY: plugin-user
plugin-user:
mkdir -p ${HOME}/.docker/cli-plugins/
cp docker-plugin/docker-regclient ${HOME}/.docker/cli-plugins/docker-regctl
.PHONY: plugin-host
plugin-host:
sudo cp docker-plugin/docker-regclient /usr/libexec/docker/cli-plugins/docker-regctl
.PHONY: util-golang-major
util-golang-major: $(GOPATH)/bin/gomajor ## check for major dependency updates
$(GOPATH)/bin/gomajor list
.PHONY: util-golang-update
util-golang-update: ## update go module versions
go get -u -t ./...
go mod tidy
[ ! -d vendor ] || go mod vendor
.PHONY: util-release-preview
util-release-preview: $(GOPATH)/bin/gorelease ## preview changes for next release
git checkout main
./.github/release.sh -d
gorelease
.PHONY: util-release-run
util-release-run: ## generate a new release
git checkout main
./.github/release.sh
.PHONY: util-version-check
util-version-check: ## check all dependencies for updates
$(VER_BUMP) check
.PHONY: util-version-update
util-version-update: ## update versions on all dependencies
$(VER_BUMP) update
$(GOPATH)/bin/gofumpt: .FORCE
@[ -f "$(GOPATH)/bin/gofumpt" ] \
&& [ "$$($(GOPATH)/bin/gofumpt -version | cut -f 1 -d ' ')" = "$(GOFUMPT_VER)" ] \
|| go install mvdan.cc/gofumpt@$(GOFUMPT_VER)
$(GOPATH)/bin/gomajor: .FORCE
@[ -f "$(GOPATH)/bin/gomajor" ] \
&& [ "$$($(GOPATH)/bin/gomajor version | grep '^version' | cut -f 2 -d ' ')" = "$(GOMAJOR_VER)" ] \
|| go install github.com/icholy/gomajor@$(GOMAJOR_VER)
$(GOPATH)/bin/goimports: .FORCE
@[ -f "$(GOPATH)/bin/goimports" ] && [ "$$(go version | cut -f3 -d' ')" = "$$(go version $(GOPATH)/bin/goimports | cut -f2 -d' ')" ] \
|| go install golang.org/x/tools/cmd/goimports@latest
$(GOPATH)/bin/gorelease: .FORCE
@[ -f "$(GOPATH)/bin/gorelease" ] && [ "$$(go version | cut -f3 -d' ')" = "$$(go version $(GOPATH)/bin/gorelease | cut -f2 -d' ')" ] \
|| go install golang.org/x/exp/cmd/gorelease@latest
$(GOPATH)/bin/gosec: .FORCE
@[ -f $(GOPATH)/bin/gosec ] \
&& [ "$$($(GOPATH)/bin/gosec -version | grep '^Version' | cut -f 2 -d ' ')" = "$(GOSEC_VER)" ] \
|| go install -ldflags '-X main.Version=$(GOSEC_VER) -X main.GitTag=$(GOSEC_VER)' \
github.com/securego/gosec/v2/cmd/gosec@$(GOSEC_VER)
$(GOPATH)/bin/staticcheck: .FORCE
@[ -f $(GOPATH)/bin/staticcheck ] \
&& [ "$$($(GOPATH)/bin/staticcheck -version | cut -f 3 -d ' ' | tr -d '()')" = "$(STATICCHECK_VER)" ] \
|| go install "honnef.co/go/tools/cmd/staticcheck@$(STATICCHECK_VER)"
$(GOPATH)/bin/govulncheck: .FORCE
@[ -f $(GOPATH)/bin/govulncheck ] \
&& [ $$(go version -m $(GOPATH)/bin/govulncheck | \
awk -F ' ' '{ if ($$1 == "mod" && $$2 == "golang.org/x/vuln") { printf "%s\n", $$3 } }') = "$(GO_VULNCHECK_VER)" ] \
|| CGO_ENABLED=0 go install "golang.org/x/vuln/cmd/govulncheck@$(GO_VULNCHECK_VER)"
$(GOPATH)/bin/osv-scanner: .FORCE
@[ -f $(GOPATH)/bin/osv-scanner ] \
&& [ "$$(osv-scanner --version | awk -F ': ' '{ if ($$1 == "osv-scanner version") { printf "%s\n", $$2 } }')" = "$(OSV_SCANNER_VER)" ] \
|| CGO_ENABLED=0 go install "github.com/google/osv-scanner/v2/cmd/osv-scanner@$(OSV_SCANNER_VER)"
.PHONY: help
help: # Display help
@awk -F ':|##' '/^[^\t].+?:.*?##/ { printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF }' $(MAKEFILE_LIST)
+116
View File
@@ -0,0 +1,116 @@
# regclient
[![Go Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/go.yml?branch=main&label=Go%20build)](https://github.com/regclient/regclient/actions/workflows/go.yml)
[![Docker Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/docker.yml?branch=main&label=Docker%20build)](https://github.com/regclient/regclient/actions/workflows/docker.yml)
[![Dependency Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/version-check.yml?branch=main&label=Dependency%20check)](https://github.com/regclient/regclient/actions/workflows/version-check.yml)
[![Vulnerability Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/vulnscans.yml?branch=main&label=Vulnerability%20check)](https://github.com/regclient/regclient/actions/workflows/vulnscans.yml)
[![Go Reference](https://pkg.go.dev/badge/github.com/regclient/regclient.svg)](https://pkg.go.dev/github.com/regclient/regclient)
![License](https://img.shields.io/github/license/regclient/regclient)
[![Go Report Card](https://goreportcard.com/badge/github.com/regclient/regclient)](https://goreportcard.com/report/github.com/regclient/regclient)
[![GitHub Downloads](https://img.shields.io/github/downloads/regclient/regclient/total?label=GitHub%20downloads)](https://github.com/regclient/regclient/releases)
regclient is a client interface to OCI conformant registries and content shipped with the OCI Image Layout.
It includes a Go library and several CLI commands.
## regclient Go Library Features
- Runs without a container runtime and without privileged access to the local host.
- Querying for a tag listing, repository listing, and remotely inspecting the contents of images.
- Efficiently copying and retagging images, only pulling layers when required, and without changing the image digest.
- Support for multi-platform images.
- Support for querying, creating, and copying OCI Artifacts, allowing arbitrary data to be stored in an OCI registry.
- Support for packaging OCI Artifacts with an Index of multiple artifacts, which can be used for platform specific artifacts.
- Support for querying OCI referrers, copying referrers, and pushing content with an OCI subject field, associating artifacts with other content on the registry.
- Support for the “digest tags” used by projects like sigstore/cosign, allowing the content to be included when copying images.
- Efficiently query for an image digest.
- Efficiently query for pull rate limits used by Docker Hub.
- Import and export content into OCI Layouts and Docker formatted tar files.
- Support OCI Layouts in all commands as a local disk equivalent of a repository.
- Support for deleting tags, manifests, and blobs.
- Ability to mutate existing images, including:
- Settings annotations or labels
- Deleting content from layers
- Changing timestamps for reproducibility
- Converting between Docker and OCI media types
- Replacing the base image layers
- Add or remove volumes and exposed ports
- Change digest algorithms
- Support for registry warning headers, which may be used to notify users of issues with the server or content they are using.
- Automatically import logins from the docker CLI, and registry certificates from the docker engine.
- Automatic retry, and fallback to a chunked blob push, when network issues are encountered.
The full Go references is available on [pkg.go.dev](https://pkg.go.dev/github.com/regclient/regclient).
## regctl Features
`regctl` is a CLI interface to the `regclient` library.
In addition to the features listed for `regclient`, `regctl` adds the following abilities:
- Generating multi-platform manifests from multiple images that may have been separately built.
- Repackage a multi-platform image with only the requested platforms.
- Push and pull arbitrary OCI artifacts.
- Recursively list all content associated with an image.
- Extract files from a layer or image.
- Compare images, showing the differences between manifests, the config, and layers.
- Formatted output using Go templates.
The project website includes [usage instructions](https://regclient.org/usage/regctl/) and a [CLI reference](https://regclient.org/cli/regctl/).
## regsync features
`regsync` is an image mirroring tool.
It will copy images between two locations with the following additional features:
- Ability to run on a cron schedule, one time synchronization, or only report stale images.
- Uses a yaml configuration.
- Each source may be an entire registry (not recommended), a repository, or a single image, with the ability to filter repositories and tags.
- Support for multi-platform images, OCI referrers, “digest tags”, and copying to or from an OCI Layout (for maintaining a mirror over an air-gap).
- Ability to mirror multiple images concurrently.
- Support for copying a single platform from multi-platform images.
- Ability to backup an existing image before overwriting the tag.
- Ability to postpone mirror step when rate limit (used by Docker Hub) is below a threshold.
- Can use user’s docker configuration for user credentials and registry certificates.
The project website includes [usage instructions](https://regclient.org/usage/regsync/) and a [CLI reference](https://regclient.org/cli/regsync/).
## regbot features
`regbot` is a scripting tool on top of the `regclient` API with the following features:
- Ability to run on a cron schedule, one time execution, or test with a dry-run mode.
- Uses a yaml configuration.
- Scripts are written in Lua and executed directly in Go.
- Built-in functions include:
- Repository list
- Tag list
- Image manifest (either head or get, and optional resolving multi-platform reference)
- Image config (this includes the creation time, labels, and other details shown in a docker image inspect)
- Image rate limit and a wait function to delay the script when rate limit remaining is below a threshold
- Image copy
- Manifest delete
- Tag delete
The project website includes [usage instructions](https://regclient.org/usage/regbot/) and a [CLI reference](https://regclient.org/cli/regbot/).
## Development Status
This project is using v0 version numbers due to Go's backwards compatibility requirements of a v1 release.
The library and commands are stable for external use.
Minor version updates may contain breaking changes, however effort is made to first deprecate and provide warnings to give users time to move off of older APIs and commands.
## Installing
See the [installation instructions](https://regclient.org/install/) on the project website for the various ways to download or build CLI binaries.
## Usage
See the [project documentation](https://regclient.org/usage/).
## Contributors
<a href="https://github.com/regclient/regclient/graphs/contributors">
<img src="https://contrib.rocks/image?repo=regclient/regclient" alt="contributor list"/>
</a>
<!-- markdownlint-disable-file MD033 -->
+5
View File
@@ -0,0 +1,5 @@
# Reporting security issues
Please report security issues directly in GitHub at <https://github.com/regclient/regclient/security/advisories/new> or alternatively email <git@bmitch.net>.
We will typically respond within 7 working days of your report. If the issue is confirmed as a vulnerability, we will open a Security Advisory and acknowledge your contributions as part of it. This project follows a 90 day disclosure timeline.
+283
View File
@@ -0,0 +1,283 @@
package regclient
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"log/slog"
"time"
"github.com/regclient/regclient/internal/pqueue"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types"
"github.com/regclient/regclient/types/blob"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/warning"
)
const blobCBFreq = time.Millisecond * 100
type blobOpt struct {
callback func(kind types.CallbackKind, instance string, state types.CallbackState, cur, total int64)
readerHook func(*blob.BReader) (*blob.BReader, error)
}
// BlobOpts define options for the Image* commands.
type BlobOpts func(*blobOpt)
// BlobWithCallback provides progress data to a callback function.
func BlobWithCallback(callback func(kind types.CallbackKind, instance string, state types.CallbackState, cur, total int64)) BlobOpts {
return func(opts *blobOpt) {
opts.callback = callback
}
}
// BlobWithReaderHook is called in [RegClient.BlobCopy] with the blob source.
// The returned [blob.BReader] is pushed to the target.
// If the hook returns an error, the copy will fail.
func BlobWithReaderHook(hook func(*blob.BReader) (*blob.BReader, error)) BlobOpts {
return func(opts *blobOpt) {
opts.readerHook = hook
}
}
// BlobCopy copies a blob between two locations.
// If the blob already exists in the target, the copy is skipped.
// A server side cross repository blob mount is attempted.
func (rc *RegClient) BlobCopy(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor, opts ...BlobOpts) error {
if !refSrc.IsSetRepo() {
return fmt.Errorf("refSrc is not set: %s%.0w", refSrc.CommonName(), errs.ErrInvalidReference)
}
if !refTgt.IsSetRepo() {
return fmt.Errorf("refTgt is not set: %s%.0w", refTgt.CommonName(), errs.ErrInvalidReference)
}
var opt blobOpt
for _, optFn := range opts {
optFn(&opt)
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
tDesc := d
tDesc.URLs = []string{} // ignore URLs when pushing to target
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackStarted, 0, d.Size)
}
// for the same repository, there's nothing to copy
if ref.EqualRepository(refSrc, refTgt) {
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
}
rc.slog.Debug("Blob copy skipped, same repo",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("digest", string(d.Digest)))
return nil
}
// check if layer already exists
if _, err := rc.BlobHead(ctx, refTgt, tDesc); err == nil {
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
}
rc.slog.Debug("Blob copy skipped, already exists",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("digest", string(d.Digest)))
return nil
}
// acquire throttle for both src and tgt to avoid deadlocks
tList := []*pqueue.Queue[reqmeta.Data]{}
schemeSrcAPI, err := rc.schemeGet(refSrc.Scheme)
if err != nil {
return err
}
schemeTgtAPI, err := rc.schemeGet(refTgt.Scheme)
if err != nil {
return err
}
if tSrc, ok := schemeSrcAPI.(scheme.Throttler); ok {
tList = append(tList, tSrc.Throttle(refSrc, false)...)
}
if tTgt, ok := schemeTgtAPI.(scheme.Throttler); ok {
tList = append(tList, tTgt.Throttle(refTgt, true)...)
}
if len(tList) > 0 {
ctxMulti, done, err := pqueue.AcquireMulti[reqmeta.Data](ctx, reqmeta.Data{Kind: reqmeta.Blob, Size: d.Size}, tList...)
if err != nil {
return err
}
if done != nil {
defer done()
}
ctx = ctxMulti
}
// try mounting blob from the source repo is the registry is the same
if ref.EqualRegistry(refSrc, refTgt) {
err := rc.BlobMount(ctx, refSrc, refTgt, d)
if err == nil {
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
}
rc.slog.Debug("Blob copy performed server side with registry mount",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("digest", string(d.Digest)))
return nil
}
rc.slog.Warn("Failed to mount blob",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("err", err.Error()))
}
// fast options failed, download layer from source and push to target
blobIO, err := rc.BlobGet(ctx, refSrc, d)
if err != nil {
if !errors.Is(err, context.Canceled) {
rc.slog.Warn("Failed to retrieve blob",
slog.String("src", refSrc.Reference),
slog.String("digest", string(d.Digest)),
slog.String("err", err.Error()))
}
return err
}
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackStarted, 0, d.Size)
ticker := time.NewTicker(blobCBFreq)
done := make(chan bool)
defer func() {
close(done)
ticker.Stop()
if ctx.Err() == nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackFinished, d.Size, d.Size)
}
}()
go func() {
for {
select {
case <-done:
return
case <-ticker.C:
offset, err := blobIO.Seek(0, io.SeekCurrent)
if err == nil && offset > 0 {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackActive, offset, d.Size)
}
}
}
}()
}
if opt.readerHook != nil {
blobIO, err = opt.readerHook(blobIO)
if err != nil {
rc.slog.Warn("Failed to apply reader hook to blob",
slog.String("src", refSrc.Reference),
slog.String("err", err.Error()))
return err
}
}
defer blobIO.Close()
if _, err := rc.BlobPut(ctx, refTgt, blobIO.GetDescriptor(), blobIO); err != nil {
if !errors.Is(err, context.Canceled) {
rc.slog.Warn("Failed to push blob",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("err", err.Error()))
}
return err
}
return nil
}
// BlobDelete removes a blob from the registry.
// This method should only be used to repair a damaged registry.
// Typically a server side garbage collection should be used to purge unused blobs.
func (rc *RegClient) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error {
if !r.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return err
}
return schemeAPI.BlobDelete(ctx, r, d)
}
// BlobGet retrieves a blob, returning a reader.
// This reader must be closed to free up resources that limit concurrent pulls.
func (rc *RegClient) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
data, err := d.GetData()
if err == nil {
return blob.NewReader(blob.WithDesc(d), blob.WithRef(r), blob.WithReader(bytes.NewReader(data))), nil
}
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
return schemeAPI.BlobGet(ctx, r, d)
}
// BlobGetOCIConfig retrieves an OCI config from a blob, automatically extracting the JSON.
func (rc *RegClient) BlobGetOCIConfig(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.OCIConfig, error) {
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
b, err := rc.BlobGet(ctx, r, d)
if err != nil {
return nil, err
}
return b.ToOCIConfig()
}
// BlobHead is used to verify if a blob exists and is accessible.
func (rc *RegClient) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
return schemeAPI.BlobHead(ctx, r, d)
}
// BlobMount attempts to perform a server side copy/mount of the blob between repositories.
func (rc *RegClient) BlobMount(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor) error {
if !refSrc.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", refSrc.CommonName(), errs.ErrInvalidReference)
}
if !refTgt.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", refTgt.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(refSrc.Scheme)
if err != nil {
return err
}
return schemeAPI.BlobMount(ctx, refSrc, refTgt, d)
}
// BlobPut uploads a blob to a repository.
// Descriptor is optional, leave size and digest to zero value if unknown.
// Reader must also be an [io.Seeker] to support chunked upload fallback.
//
// This will attempt an anonymous blob mount first which some registries may support.
// It will then try doing a full put of the blob without chunking (most widely supported).
// If the full put fails, it will fall back to a chunked upload (useful for flaky networks).
func (rc *RegClient) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error) {
if !r.IsSetRepo() {
return descriptor.Descriptor{}, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return descriptor.Descriptor{}, err
}
return schemeAPI.BlobPut(ctx, r, d, rdr)
}
+100
View File
@@ -0,0 +1,100 @@
package config
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"strings"
)
// credHelper wraps a command that manages user credentials.
type credHelper struct {
prog string
env map[string]string
}
func newCredHelper(prog string, env map[string]string) *credHelper {
return &credHelper{prog: prog, env: env}
}
func (ch *credHelper) run(arg string, input io.Reader) ([]byte, error) {
//#nosec G204 only untrusted arg is a hostname which the executed command should not trust
cmd := exec.Command(ch.prog, arg)
cmd.Env = os.Environ()
if ch.env != nil {
for k, v := range ch.env {
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
}
}
cmd.Stderr = os.Stderr
cmd.Stdin = input
return cmd.Output()
}
type credStore struct {
ServerURL string `json:"ServerURL"`
Username string `json:"Username"`
Secret string `json:"Secret"` //#nosec G117 exported struct intentionally holds secrets
}
// get requests a credential from the helper for a given host.
func (ch *credHelper) get(host *Host) error {
hostname := host.Hostname
if host.CredHost != "" {
hostname = host.CredHost
}
hostIn := strings.NewReader(hostname)
credOut := credStore{
Username: host.User,
Secret: host.Pass,
}
outB, err := ch.run("get", hostIn)
if err != nil {
outS := strings.TrimSpace(string(outB))
return fmt.Errorf("error getting credentials, output: %s, error: %w", outS, err)
}
err = json.NewDecoder(bytes.NewReader(outB)).Decode(&credOut)
if err != nil {
return fmt.Errorf("error reading credentials: %w", err)
}
if credOut.Username == tokenUser {
host.User = ""
host.Pass = ""
host.Token = credOut.Secret
} else {
host.User = credOut.Username
host.Pass = credOut.Secret
host.Token = ""
}
return nil
}
// list returns a list of hosts supported by the credential helper.
func (ch *credHelper) list() ([]Host, error) {
credList := map[string]string{}
outB, err := ch.run("list", bytes.NewReader([]byte{}))
if err != nil {
outS := strings.TrimSpace(string(outB))
return nil, fmt.Errorf("error getting credential list, output: %s, error: %w", outS, err)
}
err = json.NewDecoder(bytes.NewReader(outB)).Decode(&credList)
if err != nil {
return nil, fmt.Errorf("error reading credential list: %w", err)
}
hostList := []Host{}
for host, user := range credList {
if !HostValidate(host) {
continue
}
h := HostNewName(host)
h.User = user
h.CredHelper = ch.prog
hostList = append(hostList, *h)
}
return hostList, nil
}
// TODO: store method not implemented
+210
View File
@@ -0,0 +1,210 @@
package config
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"strings"
"github.com/regclient/regclient/internal/conffile"
"github.com/regclient/regclient/types/errs"
)
const (
// dockerEnv is the environment variable used to look for Docker's config.json.
dockerEnv = "DOCKER_CONFIG"
// dockerEnvConfig is used to inject the config as an environment variable.
dockerEnvConfig = "DOCKER_AUTH_CONFIG"
// dockerDir is the directory name for Docker's config (inside the users home directory).
dockerDir = ".docker"
// dockerConfFile is the name of Docker's config file.
dockerConfFile = "config.json"
// dockerHelperPre is the prefix of docker credential helpers.
dockerHelperPre = "docker-credential-"
)
// dockerConfig is used to parse the ~/.docker/config.json
type dockerConfig struct {
AuthConfigs map[string]dockerAuthConfig `json:"auths"`
HTTPHeaders map[string]string `json:"HttpHeaders,omitempty"`
DetachKeys string `json:"detachKeys,omitempty"`
CredentialsStore string `json:"credsStore,omitempty"`
CredentialHelpers map[string]string `json:"credHelpers,omitempty"`
Proxies map[string]dockerProxyConfig `json:"proxies,omitempty"`
}
// dockerProxyConfig contains proxy configuration settings
type dockerProxyConfig struct {
HTTPProxy string `json:"httpProxy,omitempty"`
HTTPSProxy string `json:"httpsProxy,omitempty"`
NoProxy string `json:"noProxy,omitempty"`
FTPProxy string `json:"ftpProxy,omitempty"`
AllProxy string `json:"allProxy,omitempty"`
}
// dockerAuthConfig contains the auths
type dockerAuthConfig struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"` //#nosec G117 exported struct intentionally holds secrets
Auth string `json:"auth,omitempty"`
ServerAddress string `json:"serveraddress,omitempty"`
// IdentityToken is used to authenticate the user and get
// an access token for the registry.
IdentityToken string `json:"identitytoken,omitempty"`
// RegistryToken is a bearer token to be sent to a registry
RegistryToken string `json:"registrytoken,omitempty"`
}
// DockerLoad returns a slice of hosts from the users docker config.
// This will search for the config.json in either the DOCKER_CONFIG identified directory or the default .docker directory.
// It also includes hosts extracted from the DOCKER_AUTH_CONFIG variable.
// If the config file is missing and no value is injected using an environment variable, an empty list is returned.
func DockerLoad() ([]Host, error) {
hosts := []Host{}
errList := []error{}
// load from a file
cf := conffile.New(
conffile.WithHomeDir(dockerDir, dockerConfFile, true),
conffile.WithEnvDir(dockerEnv, dockerConfFile),
)
rdr, err := cf.Open()
if err != nil && !errors.Is(err, fs.ErrNotExist) {
errList = append(errList, err)
} else if err == nil {
defer rdr.Close()
hostsFile, err := dockerParse(rdr)
if err != nil {
errList = append(errList, err)
} else {
hosts = append(hosts, hostsFile...)
}
}
// load from an env var
hostsEnv, err := DockerLoadEnv(dockerEnvConfig)
if err != nil && !errors.Is(err, errs.ErrNotFound) {
errList = append(errList, err)
} else if err == nil {
hosts = append(hosts, hostsEnv...)
}
// return the concatenated result, only wrapping an error list if necessary
if len(errList) == 1 {
return hosts, errList[0]
} else {
return hosts, errors.Join(errList...)
}
}
// DockerLoadFile returns a slice of hosts from a named docker config file.
func DockerLoadFile(fname string) ([]Host, error) {
//#nosec G304 scoping file operations to a directory is not yet a feature of regclient.
rdr, err := os.Open(fname)
if err != nil && errors.Is(err, fs.ErrNotExist) {
return []Host{}, nil
} else if err != nil {
return nil, err
}
defer rdr.Close()
return dockerParse(rdr)
}
// DockerLoadEnv returns a slice of hosts extracted from the config injected in an environment variable.
func DockerLoadEnv(envName string) ([]Host, error) {
envVal := os.Getenv(envName)
if envVal == "" {
return []Host{}, errs.ErrNotFound
}
return dockerParse(strings.NewReader(envVal))
}
// dockerParse parses a docker config into a slice of Hosts.
func dockerParse(rdr io.Reader) ([]Host, error) {
dc := dockerConfig{}
if err := json.NewDecoder(rdr).Decode(&dc); err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
hosts := []Host{}
for name, auth := range dc.AuthConfigs {
if !HostValidate(name) {
continue
}
h, err := dockerAuthToHost(name, dc, auth)
if err != nil {
continue
}
hosts = append(hosts, h)
}
// also include default entries for credential helpers
for name, helper := range dc.CredentialHelpers {
if !HostValidate(name) {
continue
}
h := HostNewName(name)
h.CredHelper = dockerHelperPre + helper
if _, ok := dc.AuthConfigs[name]; ok {
continue // skip fields with auth config
}
hosts = append(hosts, *h)
}
// add credStore entries
if dc.CredentialsStore != "" {
ch := newCredHelper(dockerHelperPre+dc.CredentialsStore, map[string]string{})
csHosts, err := ch.list()
if err == nil {
hosts = append(hosts, csHosts...)
}
}
return hosts, nil
}
// dockerAuthToHost parses an auth entry from a docker config into a Host.
func dockerAuthToHost(name string, conf dockerConfig, auth dockerAuthConfig) (Host, error) {
helper := ""
if conf.CredentialHelpers != nil && conf.CredentialHelpers[name] != "" {
helper = dockerHelperPre + conf.CredentialHelpers[name]
}
// parse base64 auth into user/pass
if auth.Auth != "" {
var err error
auth.Username, auth.Password, err = decodeAuth(auth.Auth)
if err != nil {
return Host{}, err
}
}
if (auth.Username == "" || auth.Password == "") && auth.IdentityToken == "" && helper == "" {
return Host{}, fmt.Errorf("no credentials found for %s", name)
}
h := HostNewName(name)
// ignore unknown names
if h.Name != DockerRegistry && !strings.HasSuffix(strings.TrimSuffix(name, "/"), h.Name) {
return Host{}, fmt.Errorf("rejecting entry with repository: %s", name)
}
h.User = auth.Username
h.Pass = auth.Password
h.Token = auth.IdentityToken
h.CredHelper = helper
return *h, nil
}
// decodeAuth extracts a base64 encoded user:pass into the username and password.
func decodeAuth(authStr string) (string, string, error) {
if authStr == "" {
return "", "", nil
}
decoded, err := base64.StdEncoding.DecodeString(authStr)
if err != nil {
return "", "", err
}
userPass := strings.SplitN(string(decoded), ":", 2)
if len(userPass) != 2 {
return "", "", fmt.Errorf("invalid auth configuration file")
}
return userPass[0], strings.Trim(userPass[1], "\x00"), nil
}
+521
View File
@@ -0,0 +1,521 @@
// Package config is used for all regclient configuration settings.
package config
import (
"encoding/json"
"fmt"
"io"
"log/slog"
"maps"
"slices"
"strings"
"time"
"github.com/regclient/regclient/internal/timejson"
)
// TLSConf specifies whether TLS is enabled and verified for a host.
type TLSConf int
const (
// TLSUndefined indicates TLS is not passed, defaults to Enabled.
TLSUndefined TLSConf = iota
// TLSEnabled uses TLS (https) for the connection.
TLSEnabled
// TLSInsecure uses TLS but does not verify CA.
TLSInsecure
// TLSDisabled does not use TLS (http).
TLSDisabled
)
const (
// DockerRegistry is the name resolved in docker images on Hub.
DockerRegistry = "docker.io"
// DockerRegistryAuth is the name provided in docker's config for Hub.
DockerRegistryAuth = "https://index.docker.io/v1/"
// DockerRegistryDNS is the host to connect to for Hub.
DockerRegistryDNS = "registry-1.docker.io"
// defaultExpire is the default time to expire a credential and force re-authentication.
defaultExpire = time.Hour * 1
// defaultCredHelperRetry is the time to refresh a credential from a failed credential helper command.
defaultCredHelperRetry = time.Second * 5
// defaultConcurrent is the default number of concurrent registry connections.
defaultConcurrent = 3
// defaultReqPerSec is the default maximum frequency to send requests to a registry.
defaultReqPerSec = 0
// tokenUser is the username returned by credential helpers that indicates the password is an identity token.
tokenUser = "<token>"
)
// MarshalJSON converts TLSConf to a json string using MarshalText.
func (t TLSConf) MarshalJSON() ([]byte, error) {
s, err := t.MarshalText()
if err != nil {
return []byte(""), err
}
return json.Marshal(string(s))
}
// MarshalText converts TLSConf to a string.
func (t TLSConf) MarshalText() ([]byte, error) {
var s string
switch t {
default:
s = ""
case TLSEnabled:
s = "enabled"
case TLSInsecure:
s = "insecure"
case TLSDisabled:
s = "disabled"
}
return []byte(s), nil
}
// UnmarshalJSON converts TLSConf from a json string.
func (t *TLSConf) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err != nil {
return err
}
return t.UnmarshalText([]byte(s))
}
// UnmarshalText converts TLSConf from a string.
func (t *TLSConf) UnmarshalText(b []byte) error {
switch strings.ToLower(string(b)) {
default:
return fmt.Errorf("unknown TLS value \"%s\"", b)
case "":
*t = TLSUndefined
case "enabled":
*t = TLSEnabled
case "insecure":
*t = TLSInsecure
case "disabled":
*t = TLSDisabled
}
return nil
}
// Host defines settings for connecting to a registry.
type Host struct {
Name string `json:"-" yaml:"registry,omitempty"` // Name of the registry (required) (yaml configs pass this as a field, json provides this from the object key)
TLS TLSConf `json:"tls,omitempty" yaml:"tls"` // TLS setting: enabled (default), disabled, insecure
RegCert string `json:"regcert,omitempty" yaml:"regcert"` // public pem cert of registry
ClientCert string `json:"clientCert,omitempty" yaml:"clientCert"` // public pem cert for client (mTLS)
ClientKey string `json:"clientKey,omitempty" yaml:"clientKey"` //#nosec G117 private pem cert for client (mTLS)
Hostname string `json:"hostname,omitempty" yaml:"hostname"` // hostname of registry, default is the registry name
User string `json:"user,omitempty" yaml:"user"` // username, not used with credHelper
Pass string `json:"pass,omitempty" yaml:"pass"` //#nosec G117 password, not used with credHelper
Token string `json:"token,omitempty" yaml:"token"` // token, experimental for specific APIs
CredHelper string `json:"credHelper,omitempty" yaml:"credHelper"` // credential helper command for requesting logins
CredExpire timejson.Duration `json:"credExpire,omitempty" yaml:"credExpire"` // time until credential expires
CredHost string `json:"credHost,omitempty" yaml:"credHost"` // used when a helper hostname doesn't match Hostname
PathPrefix string `json:"pathPrefix,omitempty" yaml:"pathPrefix"` // used for mirrors defined within a repository namespace
Mirrors []string `json:"mirrors,omitempty" yaml:"mirrors"` // list of other Host Names to use as mirrors
Priority uint `json:"priority,omitempty" yaml:"priority"` // priority when sorting mirrors, higher priority attempted first
RepoAuth bool `json:"repoAuth,omitempty" yaml:"repoAuth"` // tracks a separate auth per repo
API string `json:"api,omitempty" yaml:"api"` // Deprecated: registry API to use
APIOpts map[string]string `json:"apiOpts,omitempty" yaml:"apiOpts"` // options for APIs
BlobChunk int64 `json:"blobChunk,omitempty" yaml:"blobChunk"` // size of each blob chunk
BlobMax int64 `json:"blobMax,omitempty" yaml:"blobMax"` // threshold to switch to chunked upload, -1 to disable, 0 for regclient.blobMaxPut
ReqPerSec float64 `json:"reqPerSec,omitempty" yaml:"reqPerSec"` // requests per second
ReqConcurrent int64 `json:"reqConcurrent,omitempty" yaml:"reqConcurrent"` // concurrent requests, default is defaultConcurrent(3)
Scheme string `json:"scheme,omitempty" yaml:"scheme"` // Deprecated: use TLS instead
credRefresh time.Time `json:"-" yaml:"-"` // internal use, when to refresh credentials
}
// Cred defines a user credential for accessing a registry.
type Cred struct {
User, Password, Token string //#nosec G117 exported struct intentionally holds secrets
}
// HostNew creates a default Host entry.
func HostNew() *Host {
h := Host{
TLS: TLSEnabled,
APIOpts: map[string]string{},
ReqConcurrent: int64(defaultConcurrent),
ReqPerSec: float64(defaultReqPerSec),
}
return &h
}
// HostNewName creates a default Host with a hostname.
func HostNewName(name string) *Host {
return HostNewDefName(nil, name)
}
// HostNewDefName creates a host using provided defaults and hostname.
func HostNewDefName(def *Host, name string) *Host {
var h Host
if def == nil {
h = *HostNew()
} else {
h = *def
// configure required defaults
if h.TLS == TLSUndefined {
h.TLS = TLSEnabled
}
if h.APIOpts == nil {
h.APIOpts = map[string]string{}
}
if h.ReqConcurrent == 0 {
h.ReqConcurrent = int64(defaultConcurrent)
}
if h.ReqPerSec == 0 {
h.ReqPerSec = float64(defaultReqPerSec)
}
// copy any fields that are not passed by value
if len(h.APIOpts) > 0 {
orig := h.APIOpts
h.APIOpts = map[string]string{}
maps.Copy(h.APIOpts, orig)
}
if h.Mirrors != nil {
orig := h.Mirrors
h.Mirrors = make([]string, len(orig))
copy(h.Mirrors, orig)
}
}
// configure host
scheme, registry, _ := parseName(name)
if scheme == "http" {
h.TLS = TLSDisabled
}
// Docker Hub is a special case
if registry == DockerRegistry {
h.Name = DockerRegistry
h.Hostname = DockerRegistryDNS
h.CredHost = DockerRegistryAuth
return &h
}
h.Name = registry
h.Hostname = registry
if name != registry {
h.CredHost = name
}
return &h
}
// HostValidate returns true if the scheme is missing or a known value, and the path is not set.
func HostValidate(name string) bool {
scheme, _, path := parseName(name)
return path == "" && (scheme == "https" || scheme == "http")
}
// GetCred returns the credential, fetching from a credential helper if needed.
func (host *Host) GetCred() Cred {
// refresh from credHelper if needed
if host.CredHelper != "" && (host.credRefresh.IsZero() || time.Now().After(host.credRefresh)) {
host.refreshHelper()
}
return Cred{User: host.User, Password: host.Pass, Token: host.Token}
}
func (host *Host) refreshHelper() {
if host.CredHelper == "" {
return
}
if host.CredExpire <= 0 {
host.CredExpire = timejson.Duration(defaultExpire)
}
// run a cred helper, calling get method
ch := newCredHelper(host.CredHelper, map[string]string{})
err := ch.get(host)
if err != nil {
host.credRefresh = time.Now().Add(defaultCredHelperRetry)
} else {
host.credRefresh = time.Now().Add(time.Duration(host.CredExpire))
}
}
// IsZero returns true if the struct is set to the zero value or the result of [HostNew].
func (host Host) IsZero() bool {
if (host.TLS != TLSUndefined && host.TLS != TLSEnabled) ||
host.RegCert != "" ||
host.ClientCert != "" ||
host.ClientKey != "" ||
(host.Hostname != "" && host.Hostname != host.Name) ||
host.User != "" ||
host.Pass != "" ||
host.Token != "" ||
host.CredHelper != "" ||
host.CredExpire != 0 ||
host.CredHost != "" ||
host.PathPrefix != "" ||
len(host.Mirrors) != 0 ||
host.Priority != 0 ||
host.RepoAuth ||
len(host.APIOpts) != 0 ||
host.BlobChunk != 0 ||
host.BlobMax != 0 ||
(host.ReqPerSec != 0 && host.ReqPerSec != float64(defaultReqPerSec)) ||
(host.ReqConcurrent != 0 && host.ReqConcurrent != int64(defaultConcurrent)) ||
!host.credRefresh.IsZero() {
return false
}
return true
}
// Merge adds fields from a new config host entry.
func (host *Host) Merge(newHost Host, log *slog.Logger) error {
name := newHost.Name
if name == "" {
name = host.Name
}
if log == nil {
log = slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{}))
}
// merge the existing and new config host
if host.Name == "" {
// only set the name if it's not initialized, this shouldn't normally change
host.Name = newHost.Name
}
if newHost.CredHelper == "" && (newHost.Pass != "" || host.Token != "") {
// unset existing cred helper for user/pass or token
host.CredHelper = ""
host.CredExpire = 0
}
if newHost.CredHelper != "" && newHost.User == "" && newHost.Pass == "" && newHost.Token == "" {
// unset existing user/pass/token for cred helper
host.User = ""
host.Pass = ""
host.Token = ""
}
if newHost.User != "" {
if host.User != "" && host.User != newHost.User {
log.Warn("Changing login user for registry",
slog.String("orig", host.User),
slog.String("new", newHost.User),
slog.String("host", name))
}
host.User = newHost.User
}
if newHost.Pass != "" {
if host.Pass != "" && host.Pass != newHost.Pass {
log.Warn("Changing login password for registry",
slog.String("host", name))
}
host.Pass = newHost.Pass
}
if newHost.Token != "" {
if host.Token != "" && host.Token != newHost.Token {
log.Warn("Changing login token for registry",
slog.String("host", name))
}
host.Token = newHost.Token
}
if newHost.CredHelper != "" {
if host.CredHelper != "" && host.CredHelper != newHost.CredHelper {
log.Warn("Changing credential helper for registry",
slog.String("host", name),
slog.String("orig", host.CredHelper),
slog.String("new", newHost.CredHelper))
}
host.CredHelper = newHost.CredHelper
}
if newHost.CredExpire != 0 {
if host.CredExpire != 0 && host.CredExpire != newHost.CredExpire {
log.Warn("Changing credential expire for registry",
slog.String("host", name),
slog.Any("orig", host.CredExpire),
slog.Any("new", newHost.CredExpire))
}
host.CredExpire = newHost.CredExpire
}
if newHost.CredHost != "" {
if host.CredHost != "" && host.CredHost != newHost.CredHost {
log.Warn("Changing credential host for registry",
slog.String("host", name),
slog.String("orig", host.CredHost),
slog.String("new", newHost.CredHost))
}
host.CredHost = newHost.CredHost
}
if newHost.TLS != TLSUndefined {
if host.TLS != TLSUndefined && host.TLS != newHost.TLS {
tlsOrig, _ := host.TLS.MarshalText()
tlsNew, _ := newHost.TLS.MarshalText()
log.Warn("Changing TLS settings for registry",
slog.String("orig", string(tlsOrig)),
slog.String("new", string(tlsNew)),
slog.String("host", name))
}
host.TLS = newHost.TLS
}
if newHost.RegCert != "" {
if host.RegCert != "" && host.RegCert != newHost.RegCert {
log.Warn("Changing certificate settings for registry",
slog.String("orig", host.RegCert),
slog.String("new", newHost.RegCert),
slog.String("host", name))
}
host.RegCert = newHost.RegCert
}
if newHost.ClientCert != "" {
if host.ClientCert != "" && host.ClientCert != newHost.ClientCert {
log.Warn("Changing client certificate settings for registry",
slog.String("orig", host.ClientCert),
slog.String("new", newHost.ClientCert),
slog.String("host", name))
}
host.ClientCert = newHost.ClientCert
}
if newHost.ClientKey != "" {
if host.ClientKey != "" && host.ClientKey != newHost.ClientKey {
log.Warn("Changing client certificate key settings for registry",
slog.String("host", name))
}
host.ClientKey = newHost.ClientKey
}
if newHost.Hostname != "" {
if host.Hostname != "" && host.Hostname != newHost.Hostname {
log.Warn("Changing hostname settings for registry",
slog.String("orig", host.Hostname),
slog.String("new", newHost.Hostname),
slog.String("host", name))
}
host.Hostname = newHost.Hostname
}
if newHost.PathPrefix != "" {
newHost.PathPrefix = strings.Trim(newHost.PathPrefix, "/") // leading and trailing / are not needed
if host.PathPrefix != "" && host.PathPrefix != newHost.PathPrefix {
log.Warn("Changing path prefix settings for registry",
slog.String("orig", host.PathPrefix),
slog.String("new", newHost.PathPrefix),
slog.String("host", name))
}
host.PathPrefix = newHost.PathPrefix
}
if len(newHost.Mirrors) > 0 {
if len(host.Mirrors) > 0 && !slices.Equal(host.Mirrors, newHost.Mirrors) {
log.Warn("Changing mirror settings for registry",
slog.Any("orig", host.Mirrors),
slog.Any("new", newHost.Mirrors),
slog.String("host", name))
}
host.Mirrors = newHost.Mirrors
}
if newHost.Priority != 0 {
if host.Priority != 0 && host.Priority != newHost.Priority {
log.Warn("Changing priority settings for registry",
slog.Uint64("orig", uint64(host.Priority)),
slog.Uint64("new", uint64(newHost.Priority)),
slog.String("host", name))
}
host.Priority = newHost.Priority
}
if newHost.RepoAuth {
host.RepoAuth = newHost.RepoAuth
}
// TODO: eventually delete
if newHost.API != "" {
log.Warn("API field has been deprecated",
slog.String("api", newHost.API),
slog.String("host", name))
}
if len(newHost.APIOpts) > 0 {
if len(host.APIOpts) > 0 {
merged := maps.Clone(host.APIOpts)
for k, v := range newHost.APIOpts {
if host.APIOpts[k] != "" && host.APIOpts[k] != v {
log.Warn("Changing APIOpts setting for registry",
slog.String("orig", host.APIOpts[k]),
slog.String("new", newHost.APIOpts[k]),
slog.String("opt", k),
slog.String("host", name))
}
merged[k] = v
}
host.APIOpts = merged
} else {
host.APIOpts = newHost.APIOpts
}
}
if newHost.BlobChunk > 0 {
if host.BlobChunk != 0 && host.BlobChunk != newHost.BlobChunk {
log.Warn("Changing blobChunk settings for registry",
slog.Int64("orig", host.BlobChunk),
slog.Int64("new", newHost.BlobChunk),
slog.String("host", name))
}
host.BlobChunk = newHost.BlobChunk
}
if newHost.BlobMax != 0 {
if host.BlobMax != 0 && host.BlobMax != newHost.BlobMax {
log.Warn("Changing blobMax settings for registry",
slog.Int64("orig", host.BlobMax),
slog.Int64("new", newHost.BlobMax),
slog.String("host", name))
}
host.BlobMax = newHost.BlobMax
}
if newHost.ReqPerSec != 0 {
if host.ReqPerSec != 0 && host.ReqPerSec != newHost.ReqPerSec {
log.Warn("Changing reqPerSec settings for registry",
slog.Float64("orig", host.ReqPerSec),
slog.Float64("new", newHost.ReqPerSec),
slog.String("host", name))
}
host.ReqPerSec = newHost.ReqPerSec
}
if newHost.ReqConcurrent > 0 {
if host.ReqConcurrent != 0 && host.ReqConcurrent != newHost.ReqConcurrent {
log.Warn("Changing reqPerSec settings for registry",
slog.Int64("orig", host.ReqConcurrent),
slog.Int64("new", newHost.ReqConcurrent),
slog.String("host", name))
}
host.ReqConcurrent = newHost.ReqConcurrent
}
return nil
}
// parseName splits a registry into the scheme, hostname, and repository/path.
func parseName(name string) (string, string, string) {
scheme := "https"
path := ""
// Docker Hub is a special case
if name == DockerRegistryAuth || name == DockerRegistryDNS || name == DockerRegistry {
return scheme, DockerRegistry, ""
}
// handle http/https prefix
i := strings.Index(name, "://")
if i > 0 {
scheme = name[:i]
name = name[i+3:]
}
// trim any repository path
i = strings.Index(name, "/")
if i > 0 {
path = name[i+1:]
name = name[:i]
}
return scheme, name, path
}
+1905
View File
File diff suppressed because it is too large Load Diff
+923
View File
@@ -0,0 +1,923 @@
// Package auth is used for HTTP authentication
package auth
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"slices"
"strings"
"sync"
"time"
"github.com/regclient/regclient/internal/regnet"
"github.com/regclient/regclient/types/errs"
)
type charLU byte
var charLUs [256]charLU
var defaultClientID = "regclient"
// minTokenLife tokens are required to last at least 60 seconds to support older docker clients
var minTokenLife = 60
// tokenBuffer is used to renew a token before it expires to account for time to process requests on the server
var tokenBuffer = time.Second * 5
const (
isSpace charLU = 1 << iota
isToken
)
func init() {
for c := range 256 {
charLUs[c] = 0
if strings.ContainsRune(" \t\r\n", rune(c)) {
charLUs[c] |= isSpace
}
if (rune('a') <= rune(c) && rune(c) <= rune('z')) || (rune('A') <= rune(c) && rune(c) <= rune('Z') || (rune('0') <= rune(c) && rune(c) <= rune('9')) || strings.ContainsRune("-._~+/", rune(c))) {
charLUs[c] |= isToken
}
}
}
// CredsFn is passed to lookup credentials for a given hostname, response is a username and password or empty strings
type CredsFn func(host string) Cred
// Cred is returned by the CredsFn.
// If Token is provided and auth method is bearer, it will attempt to use it as a refresh token.
// Else if user and password are provided, they are attempted with all auth methods.
// Else if neither are provided and auth method is bearer, an anonymous login is attempted.
type Cred struct {
//#nosec G117 exported struct intentionally holds secrets
User, Password string // clear text username and password
Token string // refresh token only used for bearer auth
}
// challenge is the extracted contents of the WWW-Authenticate header.
type challenge struct {
authType string
params map[string]string
}
// handler handles a challenge for a host to return an auth header
type handler interface {
AddScope(scope string) error
ProcessChallenge(challenge) error
UpdateRequest(*http.Request) error
}
// handlerBuild is used to make a new handler for a specific authType and URL
type handlerBuild func(client *http.Client, clientID, host string, credFn CredsFn, slog *slog.Logger) handler
// Opts configures options for NewAuth
type Opts func(*Auth)
// Auth is used to handle authentication requests.
type Auth struct {
httpClient *http.Client
clientID string
credsFn CredsFn
hbs map[string]handlerBuild // handler builders based on authType
hs map[string]map[string]handler // handlers based on url and authType
authTypes []string
slog *slog.Logger
mu sync.Mutex
}
// NewAuth creates a new Auth
func NewAuth(opts ...Opts) *Auth {
a := &Auth{
httpClient: &http.Client{},
clientID: defaultClientID,
credsFn: DefaultCredsFn,
hbs: map[string]handlerBuild{},
hs: map[string]map[string]handler{},
authTypes: []string{},
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
}
for _, opt := range opts {
opt(a)
}
if len(a.authTypes) == 0 {
a.addDefaultHandlers()
}
return a
}
// WithCreds provides a user/pass lookup for a url
func WithCreds(f CredsFn) Opts {
return func(a *Auth) {
if f != nil {
a.credsFn = f
}
}
}
// WithHTTPClient uses a specific http client with requests
func WithHTTPClient(h *http.Client) Opts {
return func(a *Auth) {
if h != nil {
a.httpClient = h
}
}
}
// WithClientID uses a client ID with request headers
func WithClientID(clientID string) Opts {
return func(a *Auth) {
a.clientID = clientID
}
}
// WithHandler includes a handler for a specific auth type
func WithHandler(authType string, hb handlerBuild) Opts {
return func(a *Auth) {
lcat := strings.ToLower(authType)
a.hbs[lcat] = hb
a.authTypes = append(a.authTypes, lcat)
}
}
// WithDefaultHandlers includes a Basic and Bearer handler, this is automatically added with "WithHandler" is not called
func WithDefaultHandlers() Opts {
return func(a *Auth) {
a.addDefaultHandlers()
}
}
// WithLog injects a Logger
func WithLog(slog *slog.Logger) Opts {
return func(a *Auth) {
a.slog = slog
}
}
// AddScope extends an existing auth with additional scopes.
// This is used to pre-populate scopes with the Docker convention rather than
// depend on the registry to respond with the correct http status and headers.
func (a *Auth) AddScope(host, scope string) error {
a.mu.Lock()
defer a.mu.Unlock()
success := false
if a.hs[host] == nil {
return errs.ErrNoNewChallenge
}
for _, at := range a.authTypes {
if a.hs[host][at] != nil {
err := a.hs[host][at].AddScope(scope)
if err == nil {
success = true
} else if err != errs.ErrNoNewChallenge {
return err
}
}
}
if !success {
return errs.ErrNoNewChallenge
}
a.slog.Debug("Auth scope added",
slog.String("host", host),
slog.String("scope", scope))
return nil
}
// HandleResponse parses the 401 response, extracting the WWW-Authenticate
// header and verifying the requirement is different from what was included in
// the last request
func (a *Auth) HandleResponse(resp *http.Response) error {
a.mu.Lock()
defer a.mu.Unlock()
// verify response is an access denied
if resp.StatusCode != http.StatusUnauthorized {
return errs.ErrUnsupported
}
// extract host and auth header
host := resp.Request.URL.Host
cl, err := ParseAuthHeaders(resp.Header.Values("WWW-Authenticate"))
if err != nil {
return err
}
a.slog.Debug("Auth request parsed",
slog.Any("challenge", cl))
if len(cl) < 1 {
return errs.ErrEmptyChallenge
}
goodChallenge := false
// loop over the received challenge(s)
for _, c := range cl {
if _, ok := a.hbs[c.authType]; !ok {
a.slog.Warn("Unsupported auth type",
slog.String("authtype", c.authType))
continue
}
// setup a handler for the host and auth type
if _, ok := a.hs[host]; !ok {
a.hs[host] = map[string]handler{}
}
if _, ok := a.hs[host][c.authType]; !ok {
h := a.hbs[c.authType](a.httpClient, a.clientID, host, a.credsFn, a.slog)
if h == nil {
continue
}
a.hs[host][c.authType] = h
}
// process the challenge with that handler
err := a.hs[host][c.authType].ProcessChallenge(c)
if err == nil {
goodChallenge = true
} else if err == errs.ErrNoNewChallenge {
// handle race condition when another request updates the challenge
// detect that by seeing the current auth header is different
prevAH := resp.Request.Header.Get("Authorization")
err := a.hs[host][c.authType].UpdateRequest(resp.Request)
if err == nil && prevAH != resp.Request.Header.Get("Authorization") {
goodChallenge = true
}
} else {
return err
}
}
if !goodChallenge {
return errs.ErrHTTPUnauthorized
}
return nil
}
// UpdateRequest adds Authorization headers to a request
func (a *Auth) UpdateRequest(req *http.Request) error {
a.mu.Lock()
defer a.mu.Unlock()
host := req.URL.Host
if a.hs[host] == nil {
return nil
}
var err error
for _, at := range a.authTypes {
if a.hs[host][at] != nil {
err = a.hs[host][at].UpdateRequest(req)
if err != nil {
a.slog.Debug("Failed to generate auth",
slog.String("err", err.Error()),
slog.String("host", host),
slog.String("authtype", at))
continue
}
break
}
}
if err != nil {
return err
}
return nil
}
func (a *Auth) addDefaultHandlers() {
if _, ok := a.hbs["basic"]; !ok {
a.hbs["basic"] = NewBasicHandler
a.authTypes = append(a.authTypes, "basic")
}
if _, ok := a.hbs["bearer"]; !ok {
a.hbs["bearer"] = NewBearerHandler
a.authTypes = append(a.authTypes, "bearer")
}
}
// DefaultCredsFn is used to return no credentials when auth is not configured with a CredsFn
// This avoids the need to check for nil pointers
func DefaultCredsFn(h string) Cred {
return Cred{}
}
// ParseAuthHeaders extracts the scheme and realm from WWW-Authenticate headers
func ParseAuthHeaders(ahl []string) ([]challenge, error) {
var cl []challenge
for _, ah := range ahl {
c, err := parseAuthHeader(ah)
if err != nil {
return nil, fmt.Errorf("failed to parse challenge header: %s, %w", ah, err)
}
cl = append(cl, c...)
}
return cl, nil
}
// parseAuthHeader parses a single header line for WWW-Authenticate
// Example values:
// Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push"
// Basic realm="GitHub Package Registry"
func parseAuthHeader(ah string) ([]challenge, error) {
var cl []challenge
var c *challenge
curElement := []byte{}
curKey := ""
stateElement := "string"
stateSyntax := "start"
for _, b := range []byte(ah) {
switch stateElement {
case "string":
// string: ignore leading space, enter quote only if first character, handle escapes, handle valid tokens, else end
if charLUs[b]&isToken != 0 {
// add valid tokens to element
curElement = append(curElement, b)
} else if charLUs[b]&isSpace != 0 && len(curElement) == 0 {
// ignore leading spaces
} else if b == '"' && len(curElement) == 0 {
stateElement = "quote"
} else if b == '\\' {
stateElement = "escape_string"
} else {
stateElement = "end"
}
case "quote":
// quote: handle escapes, handle closing quote (quote_end to read next character), all other tokens are valid
if b == '\\' {
stateElement = "escape_quote"
} else if b == '"' {
stateElement = "quote_end"
} else {
curElement = append(curElement, b)
}
case "quote_end":
// any character after the close quote is the end of the element
stateElement = "end"
case "escape_string":
// escape_string: handle any character and return to string state
curElement = append(curElement, b)
stateElement = "string"
case "escape_quote":
// escape_quote: handle any character and return to quote state
curElement = append(curElement, b)
stateElement = "quote"
case "end":
// finished parsing element, continue to processing element according to the current state
default:
return nil, fmt.Errorf("unhandled element case: %w", errs.ErrParsingFailed)
}
if stateElement != "end" {
// continue parsing the element until it ends
continue
}
// syntax looks at each string within the overall challenge syntax
switch stateSyntax {
case "start":
// start: (start of auth_type) read auth_type and space (end_auth_type) or auth_type and comma (start)
if charLUs[b]&isSpace != 0 && len(curElement) > 0 {
stateSyntax = "end_auth_type"
} else if b == ',' && len(curElement) > 0 {
// state remains at start
} else {
return nil, fmt.Errorf("start element did not end with a space or comma: %w", errs.ErrParsingFailed)
}
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
cl = append(cl, *c)
case "start_or_param":
// start_or_param: (after param_value) read auth_type and space (end_auth_type) or param_key and equals (param_value)
if charLUs[b]&isSpace != 0 && len(curElement) > 0 {
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
cl = append(cl, *c)
stateSyntax = "end_auth_type"
} else if b == '=' && len(curElement) > 0 {
curKey = strings.ToLower(string(curElement))
stateSyntax = "param_value"
} else {
return nil, fmt.Errorf("expected auth type or param: %w", errs.ErrParsingFailed)
}
case "end_auth_type":
// end_auth_type: (after reading auth_type) read param_key and equals (param_value) or just a comma (start)
if b == '=' && len(curElement) > 0 {
curKey = strings.ToLower(string(curElement))
stateSyntax = "param_value"
} else if b == ',' && len(curElement) == 0 {
// ignore white space between end of auth_type and comma
stateSyntax = "start"
} else {
return nil, fmt.Errorf("expected param or comma: %w", errs.ErrParsingFailed)
}
case "param_value":
// param_value: (after param_key) read param_value and comma (start_or_param)
if b == ',' {
c.params[curKey] = string(curElement)
stateSyntax = "start_or_param"
curKey = ""
} else {
return nil, fmt.Errorf("expected param value: %w", errs.ErrParsingFailed)
}
default:
return nil, fmt.Errorf("unhandled syntax case: %w", errs.ErrParsingFailed)
}
// reset element state
stateElement = "string"
curElement = []byte{}
}
// at end of parsing, if the element is not empty, process according to syntax state:
if len(curElement) > 0 {
// ensure this is not within an unclosed quote or partial escape
if stateElement != "string" && stateElement != "quote_end" {
return nil, fmt.Errorf("eol element in state %s: %w", stateElement, errs.ErrParsingFailed)
}
switch stateSyntax {
case "start", "start_or_param":
// add a new auth type if a string is seen at the start, before any equals
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
cl = append(cl, *c)
case "param_value":
// add the last param key=val
c.params[curKey] = string(curElement)
case "end_auth_type":
// missing equals for param
return nil, fmt.Errorf("eol at param without value: %w", errs.ErrParsingFailed)
}
}
return cl, nil
}
// basicHandler supports Basic auth type requests
type basicHandler struct {
realm string
host string
credsFn CredsFn
}
// NewBasicHandler creates a new BasicHandler
func NewBasicHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
return &basicHandler{
realm: "",
host: host,
credsFn: credsFn,
}
}
// AddScope is not valid for BasicHandler
func (b *basicHandler) AddScope(scope string) error {
return errs.ErrNoNewChallenge
}
// ProcessChallenge for BasicHandler is a noop
func (b *basicHandler) ProcessChallenge(c challenge) error {
if _, ok := c.params["realm"]; !ok {
return errs.ErrInvalidChallenge
}
if b.realm != c.params["realm"] {
b.realm = c.params["realm"]
return nil
}
return errs.ErrNoNewChallenge
}
// UpdateRequest for BasicHandler generates base64 encoded user/pass for a host
func (b *basicHandler) UpdateRequest(req *http.Request) error {
cred := b.credsFn(b.host)
if cred.User == "" || cred.Password == "" {
return fmt.Errorf("no credentials available: %w", errs.ErrHTTPUnauthorized)
}
req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
base64.StdEncoding.EncodeToString([]byte(cred.User+":"+cred.Password))))
return nil
}
// bearerHandler supports Bearer auth type requests
type bearerHandler struct {
client *http.Client
clientID string
realm, service string
host string
credsFn CredsFn
scopes []string
tokenURL *url.URL
token bearerToken
slog *slog.Logger
}
// bearerToken is the json response to the Bearer request
type bearerToken struct {
Token string `json:"token"`
AccessToken string `json:"access_token"` //#nosec G117 exported struct intentionally holds secrets
ExpiresIn int `json:"expires_in"`
IssuedAt time.Time `json:"issued_at"`
RefreshToken string `json:"refresh_token"` //#nosec G117 exported struct intentionally holds secrets
Scope string `json:"scope"`
}
// NewBearerHandler creates a new BearerHandler
func NewBearerHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
return &bearerHandler{
client: client,
clientID: clientID,
host: host,
credsFn: credsFn,
realm: "",
service: "",
scopes: []string{},
slog: slog,
}
}
// AddScope appends a new scope if it doesn't already exist
func (b *bearerHandler) AddScope(scope string) error {
if b.scopeExists(scope) {
if b.token.Token == "" || !b.isExpired() {
return errs.ErrNoNewChallenge
}
return nil
}
b.addScope(scope)
return nil
}
func (b *bearerHandler) addScope(scope string) {
if !b.tryExtendExistingScope(scope) {
b.scopes = append(b.scopes, scope)
}
// delete old token
b.token.Token = ""
}
var knownActions = []string{"pull", "push", "delete"}
// tryExtendExistingScope extends an existing scope if both the new scope and the current scope contain only knownActions.
// It returns true if actions are added or are already present. Otherwise, it returns false,
// indicating that the new scope should be appended to b.scopes instead.
func (b *bearerHandler) tryExtendExistingScope(scope string) bool {
repo, actions, ok := parseScope(scope)
if !ok {
return false
}
scopePrefix := "repository:" + repo + ":"
for i, cur := range b.scopes {
if !strings.HasPrefix(cur, scopePrefix) {
continue
}
_, curActions, curOk := parseScope(cur)
if !curOk {
continue
}
for _, a := range actions {
if !slices.Contains(curActions, a) {
curActions = append(curActions, a)
}
}
b.scopes[i] = scopePrefix + strings.Join(curActions, ",")
return true
}
return false
}
// parseScope splits a scope into the repo and slice of actions.
// Unknown actions in the scope will set bool to false.
func parseScope(scope string) (string, []string, bool) {
scopeSplit := strings.SplitN(scope, ":", 3)
if scopeSplit[0] != "repository" || len(scopeSplit) < 3 {
return "", nil, false
}
actionSplit := strings.Split(scopeSplit[2], ",")
for _, a := range actionSplit {
if !slices.Contains(knownActions, a) {
return "", nil, false
}
}
return scopeSplit[1], actionSplit, true
}
// ProcessChallenge handles WWW-Authenticate header for bearer tokens
// Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push"
func (b *bearerHandler) ProcessChallenge(c challenge) error {
if _, ok := c.params["realm"]; !ok {
return errs.ErrInvalidChallenge
}
if _, ok := c.params["service"]; !ok {
c.params["service"] = ""
}
if _, ok := c.params["scope"]; !ok {
c.params["scope"] = ""
}
existingScope := b.scopeExists(c.params["scope"])
if b.realm == c.params["realm"] && b.service == c.params["service"] && existingScope && (b.token.Token == "" || !b.isExpired()) {
return errs.ErrNoNewChallenge
}
if b.realm == "" {
b.realm = c.params["realm"]
} else if b.realm != c.params["realm"] {
return errs.ErrInvalidChallenge
}
if b.service == "" {
b.service = c.params["service"]
} else if b.service != c.params["service"] {
return errs.ErrInvalidChallenge
}
if !existingScope {
b.addScope(c.params["scope"])
}
return nil
}
// UpdateRequest for BearerHandler adds a bearer token to the request.
func (b *bearerHandler) UpdateRequest(req *http.Request) error {
// handle relative realm values
if b.tokenURL == nil {
u, err := req.URL.Parse(b.realm)
if err != nil {
return err
}
b.tokenURL = u
}
// verify tokenURL is allowed for request URL
if err := regnet.AllowRedirect(*req.URL, *b.tokenURL); err != nil {
return err
}
// if unexpired token already exists, return it
if b.token.Token != "" && !b.isExpired() {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
return nil
}
// attempt to post if a refresh token is available or token auth is being used
cred := b.credsFn(b.host)
if b.token.RefreshToken != "" || cred.Token != "" {
if err := b.tryPost(cred); err == nil {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
return nil
} else if err != errs.ErrHTTPUnauthorized {
return fmt.Errorf("failed to request auth token (post): %w%.0w", err, errs.ErrHTTPUnauthorized)
}
}
// attempt a get (with basic auth if user/pass available)
if err := b.tryGet(cred); err == nil {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
return nil
} else if err != errs.ErrHTTPUnauthorized {
return fmt.Errorf("failed to request auth token (get): %w%.0w", err, errs.ErrHTTPUnauthorized)
}
return errs.ErrHTTPUnauthorized
}
// isExpired returns true when token issue date is either 0, token has expired,
// or will expire within buffer time
func (b *bearerHandler) isExpired() bool {
if b.token.IssuedAt.IsZero() {
return true
}
expireSec := b.token.IssuedAt.Add(time.Duration(b.token.ExpiresIn) * time.Second)
expireSec = expireSec.Add(tokenBuffer * -1)
return time.Now().After(expireSec)
}
// tryGet requests a new token with a GET request
func (b *bearerHandler) tryGet(cred Cred) error {
//#nosec G704 inputs follow specification
req, err := http.NewRequest("GET", b.tokenURL.String(), nil)
if err != nil {
return err
}
reqParams := req.URL.Query()
reqParams.Add("client_id", b.clientID)
// Note, an offline_token should not be requested by default due to broken OAuth2 implementations returning an invalid token
if b.service != "" {
reqParams.Add("service", b.service)
}
for _, s := range b.scopes {
reqParams.Add("scope", s)
}
if cred.User != "" && cred.Password != "" {
reqParams.Add("account", cred.User)
req.SetBasicAuth(cred.User, cred.Password)
}
req.Header.Add("User-Agent", b.clientID)
req.URL.RawQuery = reqParams.Encode()
//#nosec G704 inputs follow specification
resp, err := b.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return b.validateResponse(resp)
}
// tryPost requests a new token via a POST request
func (b *bearerHandler) tryPost(cred Cred) error {
form := url.Values{}
if len(b.scopes) > 0 {
form.Set("scope", strings.Join(b.scopes, " "))
}
if b.service != "" {
form.Set("service", b.service)
}
form.Set("client_id", b.clientID)
if b.token.RefreshToken != "" {
form.Set("grant_type", "refresh_token")
form.Set("refresh_token", b.token.RefreshToken)
} else if cred.Token != "" {
form.Set("grant_type", "refresh_token")
form.Set("refresh_token", cred.Token)
} else if cred.User != "" && cred.Password != "" {
form.Set("grant_type", "password")
form.Set("username", cred.User)
form.Set("password", cred.Password)
}
//#nosec G704 inputs are user controlled or follow specification
req, err := http.NewRequest("POST", b.tokenURL.String(), strings.NewReader(form.Encode()))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
req.Header.Add("User-Agent", b.clientID)
//#nosec G704 inputs are user controlled or follow specification
resp, err := b.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return b.validateResponse(resp)
}
// scopeExists check if the scope already exists within the list of scopes
func (b *bearerHandler) scopeExists(search string) bool {
if search == "" {
return true
}
searchRepo, searchActions, searchOk := parseScope(search)
if !searchOk {
return slices.Contains(b.scopes, search)
}
scopePrefix := "repository:" + searchRepo + ":"
for _, scope := range b.scopes {
if scope == search {
return true
}
if !strings.HasPrefix(scope, scopePrefix) {
continue
}
_, actions, ok := parseScope(scope)
if !ok {
continue
}
for _, sa := range searchActions {
if !slices.Contains(actions, sa) {
return false
}
}
return true
}
return false
}
// validateResponse extracts the returned token
func (b *bearerHandler) validateResponse(resp *http.Response) error {
if resp.StatusCode != 200 {
return errs.ErrHTTPUnauthorized
}
// decode response and if successful, update token
decoder := json.NewDecoder(resp.Body)
decoded := bearerToken{}
if err := decoder.Decode(&decoded); err != nil {
return err
}
b.token = decoded
if b.token.ExpiresIn < minTokenLife {
b.token.ExpiresIn = minTokenLife
}
// If token is already expired, it was sent with a zero value or
// there may be a clock skew between the client and auth server.
// Also handle cases of remote time in the future.
// But if remote time is slightly in the past, leave as is so token
// expires here before the server.
if b.isExpired() || b.token.IssuedAt.After(time.Now()) {
b.token.IssuedAt = time.Now().UTC()
}
// AccessToken and Token should be the same and we use Token elsewhere
if b.token.AccessToken != "" {
b.token.Token = b.token.AccessToken
}
return nil
}
// jwtHubHandler supports JWT auth type requests.
type jwtHubHandler struct {
client *http.Client
clientID string
realm string
host string
credsFn CredsFn
jwt string
}
type jwtHubPost struct {
User string `json:"username"`
Pass string `json:"password"` //#nosec G117 exported struct intentionally holds secrets
}
type jwtHubResp struct {
Detail string `json:"detail"`
Token string `json:"token"`
RefreshToken string `json:"refresh_token"` //#nosec G117 exported struct intentionally holds secrets
}
// NewJWTHubHandler creates a new JWTHandler for Docker Hub.
func NewJWTHubHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
// JWT handler is only tested against Hub, and the API is Hub specific
if host == "hub.docker.com" {
return &jwtHubHandler{
client: client,
clientID: clientID,
host: host,
credsFn: credsFn,
realm: "https://hub.docker.com/v2/users/login",
}
}
return nil
}
// AddScope is not valid for JWTHubHandler
func (j *jwtHubHandler) AddScope(scope string) error {
return errs.ErrNoNewChallenge
}
// ProcessChallenge handles WWW-Authenticate header for JWT auth on Docker Hub
func (j *jwtHubHandler) ProcessChallenge(c challenge) error {
cred := j.credsFn(j.host)
// use token if provided
if cred.Token != "" {
j.jwt = cred.Token
return nil
}
// send a login request to hub
bodyBytes, err := json.Marshal(jwtHubPost{
User: cred.User,
Pass: cred.Password, //#nosec G117 field name follows spec
})
if err != nil {
return err
}
req, err := http.NewRequest("POST", j.realm, bytes.NewReader(bodyBytes))
if err != nil {
return err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
req.Header.Add("User-Agent", j.clientID)
//#nosec G704 inputs are user controlled or follow specification requirements
resp, err := j.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != 200 || resp.StatusCode >= 300 {
return errs.ErrHTTPUnauthorized
}
var bodyParsed jwtHubResp
err = json.Unmarshal(body, &bodyParsed)
if err != nil {
return err
}
j.jwt = bodyParsed.Token
return nil
}
// UpdateRequest for JWTHubHandler adds JWT header
func (j *jwtHubHandler) UpdateRequest(req *http.Request) error {
if len(j.jwt) > 0 {
req.Header.Set("Authorization", fmt.Sprintf("JWT %s", j.jwt))
return nil
}
return errs.ErrHTTPUnauthorized
}
+48
View File
@@ -0,0 +1,48 @@
package auth
import (
"github.com/regclient/regclient/types/errs"
)
var (
// ErrEmptyChallenge indicates an issue with the received challenge in the WWW-Authenticate header
//
// Deprecated: replace with [errs.ErrEmptyChallenge].
//go:fix inline
ErrEmptyChallenge = errs.ErrEmptyChallenge
// ErrInvalidChallenge indicates an issue with the received challenge in the WWW-Authenticate header
//
// Deprecated: replace with [errs.ErrInvalidChallenge].
//go:fix inline
ErrInvalidChallenge = errs.ErrInvalidChallenge
// ErrNoNewChallenge indicates a challenge update did not result in any change
//
// Deprecated: replace with [errs.ErrNoNewChallenge].
//go:fix inline
ErrNoNewChallenge = errs.ErrNoNewChallenge
// ErrNotFound indicates no credentials found for basic auth
//
// Deprecated: replace with [errs.ErrNotFound].
//go:fix inline
ErrNotFound = errs.ErrNotFound
// ErrNotImplemented returned when method has not been implemented yet
//
// Deprecated: replace with [errs.ErrNotImplemented].
//go:fix inline
ErrNotImplemented = errs.ErrNotImplemented
// ErrParseFailure indicates the WWW-Authenticate header could not be parsed
//
// Deprecated: replace with [errs.ErrParseFailure].
//go:fix inline
ErrParseFailure = errs.ErrParsingFailed
// ErrUnauthorized request was not authorized
//
// Deprecated: replace with [errs.ErrUnauthorized].
//go:fix inline
ErrUnauthorized = errs.ErrHTTPUnauthorized
// ErrUnsupported indicates the request was unsupported
//
// Deprecated: replace with [errs.ErrUnsupported].
//go:fix inline
ErrUnsupported = errs.ErrUnsupported
)
+181
View File
@@ -0,0 +1,181 @@
//go:build go1.18
// Package cache is used to store values with limits.
// Items are automatically pruned when too many entries are stored, or values become stale.
package cache
import (
"sort"
"sync"
"time"
"github.com/regclient/regclient/types/errs"
)
type Cache[k comparable, v any] struct {
mu sync.Mutex
minAge time.Duration
maxAge time.Duration
minCount int
maxCount int
timer *time.Timer
entries map[k]*Entry[v]
}
type Entry[v any] struct {
used time.Time
value v
}
type sortKeys[k comparable] struct {
keys []k
lessFn func(a, b k) bool
}
type conf struct {
minAge time.Duration
maxCount int
}
type cacheOpts func(*conf)
func WithAge(age time.Duration) cacheOpts {
return func(c *conf) {
c.minAge = age
}
}
func WithCount(count int) cacheOpts {
return func(c *conf) {
c.maxCount = count
}
}
func New[k comparable, v any](opts ...cacheOpts) Cache[k, v] {
c := conf{}
for _, opt := range opts {
opt(&c)
}
maxAge := c.minAge + (c.minAge / 10)
minCount := 0
if c.maxCount > 0 {
minCount = int(float64(c.maxCount) * 0.9)
}
return Cache[k, v]{
minAge: c.minAge,
maxAge: maxAge,
minCount: minCount,
maxCount: c.maxCount,
entries: map[k]*Entry[v]{},
}
}
func (c *Cache[k, v]) Delete(key k) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
delete(c.entries, key)
if len(c.entries) == 0 && c.timer != nil {
c.timer.Stop()
c.timer = nil
}
}
func (c *Cache[k, v]) Set(key k, val v) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
c.entries[key] = &Entry[v]{
used: time.Now(),
value: val,
}
if len(c.entries) > c.maxCount {
c.pruneLocked()
} else if c.timer == nil {
// prune resets the timer, so this is only needed if the prune wasn't triggered
c.timer = time.AfterFunc(c.maxAge, c.prune)
}
}
func (c *Cache[k, v]) Get(key k) (v, error) {
if c == nil {
var val v
return val, errs.ErrNotFound
}
c.mu.Lock()
defer c.mu.Unlock()
if e, ok := c.entries[key]; ok {
if e.used.Add(c.minAge).Before(time.Now()) {
// entry expired
go c.prune()
} else {
c.entries[key].used = time.Now()
return e.value, nil
}
}
var val v
return val, errs.ErrNotFound
}
func (c *Cache[k, v]) prune() {
c.mu.Lock()
defer c.mu.Unlock()
c.pruneLocked()
}
func (c *Cache[k, v]) pruneLocked() {
// sort key list by last used date
keyList := make([]k, 0, len(c.entries))
for key := range c.entries {
keyList = append(keyList, key)
}
sk := sortKeys[k]{
keys: keyList,
lessFn: func(a, b k) bool {
return c.entries[a].used.Before(c.entries[b].used)
},
}
sort.Sort(&sk)
// prune entries
now := time.Now()
cutoff := now.Add(c.minAge * -1)
nextTime := now
delCount := len(keyList) - c.minCount
for i, key := range keyList {
if i < delCount || c.entries[key].used.Before(cutoff) {
delete(c.entries, key)
} else {
nextTime = c.entries[key].used
break
}
}
// set next timer
if len(c.entries) > 0 {
dur := nextTime.Sub(now) + c.maxAge
if c.timer == nil {
// this shouldn't be possible
c.timer = time.AfterFunc(dur, c.prune)
} else {
c.timer.Reset(dur)
}
} else if c.timer != nil {
c.timer.Stop()
c.timer = nil
}
}
func (sk *sortKeys[k]) Len() int {
return len(sk.keys)
}
func (sk *sortKeys[k]) Less(i, j int) bool {
return sk.lessFn(sk.keys[i], sk.keys[j])
}
func (sk *sortKeys[k]) Swap(i, j int) {
sk.keys[i], sk.keys[j] = sk.keys[j], sk.keys[i]
}
+188
View File
@@ -0,0 +1,188 @@
// Package conffile wraps the read and write of configuration files
package conffile
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"os/user"
"path/filepath"
)
type File struct {
perms int
fullname string
}
type Opt func(*File)
// New returns a new File.
// The last successful option determines the filename.
func New(opts ...Opt) *File {
f := File{perms: 0o600}
for _, fn := range opts {
fn(&f)
}
if f.fullname == "" {
return nil
}
return &f
}
// WithAppDir determines the filename from the XDG or Windows specification.
// By default, this is based in $HOME/.config on Linux and %APPDATA% on Windows.
// If the file does not exist, this will set the filename only if "force" is true.
func WithAppDir(unixDir, winDir, name string, force bool) Opt {
var dir string
if winDir == "" {
dir = unixDir
} else {
dir = osString(unixDir, winDir)
}
return func(f *File) {
fullname := filepath.Join(appDir(), dir, name)
if force || exists(fullname) {
f.fullname = fullname
}
}
}
// WithDirName determines the filename from a subdirectory in the user's HOME.
//
// Deprecated: Replace with [WithHomeDir]
//
//go:fix inline
func WithDirName(dir, name string) Opt {
return WithHomeDir(dir, name, true)
}
// WithEnvFile sets the fullname to the environment value if defined.
func WithEnvFile(envVar string) Opt {
return func(f *File) {
val := os.Getenv(envVar)
if val != "" {
f.fullname = val
}
}
}
// WithEnvDir sets the fullname to the environment value + filename if the environment variable is defined.
func WithEnvDir(envVar, name string) Opt {
return func(f *File) {
val := os.Getenv(envVar)
if val != "" {
f.fullname = filepath.Join(val, name)
}
}
}
// WithFullname specifies the filename.
// This will always set the filename even if the file does not exist.
func WithFullname(fullname string) Opt {
return func(f *File) {
f.fullname = fullname
}
}
// WithHomeDir determines the filename from a subdirectory in the user's HOME
// e.g. dir=".app", name="config.json", sets the fullname to "$HOME/.app/config.json".
// If the file does not exist, this will set the filename only if "force" is true.
func WithHomeDir(dir, name string, force bool) Opt {
return func(f *File) {
filename := filepath.Join(homeDir(), dir, name)
if force || exists(filename) {
f.fullname = filename
}
}
}
// WithPerms specifies the permissions to create a file with (default 0600).
func WithPerms(perms int) Opt {
return func(f *File) {
f.perms = perms
}
}
func (f *File) Name() string {
return f.fullname
}
func (f *File) Open() (io.ReadCloser, error) {
return os.Open(f.fullname)
}
func (f *File) Write(rdr io.Reader) error {
// create temp file/open
dir := filepath.Dir(f.fullname)
if err := os.MkdirAll(dir, 0o700); err != nil {
return err
}
tmp, err := os.CreateTemp(dir, filepath.Base(f.fullname))
if err != nil {
return err
}
tmpStat, err := tmp.Stat()
if err != nil {
return err
}
tmpName := tmpStat.Name()
tmpFullname := filepath.Join(dir, tmpName)
defer os.Remove(tmpFullname)
// copy from rdr to temp file
_, err = io.Copy(tmp, rdr)
errC := tmp.Close()
if err != nil {
return fmt.Errorf("failed to write config: %w", err)
}
if errC != nil {
return fmt.Errorf("failed to close config: %w", errC)
}
// adjust file ownership/permissions
mode := os.FileMode(0o600)
uid := os.Getuid()
gid := os.Getgid()
// adjust defaults based on existing file if available
stat, err := os.Stat(f.fullname)
if err == nil {
// adjust mode to existing file
if stat.Mode().IsRegular() {
mode = stat.Mode()
}
uid, gid, _ = getFileOwner(stat)
} else if !errors.Is(err, fs.ErrNotExist) {
return err
}
// update mode and owner of temp file
//#nosec G703 tempfile location is user controlled
if err := os.Chmod(tmpFullname, mode); err != nil {
return err
}
if uid > 0 && gid > 0 {
//#nosec G703 tempfile location is user controlled
_ = os.Chown(tmpFullname, uid, gid)
}
// move temp file to target filename
//#nosec G703 tempfile location is user controlled
return os.Rename(tmpFullname, f.fullname)
}
func exists(name string) bool {
_, err := os.Stat(name)
return err == nil
}
func homeDir() string {
home := os.Getenv(homeEnv)
if home == "" {
u, err := user.Current()
if err == nil {
home = u.HomeDir
}
}
return home
}
@@ -0,0 +1,37 @@
//go:build !windows
package conffile
import (
"io/fs"
"os"
"path/filepath"
"syscall"
)
const (
appDirEnv = "XDG_CONFIG_HOME"
homeEnv = "HOME"
)
func appDir() string {
appDir := os.Getenv(appDirEnv)
if appDir == "" {
home := homeDir()
appDir = filepath.Join(home, ".config")
}
return appDir
}
func getFileOwner(stat fs.FileInfo) (int, int, error) {
var uid, gid int
if sysstat, ok := stat.Sys().(*syscall.Stat_t); ok {
uid = int(sysstat.Uid)
gid = int(sysstat.Gid)
}
return uid, gid, nil
}
func osString(unix, _ string) string {
return unix
}
@@ -0,0 +1,31 @@
//go:build windows
package conffile
import (
"io/fs"
"os"
"path/filepath"
)
const (
appDirEnv = "APPDATA"
homeEnv = "USERPROFILE"
)
func appDir() string {
appDir := os.Getenv(appDirEnv)
if appDir == "" {
home := homeDir()
appDir = filepath.Join(home, "AppData")
}
return appDir
}
func getFileOwner(_ fs.FileInfo) (int, int, error) {
return 0, 0, nil
}
func osString(_, win string) string {
return win
}
+198
View File
@@ -0,0 +1,198 @@
// Package httplink parses the Link header from HTTP responses according to RFC5988
package httplink
import (
"fmt"
"strings"
"github.com/regclient/regclient/types/errs"
)
type (
Links []Link
Link struct {
URI string
Param map[string]string
}
)
type charLU byte
var charLUs [256]charLU
const (
isSpace charLU = 1 << iota
isToken
isAlphaNum
)
func init() {
for c := range 256 {
charLUs[c] = 0
if strings.ContainsRune(" \t\r\n", rune(c)) {
charLUs[c] |= isSpace
}
if (rune('a') <= rune(c) && rune(c) <= rune('z')) || (rune('A') <= rune(c) && rune(c) <= rune('Z') || (rune('0') <= rune(c) && rune(c) <= rune('9'))) {
charLUs[c] |= isAlphaNum | isToken
}
if strings.ContainsRune("!#$%&'()*+-./:<=>?@[]^_`{|}~", rune(c)) {
charLUs[c] |= isToken
}
}
}
// Parse reads "Link" http headers into an array of Link structs.
// Header array should be the output of resp.Header.Values("link").
func Parse(headers []string) (Links, error) {
links := []Link{}
for _, h := range headers {
state := "init"
var ub, pnb, pvb []byte
parms := map[string]string{}
endLink := func() {
links = append(links, Link{
URI: string(ub),
Param: parms,
})
// reset state
ub, pnb, pvb = []byte{}, []byte{}, []byte{}
parms = map[string]string{}
}
endParm := func() {
if _, ok := parms[string(pnb)]; !ok {
parms[string(pnb)] = string(pvb)
}
// reset parm
pnb, pvb = []byte{}, []byte{}
}
for i, b := range []byte(h) {
switch state {
case "init":
if b == '<' {
state = "uriQuoted"
} else if charLUs[b]&isToken != 0 {
state = "uri"
ub = append(ub, b)
} else if charLUs[b]&isSpace != 0 || b == ',' {
// noop
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "uri":
// parse tokens until space or comma
if charLUs[b]&isToken != 0 {
ub = append(ub, b)
} else if charLUs[b]&isSpace != 0 {
state = "fieldSep"
} else if b == ';' {
state = "parmName"
} else if b == ',' {
state = "init"
endLink()
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "uriQuoted":
// parse tokens until quote
if b == '>' {
state = "fieldSep"
} else {
ub = append(ub, b)
}
case "fieldSep":
if b == ';' {
state = "parmName"
} else if b == ',' {
state = "init"
endLink()
} else if charLUs[b]&isSpace != 0 {
// noop
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "parmName":
if len(pnb) > 0 && b == '=' {
state = "parmValue"
} else if len(pnb) > 0 && b == '*' {
state = "parmNameStar"
} else if charLUs[b]&isAlphaNum != 0 {
pnb = append(pnb, b)
} else if len(pnb) == 0 && charLUs[b]&isSpace != 0 {
// noop
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "parmNameStar":
if b == '=' {
state = "parmValue"
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "parmValue":
if len(pvb) == 0 {
if charLUs[b]&isToken != 0 {
pvb = append(pvb, b)
} else if b == '"' {
state = "parmValueQuoted"
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
} else {
if charLUs[b]&isToken != 0 {
pvb = append(pvb, b)
} else if charLUs[b]&isSpace != 0 {
state = "fieldSep"
endParm()
} else if b == ';' {
state = "parmName"
endParm()
} else if b == ',' {
state = "init"
endParm()
endLink()
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
}
case "parmValueQuoted":
if b == '"' {
state = "fieldSep"
endParm()
} else {
pvb = append(pvb, b)
}
}
}
// check for valid state at end of header
switch state {
case "parmValue":
endParm()
endLink()
case "uri", "fieldSep":
endLink()
case "init":
// noop
default:
return nil, fmt.Errorf("unexpected end state %s for header %s: %w", state, h, errs.ErrParsingFailed)
}
}
return links, nil
}
// Get returns a link with a specific parm value, e.g. rel="next"
func (links Links) Get(parm, val string) (Link, error) {
for _, link := range links {
if link.Param != nil && link.Param[parm] == val {
return link, nil
}
}
return Link{}, errs.ErrNotFound
}
+29
View File
@@ -0,0 +1,29 @@
// Package limitread provides a reader that will error if the limit is ever exceeded
package limitread
import (
"fmt"
"io"
"github.com/regclient/regclient/types/errs"
)
type LimitRead struct {
Reader io.Reader
Limit int64
}
func (lr *LimitRead) Read(p []byte) (int, error) {
if lr.Limit < 0 {
return 0, fmt.Errorf("read limit exceeded%.0w", errs.ErrSizeLimitExceeded)
}
if int64(len(p)) > lr.Limit+1 {
p = p[0 : lr.Limit+1]
}
n, err := lr.Reader.Read(p)
lr.Limit -= int64(n)
if lr.Limit < 0 {
return n, fmt.Errorf("read limit exceeded%.0w", errs.ErrSizeLimitExceeded)
}
return n, err
}
+257
View File
@@ -0,0 +1,257 @@
// Package pqueue implements a priority queue.
package pqueue
import (
"context"
"fmt"
"slices"
"sync"
)
type Queue[T any] struct {
mu sync.Mutex
max int
next func(queued, active []*T) int
active []*T
queued []*T
wait []*chan struct{}
}
// Opts is used to configure a new priority queue.
type Opts[T any] struct {
Max int // maximum concurrent entries, defaults to 1.
Next func(queued, active []*T) int // function to lookup index of next queued entry to release, defaults to oldest entry.
}
// New creates a new priority queue.
func New[T any](opts Opts[T]) *Queue[T] {
if opts.Max <= 0 {
opts.Max = 1
}
return &Queue[T]{
max: opts.Max,
next: opts.Next,
}
}
// Acquire adds a new entry to the queue and returns once it is ready.
// The returned function must be called when the queued job completes to release the next entry.
// If there is any error, the returned function will be nil.
func (q *Queue[T]) Acquire(ctx context.Context, e T) (func(), error) {
if q == nil {
return func() {}, nil
}
found, err := q.checkContext(ctx)
if err != nil {
return nil, err
}
if found {
return func() {}, nil
}
q.mu.Lock()
if len(q.active)+len(q.queued) < q.max {
q.active = append(q.active, &e)
q.mu.Unlock()
return q.releaseFn(&e), nil
}
// limit reached, add to queue and wait
w := make(chan struct{}, 1)
q.queued = append(q.queued, &e)
q.wait = append(q.wait, &w)
q.mu.Unlock()
// wait on both context and queue
select {
case <-ctx.Done():
// context abort, remove queued entry
q.mu.Lock()
if i := slices.Index(q.queued, &e); i >= 0 {
q.queued = slices.Delete(q.queued, i, i+1)
q.wait = slices.Delete(q.wait, i, i+1)
q.mu.Unlock()
return nil, ctx.Err()
}
q.mu.Unlock()
// queued entry found, assume race condition with context and entry being released, release next entry
q.release(&e)
return nil, ctx.Err()
case <-w:
return q.releaseFn(&e), nil
}
}
// TryAcquire attempts to add an entry on to the list of active entries.
// If the returned function is nil, the queue was not available.
// If the returned function is not nil, it must be called when the job is complete to release the next entry.
func (q *Queue[T]) TryAcquire(ctx context.Context, e T) (func(), error) {
if q == nil {
return func() {}, nil
}
found, err := q.checkContext(ctx)
if err != nil {
return nil, err
}
if found {
return func() {}, nil
}
q.mu.Lock()
defer q.mu.Unlock()
if len(q.active)+len(q.queued) < q.max {
q.active = append(q.active, &e)
return q.releaseFn(&e), nil
}
return nil, nil
}
// release next entry or noop.
func (q *Queue[T]) release(prev *T) {
q.mu.Lock()
defer q.mu.Unlock()
// remove prev entry from active list
if i := slices.Index(q.active, prev); i >= 0 {
q.active = slices.Delete(q.active, i, i+1)
}
// skip checks when at limit or nothing queued
if len(q.queued) == 0 {
if len(q.active) == 0 {
// free up slices if this was the last active entry
q.active = nil
q.queued = nil
q.wait = nil
}
return
}
if len(q.active) >= q.max {
return
}
i := 0
if q.next != nil && len(q.queued) > 1 {
i = q.next(q.queued, q.active)
// validate response
i = max(min(i, len(q.queued)-1), 0)
}
// release queued entry, move to active list, and remove from queued/wait lists
close(*q.wait[i])
q.active = append(q.active, q.queued[i])
q.queued = slices.Delete(q.queued, i, i+1)
q.wait = slices.Delete(q.wait, i, i+1)
}
// releaseFn is a convenience wrapper around [release].
func (q *Queue[T]) releaseFn(prev *T) func() {
return func() {
q.release(prev)
}
}
// TODO: is there a way to make a different context key for each generic type?
type ctxType int
var ctxKey ctxType
type valMulti[T any] struct {
qList []*Queue[T]
}
// AcquireMulti is used to simultaneously lock multiple queues without the risk of deadlock.
// The returned context needs to be used on calls to [Acquire] or [TryAcquire] which will immediately succeed since the resource is already acquired.
// Attempting to acquire other resources with [Acquire], [TryAcquire], or [AcquireMulti] using the returned context and will fail for being outside of the transaction.
// The returned function must be called to release the resources.
// The returned function is not thread safe, ensure no other simultaneous calls to [Acquire] or [TryAcquire] using the returned context have finished before it is called.
func AcquireMulti[T any](ctx context.Context, e T, qList ...*Queue[T]) (context.Context, func(), error) {
// verify context not already holding locks
qCtx := ctx.Value(ctxKey)
if qCtx != nil {
if qCtxVal, ok := qCtx.(*valMulti[T]); !ok || qCtxVal.qList != nil {
return ctx, nil, fmt.Errorf("context already used by another AcquireMulti request")
}
}
// delete nil entries
for i := len(qList) - 1; i >= 0; i-- {
if qList[i] == nil {
qList = slices.Delete(qList, i, i+1)
}
}
// empty/nil list is a noop
if len(qList) == 0 {
return ctx, func() {}, nil
}
// dedup entries from the list
for i := len(qList) - 2; i >= 0; i-- {
for j := len(qList) - 1; j > i; j-- {
if qList[i] == qList[j] {
qList[j] = qList[len(qList)-1]
qList = qList[:len(qList)-1]
}
}
}
// Loop through queues to acquire, waiting on the first, and attempting the remaining.
// If any of the remaining entries cannot be immediately acquired, reset and make it the new queue to wait on.
lockI := 0
doneList := make([]func(), len(qList))
for {
acquired := true
i := 0
done, err := qList[lockI].Acquire(ctx, e)
if err != nil {
return ctx, nil, err
}
doneList[lockI] = done
for i < len(qList) {
if i != lockI {
doneList[i], err = qList[i].TryAcquire(ctx, e)
if doneList[i] == nil || err != nil {
acquired = false
break
}
}
i++
}
if err == nil && acquired {
break
}
// cleanup on failed attempt
if lockI > i {
doneList[lockI]()
}
// track blocking index for a retry
lockI = i
for i > 0 {
i--
doneList[i]()
}
// abort on errors
if err != nil {
return ctx, nil, err
}
}
// success, update context
ctxVal := valMulti[T]{qList: qList}
newCtx := context.WithValue(ctx, ctxKey, &ctxVal)
cleanup := func() {
ctxVal.qList = nil
// dequeue in reverse order to minimize chance of another AcquireMulti being freed and immediately blocking on the next queue
for i := len(doneList) - 1; i >= 0; i-- {
doneList[i]()
}
}
return newCtx, cleanup, nil
}
func (q *Queue[T]) checkContext(ctx context.Context) (bool, error) {
qCtx := ctx.Value(ctxKey)
if qCtx == nil {
return false, nil
}
qCtxVal, ok := qCtx.(*valMulti[T])
if !ok {
return false, nil // another type is using the context, treat it as unset
}
if qCtxVal.qList == nil {
return false, nil
}
if slices.Contains(qCtxVal.qList, q) {
// instance already locked
return true, nil
}
return true, fmt.Errorf("cannot acquire new locks during a transaction")
}
+987
View File
@@ -0,0 +1,987 @@
// Package reghttp is used for HTTP requests to a registry
package reghttp
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
// crypto libraries included for go-digest
_ "crypto/sha256"
_ "crypto/sha512"
"github.com/regclient/regclient/config"
"github.com/regclient/regclient/internal/auth"
"github.com/regclient/regclient/internal/pqueue"
"github.com/regclient/regclient/internal/regnet"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/types"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/warning"
)
var (
defaultDelayInit, _ = time.ParseDuration("0.1s")
defaultDelayMax, _ = time.ParseDuration("30s")
warnRegexp = regexp.MustCompile(`^299\s+-\s+"([^"]+)"`)
)
const (
DefaultRetryLimit = 5 // number of times a request will be retried
backoffResetCount = 5 // number of successful requests needed to reduce the backoff
)
// Client is an HTTP client wrapper.
// It handles features like authentication, retries, backoff delays, TLS settings.
type Client struct {
httpClient *http.Client // upstream [http.Client], this is wrapped per repository for an auth handler on redirects
getConfigHost func(string) *config.Host // call-back to get the [config.Host] for a specific registry
host map[string]*clientHost // host specific settings, wrap access with a mutex lock
rootCAPool [][]byte // list of root CAs for configuring the http.Client transport
rootCADirs []string // list of directories for additional root CAs
retryLimit int // number of retries before failing a request, this applies to each host, and each request
delayInit time.Duration // how long to initially delay requests on a failure
delayMax time.Duration // maximum time to delay a request
slog *slog.Logger // logging for tracing and failures
userAgent string // user agent to specify in http request headers
mu sync.Mutex // mutex to prevent data races
}
type clientHost struct {
config *config.Host // config entry
httpClient *http.Client // modified http client for registry specific settings
userAgent string // user agent to specify in http request headers
slog *slog.Logger // logging for tracing and failures
auth map[string]*auth.Auth // map of auth handlers by repository
backoffCur int // current count of backoffs for this host
backoffLast time.Time // time the last request was released, this may be in the future if there is a queue, or zero if no delay is needed
backoffReset int // count of successful requests when a backoff is experienced, once [backoffResetCount] is reached, [backoffCur] is reduced by one and this is reset to 0
reqFreq time.Duration // how long between submitting requests for this host
reqNext time.Time // time to release the next request
throttle *pqueue.Queue[reqmeta.Data] // limit concurrent requests to the host
mu sync.Mutex // mutex to prevent data races
}
// Req is a request to send to a registry.
type Req struct {
MetaKind reqmeta.Kind // kind of request for the priority queue
Host string // registry name, hostname and mirrors will be looked up from host configuration
Method string // http method to call
DirectURL *url.URL // url to query, overrides repository, path, and query
Repository string // repository to scope the request
Path string // path of the request within a repository
Query url.Values // url query parameters
BodyLen int64 // length of body to send
BodyBytes []byte // bytes of the body, overridden by BodyFunc
BodyFunc func() (io.ReadCloser, error) // function to return a new body
Headers http.Header // headers to send in the request
NoPrefix bool // do not include the repository prefix
NoMirrors bool // do not send request to a mirror
ExpectLen int64 // expected size of the returned body
TransactLen int64 // size of an overall transaction for the priority queue
IgnoreErr bool // ignore http errors and do not trigger backoffs
}
// Resp is used to handle the result of a request.
type Resp struct {
ctx context.Context
client *Client
req *Req
resp *http.Response
mirror string
done bool
reader io.Reader
readCur, readMax int64
retryCount int
throttleDone func()
}
// Opts is used to configure client options.
type Opts func(*Client)
// NewClient returns a client for handling requests.
func NewClient(opts ...Opts) *Client {
c := Client{
httpClient: &http.Client{},
host: map[string]*clientHost{},
retryLimit: DefaultRetryLimit,
delayInit: defaultDelayInit,
delayMax: defaultDelayMax,
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
rootCAPool: [][]byte{},
rootCADirs: []string{},
}
for _, opt := range opts {
opt(&c)
}
return &c
}
// WithCerts adds certificates.
func WithCerts(certs [][]byte) Opts {
return func(c *Client) {
c.rootCAPool = append(c.rootCAPool, certs...)
}
}
// WithCertDirs adds directories to check for host specific certs.
func WithCertDirs(dirs []string) Opts {
return func(c *Client) {
c.rootCADirs = append(c.rootCADirs, dirs...)
}
}
// WithCertFiles adds certificates by filename.
func WithCertFiles(files []string) Opts {
return func(c *Client) {
for _, f := range files {
//#nosec G304 command is run by a user accessing their own files
cert, err := os.ReadFile(f)
if err != nil {
c.slog.Warn("Failed to read certificate",
slog.String("err", err.Error()),
slog.String("file", f))
} else {
c.rootCAPool = append(c.rootCAPool, cert)
}
}
}
}
// WithConfigHostFn adds the callback to request a [config.Host] struct.
// The function must normalize the hostname for Docker Hub support.
func WithConfigHostFn(gch func(string) *config.Host) Opts {
return func(c *Client) {
c.getConfigHost = gch
}
}
// WithDelay initial time to wait between retries (increased with exponential backoff).
func WithDelay(delayInit time.Duration, delayMax time.Duration) Opts {
return func(c *Client) {
if delayInit > 0 {
c.delayInit = delayInit
}
// delayMax must be at least delayInit, if 0 initialize to 30x delayInit
if delayMax > c.delayInit {
c.delayMax = delayMax
} else if delayMax > 0 {
c.delayMax = c.delayInit
} else {
c.delayMax = c.delayInit * 30
}
}
}
// WithHTTPClient uses a specific http client with retryable requests.
func WithHTTPClient(hc *http.Client) Opts {
return func(c *Client) {
c.httpClient = hc
}
}
// WithRetryLimit restricts the number of retries (defaults to 5).
func WithRetryLimit(rl int) Opts {
return func(c *Client) {
if rl > 0 {
c.retryLimit = rl
}
}
}
// WithLog injects a slog Logger configuration.
func WithLog(slog *slog.Logger) Opts {
return func(c *Client) {
c.slog = slog
}
}
// WithTransport uses a specific http transport with retryable requests.
func WithTransport(t *http.Transport) Opts {
return func(c *Client) {
c.httpClient = &http.Client{Transport: t}
}
}
// WithUserAgent sets a user agent header.
func WithUserAgent(ua string) Opts {
return func(c *Client) {
c.userAgent = ua
}
}
// Do runs a request, returning the response result.
func (c *Client) Do(ctx context.Context, req *Req) (*Resp, error) {
resp := &Resp{
ctx: ctx,
client: c,
req: req,
readCur: 0,
readMax: req.ExpectLen,
}
err := resp.next()
return resp, err
}
// next sends requests until a mirror responds or all requests fail.
func (resp *Resp) next() error {
var err error
c := resp.client
req := resp.req
// lookup reqHost entry
reqHost := c.getHost(req.Host)
// create sorted list of mirrors, based on backoffs, upstream, and priority
hosts := make([]*clientHost, 0, 1+len(reqHost.config.Mirrors))
if !req.NoMirrors {
for _, m := range reqHost.config.Mirrors {
hosts = append(hosts, c.getHost(m))
}
}
hosts = append(hosts, reqHost)
sort.Slice(hosts, sortHostsCmp(hosts, reqHost.config.Name))
// loop over requests to mirrors and retries
curHost := 0
for {
backoff := false
dropHost := false
retryHost := false
if len(hosts) == 0 {
if err != nil {
return err
}
return errs.ErrAllRequestsFailed
}
if curHost >= len(hosts) {
curHost = 0
}
h := hosts[curHost]
resp.mirror = h.config.Name
// there is an intentional extra retry in this check to allow for auth requests
if resp.retryCount > c.retryLimit {
return errs.ErrRetryLimitExceeded
}
resp.retryCount++
// check that context isn't canceled/done
ctxErr := resp.ctx.Err()
if ctxErr != nil {
return ctxErr
}
// wait for other concurrent requests to this host
throttleDone, throttleErr := h.throttle.Acquire(resp.ctx, reqmeta.Data{
Kind: req.MetaKind,
Size: req.BodyLen + req.ExpectLen + req.TransactLen,
})
if throttleErr != nil {
return throttleErr
}
// try each host in a closure to handle all the backoff/dropHost from one place
loopErr := func() error {
var err error
if req.Method == "HEAD" && h.config.APIOpts != nil {
var disableHead bool
disableHead, err = strconv.ParseBool(h.config.APIOpts["disableHead"])
if err == nil && disableHead {
dropHost = true
return fmt.Errorf("head requests disabled for host \"%s\": %w", h.config.Name, errs.ErrUnsupportedAPI)
}
}
// build the url
var u url.URL
if req.DirectURL != nil {
u = *req.DirectURL
} else {
u = url.URL{
Host: h.config.Hostname,
Scheme: "https",
}
path := strings.Builder{}
path.WriteString("/v2")
if h.config.PathPrefix != "" && !req.NoPrefix {
path.WriteString("/" + h.config.PathPrefix)
}
if req.Repository != "" {
path.WriteString("/" + req.Repository)
}
path.WriteString("/" + req.Path)
u.Path = path.String()
if h.config.TLS == config.TLSDisabled {
u.Scheme = "http"
}
query := url.Values{}
if req.Query != nil {
query = req.Query
}
if h.config.Hostname != reqHost.config.Hostname {
query.Set("ns", reqHost.config.Hostname)
}
u.RawQuery = query.Encode()
}
// close previous response
if resp.resp != nil && resp.resp.Body != nil {
_ = resp.resp.Body.Close()
}
// delay for backoff if needed
bu := resp.backoffGet()
if !bu.IsZero() && bu.After(time.Now()) {
sleepTime := time.Until(bu)
c.slog.Debug("Sleeping for backoff",
slog.String("Host", h.config.Name),
slog.Duration("Duration", sleepTime))
select {
case <-resp.ctx.Done():
return errs.ErrCanceled
case <-time.After(sleepTime):
}
}
var httpReq *http.Request
httpReq, err = http.NewRequestWithContext(resp.ctx, req.Method, u.String(), nil)
if err != nil {
dropHost = true
return err
}
if req.BodyFunc != nil {
body, err := req.BodyFunc()
if err != nil {
dropHost = true
return err
}
httpReq.Body = body
httpReq.GetBody = req.BodyFunc
httpReq.ContentLength = req.BodyLen
} else if len(req.BodyBytes) > 0 {
body := io.NopCloser(bytes.NewReader(req.BodyBytes))
httpReq.Body = body
httpReq.GetBody = func() (io.ReadCloser, error) { return body, nil }
httpReq.ContentLength = req.BodyLen
}
if len(req.Headers) > 0 {
httpReq.Header = req.Headers.Clone()
}
if c.userAgent != "" && httpReq.Header.Get("User-Agent") == "" {
httpReq.Header.Add("User-Agent", c.userAgent)
}
if resp.readCur > 0 && resp.readMax > 0 {
if req.Headers.Get("Range") == "" {
httpReq.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", resp.readCur, resp.readMax))
} else {
// TODO: support Seek within a range request
dropHost = true
return fmt.Errorf("unable to resume a connection within a range request")
}
}
hAuth := h.getAuth(req.Repository)
if hAuth != nil {
// include docker generated scope to emulate docker clients
if req.Repository != "" {
scope := "repository:" + req.Repository + ":pull"
if req.Method != "HEAD" && req.Method != "GET" {
scope = scope + ",push"
}
_ = hAuth.AddScope(h.config.Hostname, scope)
}
// add auth headers
err = hAuth.UpdateRequest(httpReq)
// abort on auth errors, but only after the first request has been attempted
if err != nil && resp.resp != nil {
if errors.Is(err, errs.ErrHTTPUnauthorized) {
dropHost = true
} else {
backoff = true
}
return err
}
}
// delay for the rate limit
if h.reqFreq > 0 {
sleep := time.Duration(0)
h.mu.Lock()
if time.Now().Before(h.reqNext) {
sleep = time.Until(h.reqNext)
h.reqNext = h.reqNext.Add(h.reqFreq)
} else {
h.reqNext = time.Now().Add(h.reqFreq)
}
h.mu.Unlock()
if sleep > 0 {
time.Sleep(sleep)
}
}
// send request
hc := h.getHTTPClient(req.Repository)
//#nosec G704 inputs are user controlled and sanitized
resp.resp, err = hc.Do(httpReq)
if err != nil {
c.slog.Debug("Request failed",
slog.String("URL", u.String()),
slog.String("err", err.Error()))
backoff = true
return err
}
statusCode := resp.resp.StatusCode
if statusCode < 200 || statusCode >= 300 {
switch statusCode {
case http.StatusUnauthorized:
// if auth can be done, retry same host without delay, otherwise drop/backoff
if hAuth != nil {
err = hAuth.HandleResponse(resp.resp)
} else {
err = fmt.Errorf("authentication handler unavailable")
}
if err != nil {
if errors.Is(err, errs.ErrEmptyChallenge) || errors.Is(err, errs.ErrNoNewChallenge) || errors.Is(err, errs.ErrHTTPUnauthorized) {
c.slog.Debug("Failed to handle auth request",
slog.String("URL", u.String()),
slog.String("Err", err.Error()))
} else {
c.slog.Warn("Failed to handle auth request",
slog.String("URL", u.String()),
slog.String("Err", err.Error()))
}
dropHost = true
} else {
err = fmt.Errorf("authentication required")
retryHost = true
}
return err
case http.StatusNotFound:
// if not found, drop mirror for this req, but other requests don't need backoff
dropHost = true
case http.StatusRequestedRangeNotSatisfiable:
// if range request error (blob push), drop mirror for this req, but other requests don't need backoff
dropHost = true
case http.StatusTooManyRequests, http.StatusRequestTimeout, http.StatusGatewayTimeout, http.StatusBadGateway, http.StatusInternalServerError:
// server is likely overloaded, backoff but still retry
backoff = true
default:
// all other errors indicate a bigger issue, don't retry and set backoff
backoff = true
dropHost = true
}
errHTTP := HTTPError(resp.resp.StatusCode)
errBody, _ := io.ReadAll(resp.resp.Body)
_ = resp.resp.Body.Close()
return fmt.Errorf("request failed: %w: %s", errHTTP, errBody)
}
resp.reader = resp.resp.Body
resp.done = false
// set variables from headers if found
clHeader := resp.resp.Header.Get("Content-Length")
if resp.readCur == 0 && clHeader != "" {
cl, parseErr := strconv.ParseInt(clHeader, 10, 64)
if parseErr != nil {
c.slog.Debug("failed to parse content-length header",
slog.String("err", parseErr.Error()),
slog.String("header", clHeader))
} else if resp.readMax > 0 {
if resp.readMax != cl {
return fmt.Errorf("unexpected content-length, expected %d, received %d", resp.readMax, cl)
}
} else {
resp.readMax = cl
}
}
// verify Content-Range header when range request used, fail if missing
if httpReq.Header.Get("Range") != "" && resp.resp.Header.Get("Content-Range") == "" {
dropHost = true
_ = resp.resp.Body.Close()
return fmt.Errorf("range request not supported by server")
}
return nil
}()
// return on success
if loopErr == nil {
resp.throttleDone = throttleDone
return nil
}
// backoff, dropHost, and/or go to next host in the list
if backoff {
if req.IgnoreErr {
// don't set a backoff, immediately drop the host when errors ignored
dropHost = true
} else {
boErr := resp.backoffSet()
if boErr != nil {
// reached backoff limit
dropHost = true
}
}
}
throttleDone()
// when error does not allow retries, abort with the last known err value
if err != nil && errors.Is(loopErr, errs.ErrNotRetryable) {
return err
}
err = loopErr
if dropHost {
hosts = slices.Delete(hosts, curHost, curHost+1)
} else if !retryHost {
curHost++
}
}
}
// GetThrottle returns the current [pqueue.Queue] for a host used to throttle connections.
// This can be used to acquire multiple throttles before performing a request across multiple hosts.
func (c *Client) GetThrottle(host string) *pqueue.Queue[reqmeta.Data] {
ch := c.getHost(host)
return ch.throttle
}
// HTTPResponse returns the [http.Response] from the last request.
func (resp *Resp) HTTPResponse() *http.Response {
return resp.resp
}
// Read provides a retryable read from the body of the response.
func (resp *Resp) Read(b []byte) (int, error) {
if resp.done || resp.reader == nil {
return 0, io.EOF
}
if resp.resp == nil {
return 0, errs.ErrNotFound
}
// perform the read
i, err := resp.reader.Read(b)
resp.readCur += int64(i)
if err == io.EOF || err == io.ErrUnexpectedEOF {
if resp.resp.Request.Method == "HEAD" || resp.readCur >= resp.readMax {
resp.backoffReset()
resp.done = true
} else {
// short read, retry?
resp.client.slog.Debug("EOF before reading all content, retrying",
slog.Int64("curRead", resp.readCur),
slog.Int64("contentLen", resp.readMax))
// retry
respErr := resp.backoffSet()
if respErr == nil {
respErr = resp.next()
}
// unrecoverable EOF
if respErr != nil {
resp.client.slog.Warn("Failed to recover from short read",
slog.String("err", respErr.Error()))
resp.done = true
return i, err
}
// retry successful, no EOF
return i, nil
}
}
if err == nil {
return i, nil
}
return i, err
}
// Close frees up resources from the request.
func (resp *Resp) Close() error {
if resp.throttleDone != nil {
resp.throttleDone()
resp.throttleDone = nil
}
if resp.resp == nil {
return errs.ErrNotFound
}
if !resp.done {
resp.backoffReset()
}
resp.done = true
return resp.resp.Body.Close()
}
// Seek provides a limited ability seek within the request response.
func (resp *Resp) Seek(offset int64, whence int) (int64, error) {
newOffset := resp.readCur
switch whence {
case io.SeekStart:
newOffset = offset
case io.SeekCurrent:
newOffset += offset
case io.SeekEnd:
if resp.readMax <= 0 {
return resp.readCur, fmt.Errorf("seek from end is not supported")
} else if resp.readMax+offset < 0 {
return resp.readCur, fmt.Errorf("seek past beginning of the file is not supported")
}
newOffset = resp.readMax + offset
default:
return resp.readCur, fmt.Errorf("unknown value of whence: %d", whence)
}
if newOffset != resp.readCur {
resp.readCur = newOffset
// rerun the request to restart
resp.retryCount-- // do not count a seek as a retry
err := resp.next()
if err != nil {
return resp.readCur, err
}
}
return resp.readCur, nil
}
func (resp *Resp) backoffGet() time.Time {
c := resp.client
ch := c.getHost(resp.mirror)
ch.mu.Lock()
defer ch.mu.Unlock()
if ch.backoffCur > 0 {
delay := c.delayInit << ch.backoffCur
delay = min(delay, c.delayMax)
next := ch.backoffLast.Add(delay)
now := time.Now()
if now.After(next) {
next = now
}
ch.backoffLast = next
return next
}
// reset a stale "retry-after" time
if !ch.backoffLast.IsZero() && ch.backoffLast.Before(time.Now()) {
ch.backoffLast = time.Time{}
}
return ch.backoffLast
}
func (resp *Resp) backoffSet() error {
c := resp.client
ch := c.getHost(resp.mirror)
ch.mu.Lock()
defer ch.mu.Unlock()
// check rate limit header and use that directly if possible
if resp.resp != nil && resp.resp.Header.Get("Retry-After") != "" {
ras := resp.resp.Header.Get("Retry-After")
ra, _ := time.ParseDuration(ras + "s")
if ra > 0 {
next := time.Now().Add(ra)
if ch.backoffLast.Before(next) {
ch.backoffLast = next
}
return nil
}
}
// Else track the number of backoffs and fail when the limit is exceeded.
// New requests always get at least one try, but fail fast if the server has been throwing errors.
ch.backoffCur++
if ch.backoffLast.IsZero() {
ch.backoffLast = time.Now()
}
if ch.backoffCur >= c.retryLimit {
return fmt.Errorf("%w: backoffs %d", errs.ErrBackoffLimit, ch.backoffCur)
}
return nil
}
func (resp *Resp) backoffReset() {
c := resp.client
ch := c.getHost(resp.mirror)
ch.mu.Lock()
defer ch.mu.Unlock()
if ch.backoffCur > 0 {
ch.backoffReset++
// If enough successful requests are seen, lower the backoffCur count.
// This requires multiple successful requests of a flaky server, but quickly drops when above the retry limit.
if ch.backoffReset > backoffResetCount || ch.backoffCur > c.retryLimit {
ch.backoffReset = 0
ch.backoffCur--
if ch.backoffCur == 0 {
// reset the last time to the zero value
ch.backoffLast = time.Time{}
}
}
}
}
// getHost looks up or creates a clientHost for a given registry.
func (c *Client) getHost(host string) *clientHost {
c.mu.Lock()
defer c.mu.Unlock()
if h, ok := c.host[host]; ok {
return h
}
var conf *config.Host
if c.getConfigHost != nil {
conf = c.getConfigHost(host)
} else {
conf = config.HostNewName(host)
}
if conf.Name != host {
if h, ok := c.host[conf.Name]; ok {
return h
}
}
h := &clientHost{
config: conf,
userAgent: c.userAgent,
slog: c.slog,
auth: map[string]*auth.Auth{},
}
if h.config.ReqPerSec > 0 {
h.reqFreq = time.Duration(float64(time.Second) / h.config.ReqPerSec)
}
if h.config.ReqConcurrent > 0 {
h.throttle = pqueue.New(pqueue.Opts[reqmeta.Data]{Max: int(h.config.ReqConcurrent), Next: reqmeta.DataNext})
}
// copy the http client and configure registry specific settings
hc := *c.httpClient
h.httpClient = &hc
if h.httpClient.Transport == nil {
h.httpClient.Transport = http.DefaultTransport.(*http.Transport).Clone()
}
// configure transport for insecure requests and root certs
if h.config.TLS == config.TLSInsecure || len(c.rootCAPool) > 0 || len(c.rootCADirs) > 0 || h.config.RegCert != "" || (h.config.ClientCert != "" && h.config.ClientKey != "") {
t, ok := h.httpClient.Transport.(*http.Transport)
if ok {
var tlsc *tls.Config
if t.TLSClientConfig != nil {
tlsc = t.TLSClientConfig.Clone()
} else {
//#nosec G402 the default TLS 1.2 minimum version is allowed to support older registries
tlsc = &tls.Config{}
}
if h.config.TLS == config.TLSInsecure {
tlsc.InsecureSkipVerify = true
} else {
rootPool, err := makeRootPool(c.rootCAPool, c.rootCADirs, h.config.Hostname, h.config.RegCert)
if err != nil {
c.slog.Warn("failed to setup CA pool",
slog.String("err", err.Error()))
} else {
tlsc.RootCAs = rootPool
}
}
if h.config.ClientCert != "" && h.config.ClientKey != "" {
cert, err := tls.X509KeyPair([]byte(h.config.ClientCert), []byte(h.config.ClientKey))
if err != nil {
c.slog.Warn("failed to configure client certs",
slog.String("err", err.Error()))
} else {
tlsc.Certificates = []tls.Certificate{cert}
}
}
t.TLSClientConfig = tlsc
h.httpClient.Transport = t
}
}
// wrap the transport for logging and to handle warning headers
h.httpClient.Transport = &wrapTransport{c: c, orig: h.httpClient.Transport}
c.host[conf.Name] = h
if conf.Name != host {
// save another reference for faster lookups
c.host[host] = h
}
return h
}
// getHTTPClient returns a client specific to the repo being queried.
// Repository specific authentication needs a dedicated CheckRedirect handler.
func (ch *clientHost) getHTTPClient(repo string) *http.Client {
hc := *ch.httpClient
hc.CheckRedirect = ch.checkRedirect(repo, hc.CheckRedirect)
return &hc
}
// checkRedirect wraps http.CheckRedirect to inject auth headers to specific hosts in the redirect chain
func (ch *clientHost) checkRedirect(repo string, orig func(req *http.Request, via []*http.Request) error) func(req *http.Request, via []*http.Request) error {
return func(req *http.Request, via []*http.Request) error {
// fail on too many redirects
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
// verify redirect is allowed
last := via[len(via)-1]
if err := regnet.AllowRedirect(*last.URL, *req.URL); err != nil {
return err
}
// add auth headers if appropriate for the target host
hAuth := ch.getAuth(repo)
err := hAuth.UpdateRequest(req)
if err != nil {
return err
}
// wrap original redirect check
if orig != nil {
return orig(req, via)
}
return nil
}
}
// getAuth returns an auth, which may be repository specific.
func (ch *clientHost) getAuth(repo string) *auth.Auth {
ch.mu.Lock()
defer ch.mu.Unlock()
if !ch.config.RepoAuth {
repo = "" // without RepoAuth, unset the provided repo
}
if _, ok := ch.auth[repo]; !ok {
ch.auth[repo] = auth.NewAuth(
auth.WithLog(ch.slog),
auth.WithHTTPClient(ch.httpClient),
auth.WithCreds(ch.AuthCreds()),
auth.WithClientID(ch.userAgent),
)
}
return ch.auth[repo]
}
func (ch *clientHost) AuthCreds() func(h string) auth.Cred {
if ch == nil || ch.config == nil {
return auth.DefaultCredsFn
}
return func(h string) auth.Cred {
// only return credentials to challenges from the registry server, not to any redirects
if h == ch.config.Hostname {
hCred := ch.config.GetCred()
return auth.Cred{User: hCred.User, Password: hCred.Password, Token: hCred.Token}
} else {
return auth.Cred{}
}
}
}
type wrapTransport struct {
c *Client
orig http.RoundTripper
}
func (wt *wrapTransport) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := wt.orig.RoundTrip(req)
// copy headers to censor auth field
reqHead := req.Header.Clone()
if reqHead.Get("Authorization") != "" {
reqHead.Set("Authorization", "[censored]")
}
if err != nil {
wt.c.slog.Debug("reg http request",
slog.String("req-method", req.Method),
slog.String("req-url", req.URL.String()),
slog.Any("req-headers", reqHead),
slog.String("err", err.Error()))
} else {
// extract any warnings
for _, wh := range resp.Header.Values("Warning") {
if match := warnRegexp.FindStringSubmatch(wh); len(match) == 2 {
// TODO(bmitch): pass other fields (registry hostname) with structured logging
warning.Handle(req.Context(), wt.c.slog, match[1])
}
}
wt.c.slog.Log(req.Context(), types.LevelTrace, "reg http request",
slog.String("req-method", req.Method),
slog.String("req-url", req.URL.String()),
slog.Any("req-headers", reqHead),
slog.String("resp-status", resp.Status),
slog.Any("resp-headers", resp.Header))
}
return resp, err
}
// HTTPError returns an error based on the status code.
func HTTPError(statusCode int) error {
switch statusCode {
case 401:
return fmt.Errorf("%w [http %d]", errs.ErrHTTPUnauthorized, statusCode)
case 403:
return fmt.Errorf("%w [http %d]", errs.ErrHTTPUnauthorized, statusCode)
case 404:
return fmt.Errorf("%w [http %d]", errs.ErrNotFound, statusCode)
case 429:
return fmt.Errorf("%w [http %d]", errs.ErrHTTPRateLimit, statusCode)
default:
return fmt.Errorf("%w: %s [http %d]", errs.ErrHTTPStatus, http.StatusText(statusCode), statusCode)
}
}
func makeRootPool(rootCAPool [][]byte, rootCADirs []string, hostname string, hostcert string) (*x509.CertPool, error) {
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
for _, ca := range rootCAPool {
if ok := pool.AppendCertsFromPEM(ca); !ok {
return nil, fmt.Errorf("failed to load ca: %s", ca)
}
}
for _, dir := range rootCADirs {
hostDir := filepath.Join(dir, hostname)
files, err := os.ReadDir(hostDir)
if err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to read directory %s: %w", hostDir, err)
}
continue
}
for _, f := range files {
if f.IsDir() {
continue
}
if strings.HasSuffix(f.Name(), ".crt") {
f := filepath.Join(hostDir, f.Name())
//#nosec G304 file from a known directory and extension read by the user running the command on their own host
cert, err := os.ReadFile(f)
if err != nil {
return nil, fmt.Errorf("failed to read %s: %w", f, err)
}
if ok := pool.AppendCertsFromPEM(cert); !ok {
return nil, fmt.Errorf("failed to import cert from %s", f)
}
}
}
}
if hostcert != "" {
if ok := pool.AppendCertsFromPEM([]byte(hostcert)); !ok {
// try to parse the certificate and generate a useful error
block, _ := pem.Decode([]byte(hostcert))
if block == nil {
err = fmt.Errorf("pem.Decode is nil")
} else {
_, err = x509.ParseCertificate(block.Bytes)
}
return nil, fmt.Errorf("failed to load host specific ca (registry: %s): %w: %s", hostname, err, hostcert)
}
}
return pool, nil
}
// sortHostCmp to sort host list of mirrors.
func sortHostsCmp(hosts []*clientHost, upstream string) func(i, j int) bool {
now := time.Now()
// sort by backoff first, then priority decending, then upstream name last
return func(i, j int) bool {
if now.Before(hosts[i].backoffLast) || now.Before(hosts[j].backoffLast) {
return hosts[i].backoffLast.Before(hosts[j].backoffLast)
}
if hosts[i].config.Priority != hosts[j].config.Priority {
return hosts[i].config.Priority < hosts[j].config.Priority
}
return hosts[i].config.Name != upstream
}
}
+48
View File
@@ -0,0 +1,48 @@
// Package regnet contains networking helper functions for interacting with registries.
package regnet
import (
"fmt"
"net"
"net/url"
"github.com/regclient/regclient/types/errs"
)
func AllowRedirect(src, dest url.URL) error {
if src.Scheme == "https" && dest.Scheme != "https" {
return fmt.Errorf("redirect from an https to non-https server is not allowed (%s)%.0w", dest.String(), errs.ErrHTTPRedirectRefused)
}
if !IsLocal(src.Host) && IsLocal(dest.Host) {
return fmt.Errorf("redirect to a local domain is not allowed (%s)%.0w", dest.String(), errs.ErrHTTPRedirectRefused)
}
return nil
}
func IsLocal(hostPort string) bool {
// strip trailing port
host, _, err := net.SplitHostPort(hostPort)
if err != nil {
host = hostPort
}
// parse IP
ip := net.ParseIP(host)
if ip != nil {
return isIPLocal(ip)
}
// else resolve the hostname and then check each IP
ips, err := net.LookupIP(host)
if err != nil {
return false
}
for _, ip := range ips {
if ip != nil && isIPLocal(ip) {
return true
}
}
return false
}
func isIPLocal(ip net.IP) bool {
return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
}
@@ -0,0 +1,3 @@
// Package reproducible is used for reproducibility.
// See <https://reproducible-builds.org/docs/> for more information of the issues being solved.
package reproducible
+35
View File
@@ -0,0 +1,35 @@
package reproducible
import (
"errors"
"os"
"strconv"
"time"
)
const EpocEnv = "SOURCE_DATE_EPOC"
var errInvalidEpoc = errors.New("invalid epoc var")
// TimeNow returns the current time or SOURCE_DATE_EPOC if that is set.
func TimeNow() time.Time {
now, err := TimeEpocEnv()
if err == nil {
return now
}
return time.Now().UTC()
}
// TimeEpocEnv returns the time parsed by SOURCE_DATE_EPOC.
// This should be used to override any timestamps that should be reproducible.
func TimeEpocEnv() (time.Time, error) {
sec := os.Getenv(EpocEnv)
if sec == "" {
return time.Time{}, errInvalidEpoc
}
secI, err := strconv.ParseInt(sec, 10, 64)
if err != nil {
return time.Time{}, err
}
return time.Unix(secI, 0).UTC(), nil
}
+88
View File
@@ -0,0 +1,88 @@
// Package reqmeta provides metadata on requests for prioritizing with a pqueue.
package reqmeta
type Data struct {
Kind Kind
Size int64
}
type Kind int
const (
Unknown Kind = iota
Head
Manifest
Query
Blob
)
const (
smallLimit = 4194304 // 4MiB
largePct = 0.9 // anything above 90% of largest queued entry size is large
)
func DataNext(queued, active []*Data) int {
if len(queued) == 0 {
return -1
}
// After removing one small entry, split remaining requests 50/50 between large and old (truncated int division always rounds down).
// If len active = 2, this function returns the 3rd entry (+1), minus 1 for the small, divide by 2 to split with old = goal of 1.
largeGoal := len(active) / 2
largeI := 0
var largeSize int64
if largeGoal > 0 {
// find the largest queued blob requests
for i, cur := range queued {
if cur.Kind == Blob && cur.Size > largeSize {
largeI = i
largeSize = cur.Size
}
}
}
largeCutoff := int64(float64(largeSize) * 0.9)
// count active requests by type
small := 0
large := 0
old := 0
for _, cur := range active {
if cur.Kind != Blob && cur.Size <= smallLimit {
small++
} else if cur.Kind == Blob && largeSize > 0 && cur.Size >= largeCutoff {
large++
} else {
old++
}
}
// if there is at least one active, and none are small, return the best small entry if available.
if len(active) > 0 && small == 0 {
var sizeI int64
bestI := -1
kindI := Unknown
for i, cur := range queued {
// the small search skips blobs and large requests
if cur.Kind == Blob || cur.Size > smallLimit {
continue
}
// the best small entry is the:
// - first one found if no other matches
// - one with a better Kind (Head > Manifest > Query)
// - one with the same kind but smaller request
if bestI < 0 ||
(cur.Kind != Unknown && (kindI == Unknown || cur.Kind < kindI)) ||
(cur.Kind == kindI && cur.Size > 0 && (cur.Size < sizeI || sizeI <= 0)) {
bestI = i
kindI = cur.Kind
sizeI = cur.Size
}
}
if bestI >= 0 {
return bestI
}
}
// Prefer the biggest of these blobs to minimize the size of the last running blob.
if largeGoal > 0 && large < largeGoal && largeSize > 0 {
return largeI
}
// enough small and large, or none available, so return the oldest queued entry to avoid starvation.
return 0
}
+125
View File
@@ -0,0 +1,125 @@
//go:build !wasm
// Package sloghandle provides a transition handler for migrating from logrus to slog.
package sloghandle
import (
"context"
"log/slog"
"strings"
"github.com/sirupsen/logrus"
"github.com/regclient/regclient/types"
)
func Logrus(logger *logrus.Logger) *logrusHandler {
return &logrusHandler{
logger: logger,
}
}
type logrusHandler struct {
logger *logrus.Logger
attrs []slog.Attr
groups []string
}
func (h *logrusHandler) Enabled(_ context.Context, level slog.Level) bool {
ll := h.logger.GetLevel()
if curLevel, ok := logrusToSlog[ll]; ok {
return level >= curLevel
}
return true
}
func (h *logrusHandler) Handle(ctx context.Context, r slog.Record) error {
log := logrus.NewEntry(h.logger).WithContext(ctx)
if !r.Time.IsZero() {
log = log.WithTime(r.Time)
}
fields := logrus.Fields{}
for _, a := range h.attrs {
if a.Key != "" {
fields[a.Key] = a.Value
}
}
r.Attrs(func(a slog.Attr) bool {
if a.Key != "" {
fields[a.Key] = a.Value
}
return true
})
if len(fields) > 0 {
log = log.WithFields(fields)
}
log.Log(slogToLogrus(r.Level), r.Message)
return nil
}
func (h *logrusHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
ret := h.clone()
prefix := ""
if len(h.groups) > 0 {
prefix = strings.Join(h.groups, ":") + ":"
}
for _, a := range attrs {
if a.Key == "" {
continue
}
ret.attrs = append(ret.attrs, slog.Attr{
Key: prefix + a.Key,
Value: a.Value,
})
}
return ret
}
func (h *logrusHandler) WithGroup(name string) slog.Handler {
if name == "" {
return h
}
ret := h.clone()
ret.groups = append(ret.groups, name)
return ret
}
func (h *logrusHandler) clone() *logrusHandler {
attrs := make([]slog.Attr, len(h.attrs))
copy(attrs, h.attrs)
groups := make([]string, len(h.groups))
copy(groups, h.groups)
return &logrusHandler{
logger: h.logger,
attrs: attrs,
groups: groups,
}
}
var logrusToSlog = map[logrus.Level]slog.Level{
logrus.TraceLevel: types.LevelTrace,
logrus.DebugLevel: slog.LevelDebug,
logrus.InfoLevel: slog.LevelInfo,
logrus.WarnLevel: slog.LevelWarn,
logrus.ErrorLevel: slog.LevelError,
logrus.FatalLevel: slog.LevelError + 4,
logrus.PanicLevel: slog.LevelError + 8,
}
func slogToLogrus(level slog.Level) logrus.Level {
if level <= types.LevelTrace {
return logrus.TraceLevel
} else if level <= slog.LevelDebug {
return logrus.DebugLevel
} else if level <= slog.LevelInfo {
return logrus.InfoLevel
} else if level <= slog.LevelWarn {
return logrus.WarnLevel
} else if level <= slog.LevelError {
return logrus.ErrorLevel
} else if level <= slog.LevelError+4 {
return logrus.FatalLevel
} else {
return logrus.PanicLevel
}
}
+91
View File
@@ -0,0 +1,91 @@
// Package strparse is used to parse strings
package strparse
import (
"fmt"
"github.com/regclient/regclient/types/errs"
)
// SplitCSKV splits a comma separated key=value list into a map
func SplitCSKV(s string) (map[string]string, error) {
state := "key"
key := ""
val := ""
result := map[string]string{}
procKV := func() {
if key != "" {
result[key] = val
}
state = "key"
key = ""
val = ""
}
for _, c := range s {
switch state {
case "key":
switch c {
case '"':
state = "keyQuote"
case '\\':
state = "keyEscape"
case '=':
state = "val"
case ',':
procKV()
default:
key = key + string(c)
}
case "keyQuote":
switch c {
case '"':
state = "key"
case '\\':
state = "keyEscapeQuote"
default:
key = key + string(c)
}
case "keyEscape":
key = key + string(c)
state = "key"
case "keyEscapeQuote":
key = key + string(c)
state = "keyQuote"
case "val":
switch c {
case '"':
state = "valQuote"
case ',':
procKV()
case '\\':
state = "valEscape"
default:
val = val + string(c)
}
case "valQuote":
switch c {
case '"':
state = "val"
case '\\':
state = "valEscapeQuote"
default:
val = val + string(c)
}
case "valEscape":
val = val + string(c)
state = "val"
case "valEscapeQuote":
val = val + string(c)
state = "valQuote"
default:
return nil, fmt.Errorf("unhandled state: %s", state)
}
}
switch state {
case "val", "key":
procKV()
default:
return nil, fmt.Errorf("string parsing failed, end state: %s%.0w", state, errs.ErrParsingFailed)
}
return result, nil
}
+41
View File
@@ -0,0 +1,41 @@
// Package timejson extends time methods with marshal/unmarshal for json
package timejson
import (
"encoding/json"
"errors"
"time"
)
var errInvalid = errors.New("invalid duration")
// Duration is an alias to time.Duration
// Implementation taken from https://stackoverflow.com/questions/48050945/how-to-unmarshal-json-into-durations
type Duration time.Duration
// MarshalJSON converts a duration to json
func (d Duration) MarshalJSON() ([]byte, error) {
return json.Marshal(time.Duration(d).String())
}
// UnmarshalJSON converts json to a duration
func (d *Duration) UnmarshalJSON(b []byte) error {
var v any
if err := json.Unmarshal(b, &v); err != nil {
return err
}
switch value := v.(type) {
case float64:
*d = Duration(time.Duration(value))
return nil
case string:
timeDur, err := time.ParseDuration(value)
if err != nil {
return err
}
*d = Duration(timeDur)
return nil
default:
return errInvalid
}
}
+59
View File
@@ -0,0 +1,59 @@
// Package units is taken from https://github.com/docker/go-units
package units
// Copyright 2015 Docker, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// https://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import (
"fmt"
)
var (
decimapAbbrs = []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
binaryAbbrs = []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"}
)
func getSizeAndUnit(size float64, base float64, unitList []string) (float64, string) {
i := 0
unitsLimit := len(unitList) - 1
for size >= base && i < unitsLimit {
size = size / base
i++
}
return size, unitList[i]
}
// CustomSize returns a human-readable approximation of a size using custom format.
func CustomSize(format string, size float64, base float64, unitList []string) string {
size, unit := getSizeAndUnit(size, base, unitList)
return fmt.Sprintf(format, size, unit)
}
// HumanSizeWithPrecision allows the size to be in any precision.
func HumanSizeWithPrecision(size float64, width, precision int) string {
size, unit := getSizeAndUnit(size, 1000.0, decimapAbbrs)
return fmt.Sprintf("%*.*f%s", width, precision, size, unit)
}
// HumanSize returns a human-readable approximation of a size
// with a width of 5 (eg. "2.746MB", "796.0KB").
func HumanSize(size float64) string {
return HumanSizeWithPrecision(size, 5, 3)
}
// BytesSize returns a human-readable size in bytes, kibibytes,
// mebibytes, gibibytes, or tebibytes (eg. "44.2kiB", "17.6MiB").
func BytesSize(size float64) string {
return CustomSize("%5.3f%s", size, 1024.0, binaryAbbrs)
}
+32
View File
@@ -0,0 +1,32 @@
// Package version returns details on the Go and Git repo used in the build
package version
import (
"bytes"
"fmt"
"text/tabwriter"
)
const (
stateClean = "clean"
stateDirty = "dirty"
unknown = "unknown"
biVCSDate = "vcs.time"
biVCSCommit = "vcs.revision"
biVCSModified = "vcs.modified"
)
func (i Info) MarshalPretty() ([]byte, error) {
buf := &bytes.Buffer{}
tw := tabwriter.NewWriter(buf, 0, 0, 1, ' ', 0)
fmt.Fprintf(tw, "VCSTag:\t%s\n", i.VCSTag)
fmt.Fprintf(tw, "VCSRef:\t%s\n", i.VCSRef)
fmt.Fprintf(tw, "VCSCommit:\t%s\n", i.VCSCommit)
fmt.Fprintf(tw, "VCSState:\t%s\n", i.VCSState)
fmt.Fprintf(tw, "VCSDate:\t%s\n", i.VCSDate)
fmt.Fprintf(tw, "Platform:\t%s\n", i.Platform)
fmt.Fprintf(tw, "GoVer:\t%s\n", i.GoVer)
fmt.Fprintf(tw, "GoCompiler:\t%s\n", i.GoCompiler)
err := tw.Flush()
return buf.Bytes(), err
}
@@ -0,0 +1,74 @@
//go:build go1.18
package version
import (
"fmt"
"runtime"
"runtime/debug"
"time"
)
var vcsTag = ""
type Info struct {
GoVer string `json:"goVersion"` // go version
GoCompiler string `json:"goCompiler"` // go compiler
Platform string `json:"platform"` // os/arch
VCSCommit string `json:"vcsCommit"` // commit sha
VCSDate string `json:"vcsDate"` // commit date in RFC3339 format
VCSRef string `json:"vcsRef"` // commit sha + dirty if state is not clean
VCSState string `json:"vcsState"` // clean or dirty
VCSTag string `json:"vcsTag"` // tag is not available from Go
Debug *debug.BuildInfo `json:"debug,omitempty"` // build info debugging data
}
func GetInfo() Info {
i := Info{
GoVer: unknown,
Platform: unknown,
VCSCommit: unknown,
VCSDate: unknown,
VCSRef: unknown,
VCSState: unknown,
VCSTag: vcsTag,
}
i.GoVer = runtime.Version()
i.GoCompiler = runtime.Compiler
i.Platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH)
if bi, ok := debug.ReadBuildInfo(); ok && bi != nil {
i.Debug = bi
if i.VCSTag == "" {
i.VCSTag = bi.Main.Version
}
date := biSetting(bi, biVCSDate)
if t, err := time.Parse(time.RFC3339, date); err == nil {
i.VCSDate = t.UTC().Format(time.RFC3339)
}
i.VCSCommit = biSetting(bi, biVCSCommit)
i.VCSRef = i.VCSCommit
modified := biSetting(bi, biVCSModified)
if modified == "true" {
i.VCSState = stateDirty
i.VCSRef += "-" + stateDirty
} else if modified == "false" {
i.VCSState = stateClean
}
}
return i
}
func biSetting(bi *debug.BuildInfo, key string) string {
if bi == nil {
return unknown
}
for _, setting := range bi.Settings {
if setting.Key == key {
return setting.Value
}
}
return unknown
}
+37
View File
@@ -0,0 +1,37 @@
//go:build !go1.18
package version
import (
"fmt"
"runtime"
)
type Info struct {
GoVer string `json:"goVersion"` // go version
GoCompiler string `json:"goCompiler"` // go compiler
Platform string `json:"platform"` // os/arch
VCSCommit string `json:"vcsCommit"` // commit sha
VCSDate string `json:"vcsDate"` // commit date in RFC3339 format
VCSRef string `json:"vcsRef"` // commit sha + dirty if state is not clean
VCSState string `json:"vcsState"` // clean or dirty
VCSTag string `json:"vcsTag"` // tag
}
func GetInfo() Info {
i := Info{
GoVer: unknown,
Platform: unknown,
VCSCommit: unknown,
VCSDate: unknown,
VCSRef: unknown,
VCSState: unknown,
VCSTag: "",
}
i.GoVer = runtime.Version()
i.GoCompiler = runtime.Compiler
i.Platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH)
return i
}
+206
View File
@@ -0,0 +1,206 @@
package regclient
import (
"context"
"fmt"
"log/slog"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/manifest"
"github.com/regclient/regclient/types/platform"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/warning"
)
type manifestOpt struct {
d descriptor.Descriptor
platform *platform.Platform
schemeOpts []scheme.ManifestOpts
requireDigest bool
}
// ManifestOpts define options for the Manifest* commands.
type ManifestOpts func(*manifestOpt)
// WithManifest passes a manifest to ManifestDelete.
func WithManifest(m manifest.Manifest) ManifestOpts {
return func(opts *manifestOpt) {
opts.schemeOpts = append(opts.schemeOpts, scheme.WithManifest(m))
}
}
// WithManifestCheckReferrers checks for referrers field on ManifestDelete.
// This will update the client managed referrer listing.
func WithManifestCheckReferrers() ManifestOpts {
return func(opts *manifestOpt) {
opts.schemeOpts = append(opts.schemeOpts, scheme.WithManifestCheckReferrers())
}
}
// WithManifestChild for ManifestPut indicates the manifest is not the top level manifest being copied.
// This is used by the ocidir scheme to determine what entries to include in the index.json.
func WithManifestChild() ManifestOpts {
return func(opts *manifestOpt) {
opts.schemeOpts = append(opts.schemeOpts, scheme.WithManifestChild())
}
}
// WithManifestDesc includes the descriptor for ManifestGet.
// This is used to automatically extract a Data field if available.
func WithManifestDesc(d descriptor.Descriptor) ManifestOpts {
return func(opts *manifestOpt) {
opts.d = d
}
}
// WithManifestPlatform resolves the platform specific manifest on Get and Head requests.
// This causes an additional GET query to a registry when an Index or Manifest List is encountered.
// This option is ignored if the retrieved manifest is not an Index or Manifest List.
func WithManifestPlatform(p platform.Platform) ManifestOpts {
return func(opts *manifestOpt) {
opts.platform = &p
}
}
// WithManifestRequireDigest falls back from a HEAD to a GET request when digest headers aren't received.
func WithManifestRequireDigest() ManifestOpts {
return func(opts *manifestOpt) {
opts.requireDigest = true
}
}
// ManifestDelete removes a manifest, including all tags pointing to that registry.
// The reference must include the digest to delete (see TagDelete for deleting a tag).
// All tags pointing to the manifest will be deleted.
func (rc *RegClient) ManifestDelete(ctx context.Context, r ref.Ref, opts ...ManifestOpts) error {
if !r.IsSet() {
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
for _, fn := range opts {
fn(&opt)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return err
}
return schemeAPI.ManifestDelete(ctx, r, opt.schemeOpts...)
}
// ManifestGet retrieves a manifest.
func (rc *RegClient) ManifestGet(ctx context.Context, r ref.Ref, opts ...ManifestOpts) (manifest.Manifest, error) {
if !r.IsSet() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
for _, fn := range opts {
fn(&opt)
}
if opt.d.Digest != "" {
r = r.AddDigest(opt.d.Digest.String())
data, err := opt.d.GetData()
if err == nil {
return manifest.New(
manifest.WithDesc(opt.d),
manifest.WithRaw(data),
manifest.WithRef(r),
)
}
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
m, err := schemeAPI.ManifestGet(ctx, r)
if err != nil {
return m, err
}
if opt.platform != nil && !m.IsList() {
rc.slog.Debug("ignoring platform option, image is not an index",
slog.String("platform", opt.platform.String()),
slog.String("ref", r.CommonName()))
}
// this will loop to handle a nested index
for opt.platform != nil && m.IsList() {
d, err := manifest.GetPlatformDesc(m, opt.platform)
if err != nil {
return m, err
}
r = r.SetDigest(d.Digest.String())
m, err = schemeAPI.ManifestGet(ctx, r)
if err != nil {
return m, err
}
}
return m, err
}
// ManifestHead queries for the existence of a manifest and returns metadata (digest, media-type, size).
func (rc *RegClient) ManifestHead(ctx context.Context, r ref.Ref, opts ...ManifestOpts) (manifest.Manifest, error) {
if !r.IsSet() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
for _, fn := range opts {
fn(&opt)
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
m, err := schemeAPI.ManifestHead(ctx, r)
if err != nil {
return m, err
}
if opt.platform != nil && !m.IsList() {
rc.slog.Debug("ignoring platform option, image is not an index",
slog.String("platform", opt.platform.String()),
slog.String("ref", r.CommonName()))
}
// this will loop to handle a nested index
for opt.platform != nil && m.IsList() {
if !m.IsSet() {
m, err = schemeAPI.ManifestGet(ctx, r)
}
d, err := manifest.GetPlatformDesc(m, opt.platform)
if err != nil {
return m, err
}
r = r.SetDigest(d.Digest.String())
m, err = schemeAPI.ManifestHead(ctx, r)
if err != nil {
return m, err
}
}
if opt.requireDigest && m.GetDescriptor().Digest.String() == "" {
m, err = schemeAPI.ManifestGet(ctx, r)
}
return m, err
}
// ManifestPut pushes a manifest.
// Any descriptors referenced by the manifest typically need to be pushed first.
func (rc *RegClient) ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, opts ...ManifestOpts) error {
if !r.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
opt := manifestOpt{schemeOpts: []scheme.ManifestOpts{}}
for _, fn := range opts {
fn(&opt)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return err
}
return schemeAPI.ManifestPut(ctx, r, m, opt.schemeOpts...)
}
+18
View File
@@ -0,0 +1,18 @@
package regclient
import (
"context"
"github.com/regclient/regclient/types/ping"
"github.com/regclient/regclient/types/ref"
)
// Ping verifies access to a registry or equivalent.
func (rc *RegClient) Ping(ctx context.Context, r ref.Ref) (ping.Result, error) {
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return ping.Result{}, err
}
return schemeAPI.Ping(ctx, r)
}

Some files were not shown because too many files have changed in this diff Show More