Compare commits

..
Author SHA1 Message Date
devydave a2647c3331 refactor: add option to run specific test files, file mode ssh key and ignore changes to path_to_file 2026-08-31 10:59:02 +02:00
devydave d510e13461 docs: explains -t option 2026-08-24 23:31:12 +02:00
devydave 470a34fdb1 test: adds automation script and ignore path_to_repo changes 2026-08-24 23:29:30 +02:00
devydave a28cc7afce docs: more formatting and typo 2026-08-13 20:41:04 +02:00
devydave 21dccc895d docs: formatting 2026-08-13 20:38:20 +02:00
devydave bb3c7576f8 docs: add setup documentation 2026-08-13 20:34:14 +02:00
devydave 02cbf7c014 refactor: make path editable without nix and impure 2026-08-13 19:57:51 +02:00
devydave a8c8df8604 fix: remove personal terminal choice 2026-08-12 23:01:12 +02:00
devydave 5766a25144 feat: adds host module 2026-08-12 22:43:16 +02:00
devydave 1ccbb31b70 refactor: simplify config and remove hom-manager dependency 2026-08-07 15:15:04 +02:00
devydave 5e0dac6508 refactor: adjusting hosts generation 2026-08-07 14:11:00 +02:00
devydave 38d13eae73 refactor: better variable name 2026-08-07 13:42:22 +02:00
devydave b5e22790be refactor: better name for inspect function 2026-08-07 13:36:25 +02:00
devydave beee7bd3f2 refactor: only export ssh auth socket when not existing 2026-08-07 11:09:30 +02:00
devydave 0c0e16914a refactor: remove traefik deployment because it is not needed 2026-08-07 10:36:23 +02:00
devydave 9bd8f4962a feat: adds running integrations test suite 2026-08-06 22:00:29 +02:00
devydave 4e42d4fb9e feat: adds local vm setup that can reach each other 2026-07-30 16:53:34 +02:00
390 changed files with 28157 additions and 37642 deletions
+3 -3
View File
@@ -3,12 +3,12 @@ kind: pipeline
name: coopcloud.tech/abra
steps:
- name: make check
image: golang:1.27
image: golang:1.26
commands:
- make check
- name: xgettext-go
image: golang:1.27
image: golang:1.26
environment:
GOPRIVATE: coopcloud.tech
commands:
@@ -43,7 +43,7 @@ steps:
- tag
- name: make test
image: golang:1.27
image: golang:1.26
environment:
ABRA_DIR: /root/.abra_test
commands:
-1
View File
@@ -1 +0,0 @@
vendor/** linguist-generated=true
+1
View File
@@ -7,3 +7,4 @@
/bin
dist/
tests/integration/.bats
tests/resources/path_to_repo
+1 -1
View File
@@ -1,5 +1,5 @@
# Build image
FROM golang:1.27-alpine AS build
FROM golang:1.26-alpine AS build
ENV GOPRIVATE=coopcloud.tech
+51 -1
View File
@@ -17,7 +17,7 @@ export GOPRIVATE=coopcloud.tech
all: format check build
run:
@go run -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(ABRA) $(ARGS)
@go run -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(ABRA)
install:
@go install -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(ABRA)
@@ -84,3 +84,53 @@ build-mo:
release:
@goreleaser release --clean
CLIENT_VM := abra-client
vm-client-status:
sudo systemctl status microvm@$(CLIENT_VM).service
vm-client-create:test-integration-hosts
sudo microvm -f git+file://$$(pwd) -c $(CLIENT_VM)
vm-client-start:
sudo systemctl start microvm@$(CLIENT_VM).service
vm-client-update:test-integration-hosts
sudo microvm -R -f git+file://$$(pwd) -u $(CLIENT_VM)
vm-client-run:
sudo microvm -f git+file://$$(pwd) -r $(CLIENT_VM)
vm-client-stop:
sudo systemctl stop microvm@$(CLIENT_VM)
vm-client-delete: vm-client-stop
sudo rm -rf /var/lib/microvms/$(CLIENT_VM)
vm-client-connect:
ssh abra@10.0.0.2 -i ./tests/resources/local_integration_ssh
SERVER_VM := abra-server
vm-server-status:
sudo systemctl status microvm@$(SERVER_VM).service
vm-server-create:
sudo microvm -f git+file://$$(pwd) -c $(SERVER_VM)
vm-server-start:
sudo systemctl start microvm@$(SERVER_VM).service
vm-server-update:
sudo microvm -R -f git+file://$$(pwd) -u $(SERVER_VM)
vm-server-run:
sudo microvm -f git+file://$$(pwd) -r $(SERVER_VM)
vm-server-stop:
sudo systemctl stop microvm@$(SERVER_VM)
vm-server-delete: vm-server-stop
sudo rm -rf /var/lib/microvms/$(SERVER_VM)
vm-server-connect:
ssh abra@10.0.0.3 -i ./tests/resources/local_integration_ssh
test-integration-hosts:
./scripts/tests/extra_hosts
# TEST_INTEGRATION_FILES overrides the default of running all test files
ifdef TEST_INTEGRATION_FILES
test_files := $(TEST_INTEGRATION_FILES);
else
test_files := *;
endif
test-integration: build
chmod 600 ./tests/resources/local_integration_ssh
ssh-add ./tests/resources/local_integration_ssh
@bats -Tp --verbose-run tests/integration/$(test_files)
-1
View File
@@ -19,6 +19,5 @@ var (
Minor bool
NoDomainChecks bool
Patch bool
PinDigests bool
ShowUnchanged bool
)
+3 -30
View File
@@ -130,7 +130,7 @@ interface.`),
}
for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(formatter.RemoveSha(service.Image))
img, err := reference.ParseNormalizedNamed(service.Image)
if err != nil {
log.Fatal(err)
}
@@ -293,31 +293,12 @@ interface.`),
}
}
if upgradeTag != "skip" {
var ok bool
var err error
var resolvedDigest string
if internal.PinDigests {
digest, dErr := client.GetImageMultiplatformDigest(img, upgradeTag)
if dErr != nil {
log.Fatal(i18n.G("failed to resolve digest for %s:%s: %s", image, upgradeTag, dErr.Error()))
}
resolvedDigest = digest
ok, err = recipe.UpdatePinnedTag(image, upgradeTag, digest)
} else {
ok, err = recipe.UpdateTag(image, upgradeTag)
}
ok, err := recipe.UpdateTag(image, upgradeTag)
if err != nil {
log.Fatal(err)
}
if ok {
if internal.PinDigests {
log.Info(i18n.G("tag upgraded and pinned from %s to %s@%s for %s", tag.String(), upgradeTag, resolvedDigest, image))
} else {
log.Info(i18n.G("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image))
}
log.Info(i18n.G("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image))
}
} else {
if !internal.NoInput {
@@ -436,12 +417,4 @@ func init() {
false,
i18n.G("commit changes"),
)
RecipeUpgradeCommand.Flags().BoolVarP(
&internal.PinDigests,
i18n.G("pindigests"),
i18n.GC("p", "pin container image by digest"),
false,
i18n.G("pin the container image version by manifest digest (image:tag@sha256:...)"),
)
}
Generated
+38
View File
@@ -18,6 +18,27 @@
"type": "github"
}
},
"microvm": {
"inputs": {
"nixpkgs": [
"nixpkgs"
],
"spectrum": "spectrum"
},
"locked": {
"lastModified": 1784666190,
"narHash": "sha256-xgfS6slV7J3baMooNN1UuBi51RIgg9y0DbCxfSA0668=",
"owner": "astro",
"repo": "microvm.nix",
"rev": "fa5340ac684cdce8a22b6d4a0bcebb0cc999275e",
"type": "github"
},
"original": {
"owner": "astro",
"repo": "microvm.nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1778443072,
@@ -37,9 +58,26 @@
"root": {
"inputs": {
"flake-utils": "flake-utils",
"microvm": "microvm",
"nixpkgs": "nixpkgs"
}
},
"spectrum": {
"flake": false,
"locked": {
"lastModified": 1783694892,
"narHash": "sha256-xO8f7Qng+18FK2UlB9vcrkxCaQMCt5WjCH24aW/11eg=",
"ref": "refs/heads/main",
"rev": "24c4346e30fdea8d8e80f34aec3554a15a667d24",
"revCount": 1410,
"type": "git",
"url": "https://spectrum-os.org/git/spectrum"
},
"original": {
"type": "git",
"url": "https://spectrum-os.org/git/spectrum"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
+87 -18
View File
@@ -4,34 +4,103 @@
inputs = {
nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable";
flake-utils.url = "github:numtide/flake-utils";
microvm = {
url = "github:astro/microvm.nix";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs =
{
self,
nixpkgs,
microvm,
flake-utils,
}:
flake-utils.lib.eachDefaultSystem (
system:
let
pkgs = nixpkgs.legacyPackages.${system};
in
{
packages = rec {
abra = pkgs.callPackage ./package.nix { };
default = abra;
let
system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system};
publicKey = builtins.readFile ./tests/resources/local_integration_ssh.pub;
privateKey = builtins.readFile ./tests/resources/local_integration_ssh;
pathToRepo = builtins.readFile ./tests/resources/path_to_repo;
# local DNS aliases for the server host
extraHosts = builtins.readFile ./tests/resources/extra_hosts;
username = "abra";
password = "abra";
defaultDNS = [
# Quad9.net
"9.9.9.9"
"149.112.112.112"
"2620:fe::fe"
"2620:fe::9"
];
in
{
nixosConfigurations = {
abra-client = nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
modules = [
microvm.nixosModules.microvm
./nix/hosts/client/configuration.nix
];
specialArgs = {
inherit
publicKey
privateKey
username
password
defaultDNS
extraHosts
pathToRepo
;
};
};
apps = rec {
abra = flake-utils.lib.mkApp { drv = self.packages.${system}.abra; };
default = abra;
abra-server = nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
modules = [
microvm.nixosModules.microvm
./nix/hosts/server/configuration.nix
];
specialArgs = {
inherit
publicKey
privateKey
username
password
defaultDNS
;
};
};
devShells.default = pkgs.mkShell {
packages = with pkgs; [
go_1_26
gnumake
};
nixosModules = rec {
host = {
imports = [
microvm.nixosModules.host
./nix/modules/host.nix
];
};
}
);
default = host;
};
packages = rec {
abra = pkgs.callPackage ./nix/package.nix { };
default = abra;
};
apps = rec {
abra = flake-utils.lib.mkApp { drv = self.packages.${system}.abra; };
default = abra;
};
devShells.${system}.default = pkgs.mkShell {
# testing env variables
BATS_LIB_PATH = "~/.local/share/bats/";
TEST_SERVER = "abra.local";
ABRA_DIR = "$HOME/.abra_test";
packages = with pkgs; [
go_1_26
gnumake
gopls
];
};
};
}
+12 -12
View File
@@ -13,15 +13,14 @@ require (
github.com/docker/cli v28.4.0+incompatible
github.com/docker/docker v28.5.2+incompatible
github.com/docker/go-units v0.5.0
github.com/go-git/go-git/v5 v5.19.2
github.com/go-git/go-git/v5 v5.19.1
github.com/google/go-cmp v0.7.0
github.com/leonelquinteros/gotext v1.7.2
github.com/moby/sys/signal v0.7.1
github.com/moby/term v0.5.2
github.com/pkg/errors v0.9.1
github.com/regclient/regclient v0.11.5
github.com/schollz/progressbar/v3 v3.19.1
golang.org/x/term v0.46.0
golang.org/x/term v0.44.0
gopkg.in/yaml.v3 v3.0.1
gotest.tools/v3 v3.5.2
)
@@ -51,6 +50,7 @@ require (
github.com/containerd/platforms v0.2.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
@@ -73,7 +73,7 @@ require (
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kevinburke/ssh_config v1.6.0 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/klauspost/compress v1.18.5 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
@@ -97,6 +97,7 @@ require (
github.com/opencontainers/runc v1.1.13 // indirect
github.com/opencontainers/runtime-spec v1.1.0 // indirect
github.com/pjbgf/sha1cd v0.6.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.5 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
@@ -105,7 +106,6 @@ require (
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/skeema/knownhosts v1.3.2 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@@ -123,11 +123,11 @@ require (
go.opentelemetry.io/otel/trace v1.42.0 // indirect
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.53.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.50.0 // indirect
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/text v0.39.0 // indirect
golang.org/x/net v0.53.0 // indirect
golang.org/x/text v0.36.0 // indirect
golang.org/x/time v0.15.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
@@ -151,11 +151,11 @@ require (
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/sergi/go-diff v1.4.0 // indirect
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.12.1
github.com/spf13/cobra v1.10.1
github.com/stretchr/testify v1.11.1
github.com/theupdateframework/notary v0.7.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
golang.org/x/sys v0.48.0
golang.org/x/sys v0.46.0
)
replace github.com/docker/cli v28.4.0+incompatible => git.coopcloud.tech/toolshed/docker-cli v28.5.3-0.20260202112816-30df2d0b3a00+incompatible
+19 -28
View File
@@ -306,6 +306,7 @@ github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW
github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/decentral1se/cobra v1.10.2 h1:MZ8Ifi/jRels9sZrpSccDbUlK++3b2HlBODfv0Bh6x0=
github.com/decentral1se/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
@@ -392,8 +393,8 @@ github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmm
github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY=
github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk=
github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00=
github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -426,8 +427,6 @@ github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
@@ -585,8 +584,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -700,8 +699,6 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/olareg/olareg v0.2.1 h1:RPHGIaqlVWPbKAsOYUj7e2WfEEW5M7F8I6hKMrwd4jU=
github.com/olareg/olareg v0.2.1/go.mod h1:dhr8QetC7U7jJ2m93oxDhEEOKCRbPgOK1oGyKfB4QNo=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -763,6 +760,7 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -798,8 +796,6 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
github.com/regclient/regclient v0.11.5 h1:OHRsXO0F3qHGfa4HEUv+EkMH9NXNcCTBKjNzyC/UhIA=
github.com/regclient/regclient v0.11.5/go.mod h1:DZUOfIT14WFTK2Pj4vjd93avy9O4Fdpjrf9ir23TbRE=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -861,10 +857,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/sudo-bmitch/oci-digest v0.1.2 h1:are0qzWTsFZGZ3Uvdi9OSztJszSWaab6iqquMEEB7rw=
github.com/sudo-bmitch/oci-digest v0.1.2/go.mod h1:SH6l5OIe0islKBZBedjiPOeET/0QwGL+/oYfQt51uQo=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
@@ -876,8 +870,6 @@ github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnH
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@@ -954,9 +946,8 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -975,8 +966,8 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1052,8 +1043,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1148,13 +1139,13 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1164,8 +1155,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+75
View File
@@ -0,0 +1,75 @@
# Hosts
These hosts are VMs used for running the integration tests on your local machine.
The hosts contain a client where abra with the tests will be run and a server
to simulate a remote machine.
## Prepare the host
For this setup to work you need a machine with NixOS and flakes enabled.
1. Import the abra flake
The install example is based on the [using nix flakes wiki page](https://nixos.wiki/wiki/flakes#Using_nix_flakes_with_NixOS).
```nix
inputs = {
abra = {
url = "git+https://git.coopcloud.tech/toolshed/abra.git";
};
};
```
2. Add the host module to your configuration
Now add the host module to your configuration. At the toplevel of a flake it could look like this:
```nix
desktop = inputs.nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
modules = [
abra.nixosModules.host
./configuration.nix
];
specialArgs = inputs;
};
```
3. Adjust the required config
```nix
abra.testing = {
enable = true; # loads the required config, when set true
externalInterface = "eth1"; # adjust this to your network interface that has access to the internet
};
```
You can look up the interface with `ip link`. After that rebuild your system and you are ready to go.
## Get started
All commands are run on the repository root path.
1. Set path to repo
The VM will create a shared volume from the content of the file `tests/resources/path_to_repo`.
Set the content of the file to the path of your abra repository and run the following command to prevent tracking of the file:
```sh
git update-index --assume-unchanged ./tests/resources/path_to_repo
```
This will share the repository from your machine with the VM and you can edit files while testing without rebuiling
or restarting the VM.
2. Create the VMs
```sh
make vm-client-create
make vm-server-create
```
3. Start the VMs
```sh
make vm-client-start
make vm-server-start
```
After running that command you should be able to ping the machines.
The client runs on 10.0.0.2 and server 10.0.0.3
4. Connect to the client
Running the following command will connect to the client VM via SSH.
```sh
make vm-client-connect
```
5. Run the tests
```sh
make test-integration
```
+145
View File
@@ -0,0 +1,145 @@
{
lib,
pkgs,
publicKey,
username,
pathToRepo,
defaultDNS,
extraHosts,
...
}:
let
index = 2;
mac = "00:00:00:00:00:01";
serverIp = "10.0.0.3";
in
{
imports = [
../../modules/ssh.nix
../../modules/user.nix
../../modules/docker.nix
];
microvm = {
vcpu = 4;
mem = 2049; # see issue related in microvm repo with 2048, so add 1MB
interfaces = [
{
id = "vm${toString index}";
type = "tap";
inherit mac;
}
];
shares = [
{
proto = "virtiofs";
tag = "repo";
# Source path can be absolute or relative
# to /var/lib/microvms/$hostName
source = "${lib.trim pathToRepo}";
mountPoint = "/home/${username}/abra";
}
];
};
boot.tmp = {
useTmpfs = true;
tmpfsSize = "2G";
};
networking = {
hostName = "abra-client";
useNetworkd = true;
};
systemd.network.networks."10-eth" = {
matchConfig.MACAddress = mac;
# Static IP configuration
address = [
"10.0.0.${toString index}/32"
"fec0::${lib.toHexString index}/128"
];
routes = [
{
# A route to the host
Destination = "10.0.0.0/32";
GatewayOnLink = true;
}
{
# Route to server
Destination = "${serverIp}/32";
Gateway = "10.0.0.0";
GatewayOnLink = true;
}
{
# Default route
Destination = "0.0.0.0/0";
Gateway = "10.0.0.0";
GatewayOnLink = true;
}
{
# Default route
Destination = "::/0";
Gateway = "fec0::";
GatewayOnLink = true;
}
];
networkConfig = {
# DNS servers no longer come from DHCP nor Router
# Advertisements. Perhaps you want to change the defaults:
DNS = defaultDNS;
};
};
# open the ports to allow ssh access from the host
networking.firewall.allowedTCPPorts = [ 22 ];
networking.firewall.allowedUDPPorts = [ 22 ];
networking.extraHosts = extraHosts;
environment.variables = {
CGO_ENABLED = 0;
TEST_SERVER = "abra.local";
ABRA_DIR = "$HOME/.abra_test";
};
programs.ssh = {
startAgent = true;
knownHostsFiles = [
(pkgs.writeText "local.keys" ''
abra.local ${publicKey}
'')
];
extraConfig = ''
Host abra.local
HostName abra.local
Port 22
User ${username}
'';
};
programs.git = {
enable = true;
# many recipe commands need a preset git user
config = [
{
user = {
name = "abra dev";
email = "helo@coopcloud.tech";
};
}
];
};
environment.systemPackages = with pkgs; [
# build dependencies, copied from flake.nix
go_1_26
gnumake
# testing dependencies
(bats.withLibraries (p: [
p.bats-assert
p.bats-file
p.bats-support
]))
jq
wget
];
}
+42
View File
@@ -0,0 +1,42 @@
{
serverIp,
username,
...
}:
{
# Home Manager needs a bit of information about you and the
# paths it should manage.
home.username = username;
home.homeDirectory = "/home/${username}";
# This value determines the Home Manager release that your
# configuration is compatible with. This helps avoid breakage
# when a new Home Manager release introduces backwards
# incompatible changes.
#
# You can update Home Manager without changing this value. See
# the Home Manager release notes for a list of state version
# changes in each release.
home.stateVersion = "26.05";
# Let Home Manager install and manage itself.
programs.home-manager.enable = true;
programs.ssh = {
enable = true;
enableDefaultConfig = false;
settings = {
"abra.local" = {
HostName = serverIp;
User = username;
Port = 22;
IdentityFile = "/home/${username}/abra/tests/resources/local_integration_ssh";
};
"*.abra.local" = {
HostName = serverIp;
User = username;
Port = 22;
IdentityFile = "/home/${username}/abra/tests/resources/local_integration_sshn";
};
};
};
}
+91
View File
@@ -0,0 +1,91 @@
{
lib,
defaultDNS,
...
}:
let
index = 3;
mac = "00:00:00:00:00:02";
clientIp = "10.0.0.2";
in
{
imports = [
../../modules/ssh.nix
../../modules/user.nix
../../modules/docker.nix
];
microvm = {
vcpu = 4;
mem = 8097; # see issue related in microvm repo with 2048, so add 1MB
interfaces = [
{
id = "vm${toString index}";
type = "tap";
inherit mac;
}
];
};
boot.tmp = {
useTmpfs = true;
tmpfsSize = "8G";
};
networking = {
hostName = "abra-server";
useNetworkd = true;
};
systemd.network.networks."11-eth" = {
matchConfig.MACAddress = mac;
# Static IP configuration
address = [
"10.0.0.${toString index}/32"
"fec0::${lib.toHexString index}/128"
];
routes = [
{
# A route to the host
Destination = "10.0.0.0/32";
GatewayOnLink = true;
}
{
# Route to server
Destination = "${clientIp}/32";
Gateway = "10.0.0.0";
GatewayOnLink = true;
}
{
# Default route
Destination = "0.0.0.0/0";
Gateway = "10.0.0.0";
GatewayOnLink = true;
}
{
# Default route
Destination = "::/0";
Gateway = "fec0::";
GatewayOnLink = true;
}
];
networkConfig = {
# DNS servers no longer come from DHCP nor Router
# Advertisements. Perhaps you want to change the defaults:
DNS = defaultDNS;
};
};
# open the ports to allow ssh access from the host
networking.firewall.allowedTCPPorts = [
22
80
443
1312 # deploy with udp and tcp on same port test
];
networking.firewall.allowedUDPPorts = [
22
80
443
1312 # deploy with udp and tcp on same port test
];
}
+7
View File
@@ -0,0 +1,7 @@
{ ... }:
{
virtualisation.docker = {
enable = true;
liveRestore = false;
};
}
+59
View File
@@ -0,0 +1,59 @@
{
lib,
config,
...
}:
let
maxVMs = 3;
in
{
options = {
abra.testing.enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Enable the abra integration testing framework";
};
abra.testing.externalInterface = lib.mkOption {
type = lib.types.str;
default = null;
description = "Change this to the interface with upstream Internet access";
};
};
config = lib.mkIf config.abra.testing.enable {
networking.useNetworkd = true;
systemd.network.wait-online.enable = false;
systemd.network.networks = builtins.listToAttrs (
map (index: {
name = "30-vm${toString index}";
value = {
matchConfig.Name = "vm${toString index}";
# Host's addresses
address = [
"10.0.0.0/32"
"fec0::/128"
];
# Setup routes to the VM
routes = [
{
Destination = "10.0.0.${toString index}/32";
}
{
Destination = "fec0::${lib.toHexString index}/128";
}
];
# Enable routing
networkConfig = {
IPv4Forwarding = true;
IPv6Forwarding = true;
};
};
}) (lib.genList (i: i + 1) maxVMs)
);
networking.nat = {
enable = true;
internalIPs = [ "10.0.0.0/24" ];
externalInterface = config.abra.testing.externalInterface;
};
};
}
+33
View File
@@ -0,0 +1,33 @@
{
privateKey,
publicKey,
username,
...
}:
{
# default host key location
environment.etc."ssh/ssh_host_ed25519_key" = {
text = privateKey;
mode = "0600";
};
environment.etc."ssh/ssh_host_ed25519_key.pub" = {
text = publicKey;
mode = "0644";
};
services.openssh = {
enable = true;
settings = {
PasswordAuthentication = false;
PermitRootLogin = "no";
AllowUsers = [ "${username}" ];
};
generateHostKeys = false;
hostKeys = [
{
path = "/etc/ssh/ssh_host_ed25519_key";
type = "ed25519";
}
];
};
}
+22
View File
@@ -0,0 +1,22 @@
{
username,
password,
publicKey,
...
}:
{
users.users."${username}" = {
isNormalUser = true;
password = password;
home = "/home/${username}";
description = "Abra user";
extraGroups = [
"wheel"
"networkmanager"
"docker"
];
openssh.authorizedKeys.keys = [
publicKey
];
};
}
View File
-23
View File
@@ -9,12 +9,8 @@ import (
"github.com/containers/image/docker"
"github.com/containers/image/types"
"github.com/distribution/reference"
"github.com/regclient/regclient"
"github.com/regclient/regclient/types/ref"
)
var rc = regclient.New()
// GetRegistryTags retrieves all tags of an image from a container registry.
func GetRegistryTags(img reference.Named) ([]string, error) {
var tags []string
@@ -32,22 +28,3 @@ func GetRegistryTags(img reference.Named) ([]string, error) {
return tags, nil
}
// GetImageDigest resolves the content digest (sha256:...) for a specific
// image:tag reference by querying the registry manifest. This gives us a
// digest to pin to while keeping support for multi-arch deployments
// See: https://docs.docker.com/build/building/multi-platform/#why-multi-platform-builds
// and: https://docs.docker.com/dhi/explore/security-concepts/digests/#multi-platform-images-and-manifests
func GetImageMultiplatformDigest(img reference.Named, tag string) (string, error) {
r, err := ref.New(fmt.Sprintf("%s:%s", img.Name(), tag))
if err != nil {
return "", fmt.Errorf("parsing reference %s:%s: %w", img.Name(), tag, err)
}
m, err := rc.ManifestHead(context.Background(), r)
if err != nil {
return "", fmt.Errorf("fetching manifest for %s:%s: %w", img.Name(), tag, err)
}
return m.GetDescriptor().Digest.String(), nil
}
+1 -2
View File
@@ -34,8 +34,7 @@ func SmallSHA(hash string) string {
return hash[:8]
}
// RemoveSha remove image sha (digest suffix) from a string that are added in some docker outputs
// e.g., "image:tag@sha256:..." -> "image:tag"
// RemoveSha remove image sha from a string that are added in some docker outputs
func RemoveSha(str string) string {
return strings.Split(str, "@")[0]
}
-7
View File
@@ -6,13 +6,6 @@ import (
"github.com/stretchr/testify/assert"
)
func TestRemoveSha(t *testing.T) {
assert.Equal(t, "ubuntu:latest", RemoveSha("ubuntu:latest@sha256:1234567890abcdef"))
assert.Equal(t, "ubuntu:latest", RemoveSha("ubuntu:latest"))
assert.Equal(t, "my-repo/my-image:v1", RemoveSha("my-repo/my-image:v1@sha256:abcdef1234567890"))
assert.Equal(t, "my-repo/my-image", RemoveSha("my-repo/my-image@sha256:abcdef1234567890"))
}
func TestBoldDirtyDefault(t *testing.T) {
assert.Equal(t, "foo", BoldDirtyDefault("foo"))
}
+2
View File
@@ -52,6 +52,7 @@ func Clone(dir, url string) error {
URL: url,
Tags: git.AllTags,
ReferenceName: plumbing.ReferenceName("refs/heads/main"),
SingleBranch: true,
})
if err != nil && gitCloneIgnoreErr(err) {
@@ -70,6 +71,7 @@ func Clone(dir, url string) error {
URL: url,
Tags: git.AllTags,
ReferenceName: plumbing.ReferenceName("refs/heads/master"),
SingleBranch: true,
})
if err != nil && gitCloneIgnoreErr(err) {
+76 -104
View File
@@ -7,7 +7,7 @@
msgid ""
msgstr "Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2026-09-10 00:13-0400\n"
"POT-Creation-Date: 2026-06-14 17:56+0200\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -165,7 +165,7 @@ msgid " # standard rollback\n"
" abra app rollback 1312.net 2.0.0+1.2.3"
msgstr ""
#: ./pkg/formatter/formatter.go:47
#: ./pkg/formatter/formatter.go:46
msgid " ago"
msgstr ""
@@ -189,7 +189,7 @@ msgstr ""
msgid "%d volumes removed successfully"
msgstr ""
#: ./pkg/recipe/git.go:198
#: ./pkg/recipe/git.go:197
#, c-format
msgid "%s (%s) has locally unstaged changes?"
msgstr ""
@@ -219,7 +219,7 @@ msgstr ""
msgid "%s created (version: %s)"
msgstr ""
#: ./cli/recipe/upgrade.go:354
#: ./cli/recipe/upgrade.go:335
#, c-format
msgid "%s currently has these unstaged changes 👇"
msgstr ""
@@ -324,7 +324,7 @@ msgstr ""
msgid "%s is already fetched"
msgstr ""
#: ./pkg/recipe/compose.go:299
#: ./pkg/recipe/compose.go:233
#, c-format
msgid "%s is already set, nothing to do?"
msgstr ""
@@ -419,7 +419,7 @@ msgstr ""
msgid "%s sanitised as %s for new app"
msgstr ""
#: ./pkg/recipe/git.go:451
#: ./pkg/recipe/git.go:445
#, c-format
msgid "%s service is missing image tag?"
msgstr ""
@@ -739,7 +739,7 @@ msgid "Create a new version of a recipe.\n"
" ssh-add ~/.ssh/<my-ssh-private-key-for-git-coopcloud-tech>"
msgstr ""
#: ./pkg/formatter/formatter.go:135
#: ./pkg/formatter/formatter.go:134
msgid "CreateOverview: only accepts rows of len == 2"
msgstr ""
@@ -902,7 +902,7 @@ msgstr ""
msgid "Generate the recipe catalogue"
msgstr ""
#: ./pkg/recipe/git.go:465
#: ./pkg/recipe/git.go:459
#, c-format
msgid "GetRecipeVersions encountered error for %s: %s (collected %d versions)"
msgstr ""
@@ -1687,7 +1687,7 @@ msgctxt "recipe fetch"
msgid "a"
msgstr ""
#: ./cli/recipe/upgrade.go:427
#: ./cli/recipe/upgrade.go:408
msgctxt "recipe upgrade"
msgid "a"
msgstr ""
@@ -1737,7 +1737,7 @@ msgstr ""
msgid "accepts at most 2 args with --local/-l"
msgstr ""
#: ./pkg/recipe/compose.go:315
#: ./pkg/recipe/compose.go:249
#, c-format
msgid "add '- \"%s\"' manually to the 'app' service in %s"
msgstr ""
@@ -1780,7 +1780,7 @@ msgstr ""
msgid "all-services"
msgstr ""
#: ./cli/recipe/upgrade.go:426
#: ./cli/recipe/upgrade.go:407
msgid "all-tags"
msgstr ""
@@ -1815,7 +1815,7 @@ msgstr ""
msgid "are you sure?"
msgstr ""
#: ./pkg/recipe/git.go:163
#: ./pkg/recipe/git.go:162
#, c-format
msgid "attempting to checkout '%s' as chaos commit"
msgstr ""
@@ -1919,7 +1919,7 @@ msgstr ""
msgid "c"
msgstr ""
#: ./cli/recipe/upgrade.go:435
#: ./cli/recipe/upgrade.go:416
msgctxt "recipe upgrade"
msgid "c"
msgstr ""
@@ -1928,7 +1928,7 @@ msgstr ""
msgid "can not insert from file and read from stdin"
msgstr ""
#: ./cli/recipe/upgrade.go:342
#: ./cli/recipe/upgrade.go:323
#, c-format
msgid "can upgrade service: %s, image: %s, tag: %s ::"
msgstr ""
@@ -1951,7 +1951,7 @@ msgstr ""
msgid "cancelled"
msgstr ""
#: ./pkg/catalogue/catalogue.go:59 ./pkg/recipe/git.go:252
#: ./pkg/catalogue/catalogue.go:59 ./pkg/recipe/git.go:251
#, c-format
msgid "cannot ensure %s is up-to-date, no git remotes configured"
msgstr ""
@@ -1966,7 +1966,7 @@ msgstr ""
msgid "cannot get label %s for %s"
msgstr ""
#: ./pkg/recipe/git.go:59
#: ./pkg/recipe/git.go:58
#, c-format
msgid "cannot redeploy previous chaos version (%s), did you mean to use \"--chaos\"?"
msgstr ""
@@ -2108,7 +2108,7 @@ msgstr ""
msgid "chore: publish new catalogue release changes"
msgstr ""
#: ./cli/recipe/upgrade.go:371
#: ./cli/recipe/upgrade.go:352
msgid "chore: update image tags"
msgstr ""
@@ -2118,7 +2118,7 @@ msgstr ""
msgid "cmd"
msgstr ""
#: ./pkg/recipe/git.go:476
#: ./pkg/recipe/git.go:470
#, c-format
msgid "collected %s for %s"
msgstr ""
@@ -2170,19 +2170,19 @@ msgstr ""
msgid "commandconn: starting %s with %v"
msgstr ""
#: ./cli/recipe/upgrade.go:434
#: ./cli/recipe/upgrade.go:415
msgid "commit"
msgstr ""
#: ./cli/recipe/upgrade.go:437
#: ./cli/recipe/upgrade.go:418
msgid "commit changes"
msgstr ""
#: ./cli/recipe/upgrade.go:361
#: ./cli/recipe/upgrade.go:342
msgid "commit changes?"
msgstr ""
#: ./cli/recipe/upgrade.go:375
#: ./cli/recipe/upgrade.go:356
#, c-format
msgid "committed changes as '%s'"
msgstr ""
@@ -2230,7 +2230,7 @@ msgstr ""
msgid "connection timed out for %s"
msgstr ""
#: ./pkg/recipe/compose.go:257
#: ./pkg/recipe/compose.go:191
#, c-format
msgid "considering %s config(s) for label update"
msgstr ""
@@ -2258,7 +2258,7 @@ msgstr ""
msgid "context lacks Docker endpoint"
msgstr ""
#: ./pkg/recipe/compose.go:295
#: ./pkg/recipe/compose.go:229
#, c-format
msgid "coop-cloud.${STACK_NAME}.version=%s"
msgstr ""
@@ -2456,12 +2456,12 @@ msgstr ""
msgid "destination directory does not exist"
msgstr ""
#: ./pkg/recipe/git.go:379
#: ./pkg/recipe/git.go:373
#, c-format
msgid "detected %s as tags for recipe %s"
msgstr ""
#: ./pkg/formatter/formatter.go:81
#: ./pkg/formatter/formatter.go:80
msgid "detected ABRA_CI=1"
msgstr ""
@@ -2653,7 +2653,7 @@ msgstr ""
msgid "ensure recipe: %s"
msgstr ""
#: ./pkg/recipe/git.go:57
#: ./pkg/recipe/git.go:56
#, c-format
msgid "ensuring env version %s"
msgstr ""
@@ -2751,12 +2751,12 @@ msgstr ""
msgid "failed to check git status of %s: %s"
msgstr ""
#: ./pkg/git/branch.go:95 ./pkg/recipe/git.go:232
#: ./pkg/git/branch.go:95 ./pkg/recipe/git.go:231
#, c-format
msgid "failed to check out %s in %s"
msgstr ""
#: ./pkg/recipe/git.go:418
#: ./pkg/recipe/git.go:412
#, c-format
msgid "failed to check out %s in %s: %s"
msgstr ""
@@ -2806,7 +2806,7 @@ msgstr ""
msgid "failed to generate random bytes: %w"
msgstr ""
#: ./pkg/recipe/git.go:427
#: ./pkg/recipe/git.go:421
#, c-format
msgid "failed to get compose config for %s: %s"
msgstr ""
@@ -2839,12 +2839,12 @@ msgstr ""
msgid "failed to migrate app config: %s"
msgstr ""
#: ./pkg/client/registry.go:24
#: ./pkg/client/registry.go:20
#, c-format
msgid "failed to parse image %s, saw: %s"
msgstr ""
#: ./pkg/recipe/git.go:437
#: ./pkg/recipe/git.go:431
#, c-format
msgid "failed to parse image for %s in %s: %s"
msgstr ""
@@ -2898,11 +2898,6 @@ msgstr ""
msgid "failed to resize tty, using default size"
msgstr ""
#: ./cli/recipe/upgrade.go:303
#, c-format
msgid "failed to resolve digest for %s:%s: %s"
msgstr ""
#: ./cli/app/new.go:138
#, c-format
msgid "failed to retrieve latest commit for %s: %s"
@@ -2957,7 +2952,7 @@ msgstr ""
msgid "fetch all recipes"
msgstr ""
#: ./pkg/catalogue/catalogue.go:84 ./pkg/recipe/git.go:290
#: ./pkg/catalogue/catalogue.go:84 ./pkg/recipe/git.go:284
#, c-format
msgid "fetched latest git changes for %s"
msgstr ""
@@ -3075,12 +3070,12 @@ msgstr ""
msgid "git changes pushed"
msgstr ""
#: ./pkg/recipe/git.go:423
#: ./pkg/recipe/git.go:417
#, c-format
msgid "git checkout: %s in %s"
msgstr ""
#: ./pkg/git/clone.go:63 ./pkg/git/clone.go:100
#: ./pkg/git/clone.go:64 ./pkg/git/clone.go:102
#, c-format
msgid "git clone %s: cancelled due to interrupt"
msgstr ""
@@ -3090,17 +3085,17 @@ msgstr ""
msgid "git clone: %s"
msgstr ""
#: ./pkg/git/clone.go:87
#: ./pkg/git/clone.go:89
#, c-format
msgid "git clone: %s already exists"
msgstr ""
#: ./pkg/git/clone.go:58 ./pkg/git/clone.go:76 ./pkg/git/clone.go:85
#: ./pkg/git/clone.go:59 ./pkg/git/clone.go:78 ./pkg/git/clone.go:87
#, c-format
msgid "git clone: %s cloned successfully"
msgstr ""
#: ./pkg/git/clone.go:67
#: ./pkg/git/clone.go:68
msgid "git clone: main branch failed, attempting master branch"
msgstr ""
@@ -3155,7 +3150,7 @@ msgstr ""
msgid "git.coopcloud.tech repo exists"
msgstr ""
#: ./pkg/recipe/git.go:390
#: ./pkg/recipe/git.go:384
#, c-format
msgid "git: opening repository in %s"
msgstr ""
@@ -3296,15 +3291,15 @@ msgstr ""
msgid "including VOLUMES=%v in backupbot exec invocation"
msgstr ""
#: ./cli/recipe/release.go:634 ./cli/recipe/upgrade.go:397
#: ./cli/recipe/release.go:634 ./cli/recipe/upgrade.go:378
msgid "increase the major part of the version"
msgstr ""
#: ./cli/recipe/release.go:642 ./cli/recipe/upgrade.go:405
#: ./cli/recipe/release.go:642 ./cli/recipe/upgrade.go:386
msgid "increase the minor part of the version"
msgstr ""
#: ./cli/recipe/release.go:650 ./cli/recipe/upgrade.go:413
#: ./cli/recipe/release.go:650 ./cli/recipe/upgrade.go:394
msgid "increase the patch part of the version"
msgstr ""
@@ -3469,7 +3464,7 @@ msgstr ""
msgid "list [flags]"
msgstr ""
#: ./cli/recipe/upgrade.go:429
#: ./cli/recipe/upgrade.go:410
msgid "list all tags, not just upgrades"
msgstr ""
@@ -3539,11 +3534,11 @@ msgstr ""
#. with no spaces in between
#. translators: `abra man` aliases. use a comma separated list of aliases
#. with no spaces in between
#: ./cli/app/list.go:319 ./cli/app/move.go:34 ./cli/app/ps.go:205 ./cli/app/secret.go:553 ./cli/app/secret.go:649 ./cli/recipe/list.go:104 ./cli/recipe/upgrade.go:419 ./cli/recipe/version.go:139 ./cli/run.go:152 ./cli/server/list.go:106
#: ./cli/app/list.go:319 ./cli/app/move.go:34 ./cli/app/ps.go:205 ./cli/app/secret.go:553 ./cli/app/secret.go:649 ./cli/recipe/list.go:104 ./cli/recipe/upgrade.go:400 ./cli/recipe/version.go:139 ./cli/run.go:152 ./cli/server/list.go:106
msgid "m"
msgstr ""
#: ./cli/app/list.go:318 ./cli/app/ps.go:204 ./cli/app/secret.go:552 ./cli/app/secret.go:648 ./cli/recipe/list.go:103 ./cli/recipe/upgrade.go:418 ./cli/recipe/version.go:138 ./cli/server/list.go:105
#: ./cli/app/list.go:318 ./cli/app/ps.go:204 ./cli/app/secret.go:552 ./cli/app/secret.go:648 ./cli/recipe/list.go:103 ./cli/recipe/upgrade.go:399 ./cli/recipe/version.go:138 ./cli/server/list.go:105
msgid "machine"
msgstr ""
@@ -3552,7 +3547,7 @@ msgstr ""
msgid "main app service version for %s is empty?"
msgstr ""
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:66 ./cli/internal/recipe.go:80 ./cli/recipe/release.go:631 ./cli/recipe/upgrade.go:394
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:66 ./cli/internal/recipe.go:80 ./cli/recipe/release.go:631 ./cli/recipe/upgrade.go:375
msgid "major"
msgstr ""
@@ -3581,7 +3576,7 @@ msgstr ""
msgid "migrating app config from %s to %s"
msgstr ""
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:68 ./cli/internal/recipe.go:82 ./cli/recipe/release.go:639 ./cli/recipe/upgrade.go:402
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:68 ./cli/internal/recipe.go:82 ./cli/recipe/release.go:639 ./cli/recipe/upgrade.go:383
msgid "minor"
msgstr ""
@@ -3736,7 +3731,7 @@ msgstr ""
msgid "no changes discovered in %s, nothing to publish?"
msgstr ""
#: ./cli/recipe/upgrade.go:380
#: ./cli/recipe/upgrade.go:361
msgid "no changes, skip creating commit"
msgstr ""
@@ -3761,7 +3756,7 @@ msgstr ""
msgid "no domain provided"
msgstr ""
#: ./pkg/recipe/compose.go:314
#: ./pkg/recipe/compose.go:248
msgid "no existing label found, automagic insertion not supported yet"
msgstr ""
@@ -3902,7 +3897,7 @@ msgstr ""
msgid "not requesting a remote TTY"
msgstr ""
#: ./cli/recipe/upgrade.go:324
#: ./cli/recipe/upgrade.go:305
#, c-format
msgid "not upgrading %s, skipping as requested"
msgstr ""
@@ -3961,17 +3956,12 @@ msgstr ""
msgid "p"
msgstr ""
#: ./cli/recipe/upgrade.go:443
msgctxt "pin container image by digest"
msgid "p"
msgstr ""
#: ./cli/recipe/upgrade.go:163
#, c-format
msgid "parsed %s for %s"
msgstr ""
#: ./pkg/recipe/compose.go:160
#: ./pkg/recipe/compose.go:159
#, c-format
msgid "parsed %s from %s"
msgstr ""
@@ -4009,7 +3999,7 @@ msgstr ""
msgid "pass command not found on $PATH, is it installed?"
msgstr ""
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:70 ./cli/internal/recipe.go:84 ./cli/recipe/release.go:647 ./cli/recipe/upgrade.go:410
#: ./cli/internal/recipe.go:48 ./cli/internal/recipe.go:70 ./cli/internal/recipe.go:84 ./cli/recipe/release.go:647 ./cli/recipe/upgrade.go:391
msgid "patch"
msgstr ""
@@ -4025,14 +4015,6 @@ msgstr ""
msgid "perform action without further prompt"
msgstr ""
#: ./cli/recipe/upgrade.go:445
msgid "pin the container image version by manifest digest (image:tag@sha256:...)"
msgstr ""
#: ./cli/recipe/upgrade.go:442
msgid "pindigests"
msgstr ""
#. translators: `abra app env pull` aliases. use a comma separated list of
#. aliases with no spaces in between
#: ./cli/app/env.go:39
@@ -4071,7 +4053,7 @@ msgstr ""
msgid "prefer offline & filesystem access"
msgstr ""
#: ./cli/app/list.go:321 ./cli/app/ps.go:207 ./cli/app/secret.go:555 ./cli/app/secret.go:651 ./cli/recipe/list.go:106 ./cli/recipe/upgrade.go:421 ./cli/recipe/version.go:141 ./cli/server/list.go:108
#: ./cli/app/list.go:321 ./cli/app/ps.go:207 ./cli/app/secret.go:555 ./cli/app/secret.go:651 ./cli/recipe/list.go:106 ./cli/recipe/upgrade.go:402 ./cli/recipe/version.go:141 ./cli/server/list.go:108
msgid "print machine-readable output"
msgstr ""
@@ -4079,7 +4061,7 @@ msgstr ""
msgid "proceed?"
msgstr ""
#: ./pkg/recipe/git.go:410
#: ./pkg/recipe/git.go:404
#, c-format
msgid "processing %s for %s"
msgstr ""
@@ -4147,7 +4129,7 @@ msgstr ""
msgid "re"
msgstr ""
#: ./pkg/recipe/git.go:158
#: ./pkg/recipe/git.go:157
#, c-format
msgid "read %s as tags for recipe %s"
msgstr ""
@@ -4883,17 +4865,17 @@ msgstr ""
msgid "skipping secret (because it already exists) on %s: %s"
msgstr ""
#: ./pkg/recipe/git.go:419
#: ./pkg/recipe/git.go:413
#, c-format
msgid "skipping tag %s: checkout failed: %s"
msgstr ""
#: ./pkg/recipe/git.go:428
#: ./pkg/recipe/git.go:422
#, c-format
msgid "skipping tag %s: invalid compose config: %s"
msgstr ""
#: ./pkg/recipe/git.go:438
#: ./pkg/recipe/git.go:432
#, c-format
msgid "skipping tag %s: invalid image reference in service %s: %s"
msgstr ""
@@ -4987,7 +4969,7 @@ msgstr ""
msgid "store secrets in a local pass store"
msgstr ""
#: ./pkg/formatter/formatter.go:239
#: ./pkg/formatter/formatter.go:238
#, c-format
msgid "stripped %s to %s for parsing"
msgstr ""
@@ -4996,7 +4978,7 @@ msgstr ""
msgid "succeeded"
msgstr ""
#: ./pkg/recipe/git.go:185
#: ./pkg/recipe/git.go:184
#, c-format
msgid "successfully checked %s out to %s in %s"
msgstr ""
@@ -5025,7 +5007,7 @@ msgstr ""
msgid "sync [flags]"
msgstr ""
#: ./pkg/recipe/compose.go:309
#: ./pkg/recipe/compose.go:243
#, c-format
msgid "synced label %s to service %s"
msgstr ""
@@ -5043,12 +5025,7 @@ msgstr ""
msgid "tag at commit %s is unannotated or otherwise broken"
msgstr ""
#: ./cli/recipe/upgrade.go:317
#, c-format
msgid "tag upgraded and pinned from %s to %s@%s for %s"
msgstr ""
#: ./cli/recipe/upgrade.go:319
#: ./cli/recipe/upgrade.go:301
#, c-format
msgid "tag upgraded from %s to %s for %s"
msgstr ""
@@ -5161,17 +5138,17 @@ msgstr ""
msgid "un"
msgstr ""
#: ./pkg/recipe/git.go:194
#: ./pkg/recipe/git.go:193
#, c-format
msgid "unable to check git clean status in %s: %s"
msgstr ""
#: ./pkg/recipe/git.go:263
#: ./pkg/recipe/git.go:262
#, c-format
msgid "unable to check out default branch in %s: %s"
msgstr ""
#: ./pkg/git/clone.go:98
#: ./pkg/git/clone.go:100
#, c-format
msgid "unable to clean up git clone of %s: %s"
msgstr ""
@@ -5245,7 +5222,7 @@ msgstr ""
msgid "unable to discover SSH remote for %s"
msgstr ""
#: ./pkg/recipe/git.go:274
#: ./pkg/recipe/git.go:268
#, c-format
msgid "unable to fetch tags in %s: %s"
msgstr ""
@@ -5255,7 +5232,7 @@ msgstr ""
msgid "unable to get container matching %s: %s"
msgstr ""
#: ./pkg/recipe/git.go:286
#: ./pkg/recipe/git.go:280
#, c-format
msgid "unable to git pull in %s: %s"
msgstr ""
@@ -5284,12 +5261,12 @@ msgstr ""
msgid "unable to look up server context for %s: %s"
msgstr ""
#: ./cli/recipe/fetch.go:77 ./pkg/git/read.go:26 ./pkg/lint/recipe.go:491 ./pkg/recipe/git.go:243
#: ./cli/recipe/fetch.go:77 ./pkg/git/read.go:26 ./pkg/lint/recipe.go:491 ./pkg/recipe/git.go:242
#, c-format
msgid "unable to open %s: %s"
msgstr ""
#: ./pkg/recipe/git.go:258
#: ./pkg/recipe/git.go:257
#, c-format
msgid "unable to open git work tree in %s: %s"
msgstr ""
@@ -5314,7 +5291,7 @@ msgstr ""
msgid "unable to parse %s, error was: %s, skipping upgrade for %s"
msgstr ""
#: ./pkg/recipe/compose.go:154
#: ./pkg/recipe/compose.go:153
#, c-format
msgid "unable to parse %s, skipping"
msgstr ""
@@ -5349,7 +5326,7 @@ msgstr ""
msgid "unable to read new env %s: %s"
msgstr ""
#: ./pkg/recipe/git.go:248
#: ./pkg/recipe/git.go:247
#, c-format
msgid "unable to read remotes in %s: %s"
msgstr ""
@@ -5384,7 +5361,7 @@ msgstr ""
msgid "unable to reset commit after failed release attempt: %s"
msgstr ""
#: ./pkg/recipe/git.go:167
#: ./pkg/recipe/git.go:166
#, c-format
msgid "unable to resolve '%s': %s"
msgstr ""
@@ -5541,16 +5518,11 @@ msgstr ""
msgid "updating %s"
msgstr ""
#: ./pkg/recipe/compose.go:176 ./pkg/recipe/compose.go:303
#: ./pkg/recipe/compose.go:171 ./pkg/recipe/compose.go:237
#, c-format
msgid "updating %s to %s in %s"
msgstr ""
#: ./pkg/recipe/compose.go:236
#, c-format
msgid "updating pinned reference %s to %s in %s"
msgstr ""
#. translators: `abra upgrade` command for autocompletion
#: ./cli/run.go:103
msgid "upgrade"
@@ -5834,11 +5806,11 @@ msgstr ""
msgid "writing recipe version failed: %s"
msgstr ""
#: ./cli/recipe/release.go:632 ./cli/recipe/upgrade.go:395
#: ./cli/recipe/release.go:632 ./cli/recipe/upgrade.go:376
msgid "x"
msgstr ""
#: ./cli/recipe/release.go:640 ./cli/recipe/upgrade.go:403
#: ./cli/recipe/release.go:640 ./cli/recipe/upgrade.go:384
msgid "y"
msgstr ""
@@ -5850,7 +5822,7 @@ msgstr ""
msgid "you can only use one version flag: --major, --minor or --patch"
msgstr ""
#: ./cli/recipe/release.go:648 ./cli/recipe/upgrade.go:411
#: ./cli/recipe/release.go:648 ./cli/recipe/upgrade.go:392
msgid "z"
msgstr ""
Binary file not shown.
+330 -694
View File
File diff suppressed because it is too large Load Diff
-29
View File
@@ -125,13 +125,6 @@ var LintRules = map[string][]LintRule{
HowToResolve: i18n.G("reduce length of secret names to 12 chars"),
Function: LintSecretLengths,
},
{
Ref: "R016",
Level: i18n.G("warn"),
Description: i18n.G("all images use a digest"),
HowToResolve: i18n.G("use a digest for all images"),
Function: LintAllImagesTaggedWithDigest,
},
},
"error": {
{
@@ -340,28 +333,6 @@ func LintAllImagesTagged(recipe recipe.Recipe) (bool, error) {
return true, nil
}
func LintAllImagesTaggedWithDigest(recipe recipe.Recipe) (bool, error) {
config, err := recipe.GetComposeConfig(nil)
if err != nil {
return false, err
}
for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(service.Image)
if err != nil {
return false, err
}
if reference.IsNameOnly(img) {
return false, nil
}
_, ok := img.(reference.Digested)
if !ok {
return false, nil
}
}
return true, nil
}
func LintNoUnstableTags(recipe recipe.Recipe) (bool, error) {
config, err := recipe.GetComposeConfig(nil)
if err != nil {
+1 -67
View File
@@ -122,7 +122,6 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
log.Debug(i18n.G("considering %s config(s) for tag update", strings.Join(composeFiles, ", ")))
updated := false
for _, composeFile := range composeFiles {
opts := stack.Deploy{Composefiles: []string{composeFile}}
@@ -167,10 +166,6 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
old := fmt.Sprintf("%s:%s", composeImage, composeTag)
new := fmt.Sprintf("%s:%s", composeImage, tag)
if old == new {
continue
}
replacedBytes := strings.Replace(string(bytes), old, new, -1)
log.Debug(i18n.G("updating %s to %s in %s", old, new, compose.Filename))
@@ -178,72 +173,11 @@ func (r Recipe) UpdateTag(image, tag string) (bool, error) {
if err := os.WriteFile(compose.Filename, []byte(replacedBytes), 0o764); err != nil {
return false, err
}
updated = true
}
}
}
return updated, nil
}
// UpdatePinnedTag updates an image reference to a pinned version (image:tag@sha256:...) in-place on local compose files.
func (r Recipe) UpdatePinnedTag(image, tag, digest string) (bool, error) {
glob := fmt.Sprintf("%s/compose**yml", r.Dir)
image = formatter.StripTagMeta(image)
composeFiles, err := filepath.Glob(glob)
if err != nil {
return false, err
}
updated := false
for _, composeFile := range composeFiles {
opts := stack.Deploy{Composefiles: []string{composeFile}}
sampleEnv, err := r.SampleEnv()
if err != nil {
return false, err
}
compose, err := loader.LoadComposefile(opts, sampleEnv)
if err != nil {
return false, err
}
for _, service := range compose.Services {
if service.Image == "" {
continue
}
img, _ := reference.ParseNormalizedNamed(formatter.RemoveSha(service.Image))
composeImage := formatter.StripTagMeta(reference.Path(img))
if image == composeImage {
bytes, err := ioutil.ReadFile(composeFile)
if err != nil {
return false, err
}
old := service.Image
new := fmt.Sprintf("%s:%s@%s", composeImage, tag, digest)
if old == new {
continue
}
replacedBytes := strings.Replace(string(bytes), old, new, -1)
log.Debug(i18n.G("updating pinned reference %s to %s in %s", old, new, compose.Filename))
if err := os.WriteFile(compose.Filename, []byte(replacedBytes), 0o764); err != nil {
return false, err
}
updated = true
}
}
}
return updated, nil
return false, nil
}
// UpdateLabel updates a label in-place on file system local compose files.
+2 -8
View File
@@ -16,7 +16,6 @@ import (
"coopcloud.tech/tagcmp"
"github.com/distribution/reference"
"github.com/go-git/go-git/v5"
gitCfg "github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
)
@@ -263,12 +262,7 @@ func (r Recipe) EnsureUpToDate() error {
return errors.New(i18n.G("unable to check out default branch in %s: %s", r.Dir, err))
}
// the refspec is passed explicitly, because a repository cloned by an older abra has a
// single-branch refspec stored in its config and would otherwise never see other branches
fetchOpts := &git.FetchOptions{
Tags: git.AllTags,
RefSpecs: []gitCfg.RefSpec{"+refs/heads/*:refs/remotes/origin/*"},
}
fetchOpts := &git.FetchOptions{Tags: git.AllTags}
if err := repo.Fetch(fetchOpts); err != nil {
if !strings.Contains(err.Error(), "already up-to-date") {
return errors.New(i18n.G("unable to fetch tags in %s: %s", r.Dir, err))
@@ -432,7 +426,7 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, []string, error) {
versionMeta := make(map[string]ServiceMeta)
for _, service := range config.Services {
img, err := reference.ParseNormalizedNamed(formatter.RemoveSha(service.Image))
img, err := reference.ParseNormalizedNamed(service.Image)
if err != nil {
log.Debug(i18n.G("failed to parse image for %s in %s: %s", service.Name, tag, err))
warnMsg = append(warnMsg, i18n.G("skipping tag %s: invalid image reference in service %s: %s", tag, service.Name, err))
+1 -1
View File
@@ -3,7 +3,7 @@ version: "3.8"
services:
app:
image: nginx:1.31.5
image: nginx:1.31.2
secrets:
- test_pass_one
- test_pass_two
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
# This file generates local DNS aliases for the client VM for integration testing
echo "generating hosts file..."
hosts_file_path="./tests/resources/extra_hosts"
echo "removing old file..."
rm -f $hosts_file_path
# the server domain
server_ip="10.0.0.3"
domain="abra.local"
echo "$server_ip $domain" >> $hosts_file_path
# static subdomains used in integration tests
subdomains=("gitea" "zammad" "custom-html" "foo" "foobar")
for subdomain in "${subdomains[@]}"
do
echo "$server_ip $subdomain.$domain" >> $hosts_file_path
done
search_dir=./tests/integration
for entry in "$search_dir"/*.bats
do
file_name=$(basename "${entry}")
# converting file names to testing subdomains
subdomain=$(echo "$file_name" | tr . _)
echo "$server_ip $subdomain.$domain" >> $hosts_file_path
done
echo "writing extra hosts success"
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env bash
# ignore changes to path_to_repo
git update-index --assume-unchanged ./tests/resources/path_to_repo
# write path to repo
pwd > ./tests/resources/path_to_repo
# create or update a VM, start it and wait for successful ping
start_vm () {
if [ -d "/var/lib/microvms/abra-$1" ]; then
make "vm-$1-update"
else
make "vm-$1-create"
fi
make "vm-$1-start"
host_available=0
while [ $host_available == 0 ]; do
if ping -c 1 "$2" &> /dev/null
then
host_available=1
else
echo "could not ping $1, waiting for 5 seconds..."
sleep 5
fi
done
}
start_vm "client" "10.0.0.2"
start_vm "server" "10.0.0.3"
read -rp "Press enter to run all tests or specify test files: " TEST_INTEGRATION_FILES
# -t option is needed for stdin output that is checked in integration tests
ssh -t abra@10.0.0.2 -i ./tests/resources/local_integration_ssh "cd abra; make test-integration TEST_INTEGRATION_FILES=$TEST_INTEGRATION_FILES"
+2 -1
View File
@@ -7,4 +7,5 @@
* Integration tests are in `./tests/integration`. Please see [these
docs](https://docs.coopcloud.tech/abra/hack/#integration-tests) for
instructions and tips on how to run them.
instructions and tips on how to run them. If you want to run it locally via nix,
read the [nix testing docs](../nix/hosts/README.md).
+9 -5
View File
@@ -598,8 +598,7 @@ teardown(){
run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
assert_output --partial "WITH_COMMENT=foo"
@@ -610,8 +609,7 @@ teardown(){
run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --force
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
refute_output --partial "WITH_COMMENT=foo"
assert_output --partial "WITH_COMMENT=bar"
@@ -629,8 +627,14 @@ teardown(){
run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
assert_success
run docker service inspect --format '{{ range .Endpoint.Ports }}{{ .Protocol }}={{ .PublishedPort }}{{ end }}' \
if _is_local;
then
run docker service inspect --format "{{ range .Endpoint.Ports }}{{ .Protocol }}={{ .PublishedPort }}{{ end }}" \
"${TEST_APP_DOMAIN//./_}_app"
else
run ssh "$TEST_SERVER" "docker service inspect --format '{{ range .Endpoint.Ports }}{{ .Protocol }}={{ .PublishedPort }}{{ end }}' ${TEST_APP_DOMAIN//./_}_app"
fi
assert_success
assert_output --partial "tcp=1312"
assert_output --partial "udp=1312"
@@ -133,8 +133,7 @@ teardown(){
run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
assert_output --partial "$TEST_RECIIPE:0.1.0+1.20.0"
}
-33
View File
@@ -122,39 +122,6 @@ teardown(){
assert_failure
}
@test "create new app with commit from another branch" {
branchHash=$(_get_other_branch_hash)
if [[ -z "$branchHash" ]]; then
skip "$TEST_RECIPE has no branch besides main"
fi
# re-clone the recipe the way an older abra did, with a single-branch refspec. --no-local
# forces a real transfer, a local clone would hardlink the whole object database and leave
# the commit reachable
run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
assert_success
run git clone -q --no-local --single-branch --branch main \
"$ABRA_DIR/origin-recipes/$TEST_RECIPE.git" "$ABRA_DIR/recipes/$TEST_RECIPE"
assert_success
# the commit has to be genuinely missing, otherwise this test passes for the wrong reason
run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --verify "$branchHash^{commit}"
assert_failure
run $ABRA app new "$TEST_RECIPE" "$branchHash" \
--no-input \
--server "$TEST_SERVER" \
--domain "$TEST_APP_DOMAIN"
assert_success
assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
# the recipe names itself in TYPE, which differs per branch, so only the version is checked
run grep -q "TYPE=.*:${branchHash}$" \
"$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
assert_success
}
@test "does not overwrite existing env files" {
run $ABRA app new "$TEST_RECIPE" \
--no-input \
+11 -4
View File
@@ -140,15 +140,22 @@ teardown(){
_undeploy_app
sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
run docker config create "${sanitisedDomainName}_test_conf_v99" "$ABRA_DIR/recipes/abra-test-recipe/abra.sh"
assert_success
remote_ssh_command=""
if _is_local;
then
run docker config create "${sanitisedDomainName}_test_conf_v99" "$ABRA_DIR/recipes/abra-test-recipe/abra.sh"
else
remote_ssh_command="ssh '$TEST_SERVER'"
run cat "$ABRA_DIR/recipes/abra-test-recipe/abra.sh" | ssh "$TEST_SERVER" docker config create "${sanitisedDomainName}_test_conf_v99 -"
assert_success
fi
assert bash -c "docker config ls | grep -q test_conf_v99"
assert bash -c "$remote_ssh_command docker config ls | grep -q test_conf_v99"
run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
assert_success
refute bash -c "docker config ls | grep -q test_conf_v99"
refute bash -c "$remote_ssh_command docker config ls | grep -q test_conf_v99"
}
@test "remove .env file" {
@@ -54,8 +54,7 @@ teardown(){
--no-input
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
assert_output --partial "$TEST_RECIIPE:0.1.0+1.20.0"
}
@@ -53,8 +53,7 @@ teardown(){
run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input
assert_success
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
_inspect_env_test_app
assert_success
assert_output --partial "$TEST_RECIIPE:0.2.0+1.21.0"
assert_output --partial "$TEST_RECIPE:0.2.0+1.21.0"
}
+11 -2
View File
@@ -31,9 +31,18 @@ _ensure_ssh_agent() {
exit 1
fi
export SSH_AUTH_SOCK="$HOME/.ssh/ssh_auth_sock"
if [ ! -S ~/.ssh/ssh_auth_sock ]; then
if [[ ! -v SSH_AUTH_SOCK ]]; then
export SSH_AUTH_SOCK="$HOME/.ssh/ssh_auth_sock"
eval `ssh-agent`
ln -sf "$SSH_AUTH_SOCK" ~/.ssh/ssh_auth_sock
fi
}
_is_local() {
if [[ "$TEST_SERVER" == "default" ]];
then
return 0
else
return 1
fi
}
+30 -6
View File
@@ -1,13 +1,37 @@
#!/usr/bin/env bash
_ensure_swarm() {
if [ "$(docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
run docker swarm init --advertise-addr 127.0.0.1:2377
assert_success
if [ "$(docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
run docker swarm init --advertise-addr 127.0.0.1:2377
assert_success
fi
if ! $(docker network ls | grep -q 'proxy'); then
run docker network create -d overlay proxy
assert_success
if ! docker network ls | grep -q 'proxy'; then
run docker network create -d overlay proxy
assert_success
fi
if ! _is_local;
then
if [ "$(ssh "$TEST_SERVER" docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
run ssh "$TEST_SERVER" docker swarm init --advertise-addr 127.0.0.1:2377
assert_success
fi
if ! ssh "$TEST_SERVER" docker network ls | grep -q 'proxy'; then
run ssh "$TEST_SERVER" docker network create -d overlay proxy
assert_success
fi
fi
}
_inspect_env_test_app() {
if _is_local;
then
containerId="$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)"
run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' "$containerId"
else
containerId="$(ssh "$TEST_SERVER" docker ps -f name="$TEST_APP_DOMAIN" -q)"
run ssh "$TEST_SERVER" "docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' '$containerId'"
fi
}
-7
View File
@@ -56,13 +56,6 @@ _get_tag_hash() {
echo $(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-list -n 1 "$1")
}
_get_other_branch_hash() {
# asked from the origin mirror, not from the recipe checkout, so that the result does not
# depend on what has been fetched. empty when the recipe only has a default branch
echo $(git ls-remote "$ABRA_DIR/origin-recipes/$TEST_RECIPE.git" \
| grep 'refs/heads/' | grep -v 'refs/heads/main$' | head -1 | cut -f1)
}
_get_head_hash() {
echo $(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H" HEAD)
}
-2
View File
@@ -38,8 +38,6 @@ teardown(){
run $ABRA recipe upgrade "custom-html" --no-input
assert_success
assert_output --partial 'can upgrade service: app'
assert_exists "$ABRA_DIR/recipes/custom-html"
}
+3 -3
View File
@@ -27,7 +27,7 @@ teardown(){
assert bash -c "docker context ls | grep -q $TEST_SERVER"
server_dir_perms=$(stat -c "%a" "$ABRA_DIR/servers/$TEST_SERVER")
assert_equal $server_dir_perms "700"
assert_equal "$server_dir_perms" "700"
}
@test "error if using name and --local together" {
@@ -43,8 +43,8 @@ teardown(){
assert bash -c "docker context ls | grep -q default"
assert_output --partial 'local server successfully added'
server_dir_perms=$(stat -c "%a" "$ABRA_DIR/servers/$TEST_SERVER")
assert_equal $server_dir_perms "700"
server_dir_perms=$(stat -c "%a" "$ABRA_DIR/servers/default")
assert_equal "$server_dir_perms" "700"
}
@test "create local server fails when no docker swarm" {
+57
View File
@@ -0,0 +1,57 @@
10.0.0.3 abra.local
10.0.0.3 gitea.abra.local
10.0.0.3 zammad.abra.local
10.0.0.3 custom-html.abra.local
10.0.0.3 foo.abra.local
10.0.0.3 foobar.abra.local
10.0.0.3 app_check_bats.abra.local
10.0.0.3 app_cmd_bats.abra.local
10.0.0.3 app_config_bats.abra.local
10.0.0.3 app_cp_bats.abra.local
10.0.0.3 app_deploy_bats.abra.local
10.0.0.3 app_deploy_env_version_bats.abra.local
10.0.0.3 app_deploy_overview_bats.abra.local
10.0.0.3 app_deploy_remote_recipes_bats.abra.local
10.0.0.3 app_env_bats.abra.local
10.0.0.3 app_env_version_bats.abra.local
10.0.0.3 app_labels_bats.abra.local
10.0.0.3 app_list_bats.abra.local
10.0.0.3 app_logs_bats.abra.local
10.0.0.3 app_move_bats.abra.local
10.0.0.3 app_new_bats.abra.local
10.0.0.3 app_ps_bats.abra.local
10.0.0.3 app_remove_bats.abra.local
10.0.0.3 app_restart_bats.abra.local
10.0.0.3 app_rollback_bats.abra.local
10.0.0.3 app_rollback_env_version_bats.abra.local
10.0.0.3 app_rollback_overview_bats.abra.local
10.0.0.3 app_run_bats.abra.local
10.0.0.3 app_secret_bats.abra.local
10.0.0.3 app_secret_env_version_bats.abra.local
10.0.0.3 app_services_bats.abra.local
10.0.0.3 app_undeploy_bats.abra.local
10.0.0.3 app_undeploy_env_version_bats.abra.local
10.0.0.3 app_undeploy_overview_bats.abra.local
10.0.0.3 app_upgrade_bats.abra.local
10.0.0.3 app_upgrade_env_version_bats.abra.local
10.0.0.3 app_upgrade_overview_bats.abra.local
10.0.0.3 app_volume_bats.abra.local
10.0.0.3 autocomplete_bats.abra.local
10.0.0.3 catalogue_bats.abra.local
10.0.0.3 dirs_bats.abra.local
10.0.0.3 install_bats.abra.local
10.0.0.3 recipe_diff_bats.abra.local
10.0.0.3 recipe_fetch_bats.abra.local
10.0.0.3 recipe_lint_bats.abra.local
10.0.0.3 recipe_list_bats.abra.local
10.0.0.3 recipe_new_bats.abra.local
10.0.0.3 recipe_release_bats.abra.local
10.0.0.3 recipe_reset_bats.abra.local
10.0.0.3 recipe_upgrade_bats.abra.local
10.0.0.3 recipe_version_bats.abra.local
10.0.0.3 server_add_bats.abra.local
10.0.0.3 server_list_bats.abra.local
10.0.0.3 server_prune_bats.abra.local
10.0.0.3 server_remove_bats.abra.local
10.0.0.3 upgrade_bats.abra.local
10.0.0.3 version_bats.abra.local
+7
View File
@@ -0,0 +1,7 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACBQ967UhxmFkq71WXvVRkEmtehCGsEnBVppAXxgtHHm7wAAAJhFGwTnRRsE
5wAAAAtzc2gtZWQyNTUxOQAAACBQ967UhxmFkq71WXvVRkEmtehCGsEnBVppAXxgtHHm7w
AAAEAnHEZcf2NeRQEcJC/aVgUWsdOz+vQgEG9ZY+3ErCeaKFD3rtSHGYWSrvVZe9VGQSa1
6EIawScFWmkBfGC0cebvAAAAFWFicmEgaW50ZWdyYXRpb24gdGVzdA==
-----END OPENSSH PRIVATE KEY-----
@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFD3rtSHGYWSrvVZe9VGQSa16EIawScFWmkBfGC0cebv abra integration test
+1
View File
@@ -0,0 +1 @@
/path/to/this/repo
+1 -1
View File
@@ -3,7 +3,7 @@ version: "3.8"
services:
app:
image: nginx:1.31.5
image: nginx:1.31.2
networks:
- proxy
deploy:
+3 -3
View File
@@ -3,16 +3,16 @@ kind: pipeline
name: coopcloud.tech/tagcmp
steps:
- name: gofmt
image: golang:1.27
image: golang:1.26
commands:
- test -z "$(gofmt -l .)"
- name: go build
image: golang:1.27
image: golang:1.26
commands:
- go build -v .
- name: go test
image: golang:1.27
image: golang:1.26
commands:
- go test . -cover
@@ -18,7 +18,6 @@
// tag is deprecated and thus should not be used.
// Go versions prior to 1.4 are disabled because they use a different layout
// for interfaces which make the implementation of unsafeReflectValue more complex.
//go:build !js && !appengine && !safe && !disableunsafe && go1.4
// +build !js,!appengine,!safe,!disableunsafe,go1.4
package spew
@@ -16,7 +16,6 @@
// when the code is running on Google App Engine, compiled by GopherJS, or
// "-tags safe" is added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
//go:build js || appengine || safe || disableunsafe || !go1.4
// +build js appengine safe disableunsafe !go1.4
package spew
@@ -254,15 +254,15 @@ pointer addresses used to indirect to the final value. It provides the
following features over the built-in printing facilities provided by the fmt
package:
- Pointers are dereferenced and followed
- Circular data structures are detected and handled properly
- Custom Stringer/error interfaces are optionally invoked, including
on unexported types
- Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
- Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
The configuration options are controlled by modifying the public members
of c. See ConfigState for options documentation.
@@ -295,12 +295,12 @@ func (c *ConfigState) convertArgs(args []interface{}) (formatters []interface{})
// NewDefaultConfig returns a ConfigState with the following default settings.
//
// Indent: " "
// MaxDepth: 0
// DisableMethods: false
// DisablePointerMethods: false
// ContinueOnMethod: false
// SortKeys: false
// Indent: " "
// MaxDepth: 0
// DisableMethods: false
// DisablePointerMethods: false
// ContinueOnMethod: false
// SortKeys: false
func NewDefaultConfig() *ConfigState {
return &ConfigState{Indent: " "}
}
@@ -21,36 +21,35 @@ debugging.
A quick overview of the additional features spew provides over the built-in
printing facilities for Go data types are as follows:
- Pointers are dereferenced and followed
- Circular data structures are detected and handled properly
- Custom Stringer/error interfaces are optionally invoked, including
on unexported types
- Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
- Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output (only when using
Dump style)
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output (only when using
Dump style)
There are two different approaches spew allows for dumping Go data structures:
- Dump style which prints with newlines, customizable indentation,
and additional debug information such as types and all pointer addresses
used to indirect to the final value
- A custom Formatter interface that integrates cleanly with the standard fmt
package and replaces %v, %+v, %#v, and %#+v to provide inline printing
similar to the default %v while providing the additional functionality
outlined above and passing unsupported format verbs such as %x and %q
along to fmt
* Dump style which prints with newlines, customizable indentation,
and additional debug information such as types and all pointer addresses
used to indirect to the final value
* A custom Formatter interface that integrates cleanly with the standard fmt
package and replaces %v, %+v, %#v, and %#+v to provide inline printing
similar to the default %v while providing the additional functionality
outlined above and passing unsupported format verbs such as %x and %q
along to fmt
# Quick Start
Quick Start
This section demonstrates how to quickly get started with spew. See the
sections below for further details on formatting and configuration options.
To dump a variable with full newlines, indentation, type, and pointer
information use Dump, Fdump, or Sdump:
spew.Dump(myVar1, myVar2, ...)
spew.Fdump(someWriter, myVar1, myVar2, ...)
str := spew.Sdump(myVar1, myVar2, ...)
@@ -59,13 +58,12 @@ Alternatively, if you would prefer to use format strings with a compacted inline
printing style, use the convenience wrappers Printf, Fprintf, etc with
%v (most compact), %+v (adds pointer addresses), %#v (adds types), or
%#+v (adds types and pointer addresses):
spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
# Configuration Options
Configuration Options
Configuration of spew is handled by fields in the ConfigState type. For
convenience, all of the top-level functions use a global state available
@@ -76,52 +74,51 @@ equivalent to the top-level functions. This allows concurrent configuration
options. See the ConfigState documentation for more details.
The following configuration options are available:
* Indent
String to use for each indentation level for Dump functions.
It is a single space by default. A popular alternative is "\t".
- Indent
String to use for each indentation level for Dump functions.
It is a single space by default. A popular alternative is "\t".
* MaxDepth
Maximum number of levels to descend into nested data structures.
There is no limit by default.
- MaxDepth
Maximum number of levels to descend into nested data structures.
There is no limit by default.
* DisableMethods
Disables invocation of error and Stringer interface methods.
Method invocation is enabled by default.
- DisableMethods
Disables invocation of error and Stringer interface methods.
Method invocation is enabled by default.
* DisablePointerMethods
Disables invocation of error and Stringer interface methods on types
which only accept pointer receivers from non-pointer variables.
Pointer method invocation is enabled by default.
- DisablePointerMethods
Disables invocation of error and Stringer interface methods on types
which only accept pointer receivers from non-pointer variables.
Pointer method invocation is enabled by default.
* DisablePointerAddresses
DisablePointerAddresses specifies whether to disable the printing of
pointer addresses. This is useful when diffing data structures in tests.
- DisablePointerAddresses
DisablePointerAddresses specifies whether to disable the printing of
pointer addresses. This is useful when diffing data structures in tests.
* DisableCapacities
DisableCapacities specifies whether to disable the printing of
capacities for arrays, slices, maps and channels. This is useful when
diffing data structures in tests.
- DisableCapacities
DisableCapacities specifies whether to disable the printing of
capacities for arrays, slices, maps and channels. This is useful when
diffing data structures in tests.
* ContinueOnMethod
Enables recursion into types after invoking error and Stringer interface
methods. Recursion after method invocation is disabled by default.
- ContinueOnMethod
Enables recursion into types after invoking error and Stringer interface
methods. Recursion after method invocation is disabled by default.
* SortKeys
Specifies map keys should be sorted before being printed. Use
this to have a more deterministic, diffable output. Note that
only native types (bool, int, uint, floats, uintptr and string)
and types which implement error or Stringer interfaces are
supported with other types sorted according to the
reflect.Value.String() output which guarantees display
stability. Natural map order is used by default.
- SortKeys
Specifies map keys should be sorted before being printed. Use
this to have a more deterministic, diffable output. Note that
only native types (bool, int, uint, floats, uintptr and string)
and types which implement error or Stringer interfaces are
supported with other types sorted according to the
reflect.Value.String() output which guarantees display
stability. Natural map order is used by default.
* SpewKeys
Specifies that, as a last resort attempt, map keys should be
spewed to strings and sorted by those strings. This is only
considered if SortKeys is true.
- SpewKeys
Specifies that, as a last resort attempt, map keys should be
spewed to strings and sorted by those strings. This is only
considered if SortKeys is true.
# Dump Usage
Dump Usage
Simply call spew.Dump with a list of variables you want to dump:
@@ -136,7 +133,7 @@ A third option is to call spew.Sdump to get the formatted output as a string:
str := spew.Sdump(myVar1, myVar2, ...)
# Sample Dump Output
Sample Dump Output
See the Dump example for details on the setup of the types and variables being
shown here.
@@ -153,14 +150,13 @@ shown here.
Byte (and uint8) arrays and slices are displayed uniquely like the hexdump -C
command as shown.
([]uint8) (len=32 cap=32) {
00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... |
00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0|
00000020 31 32 |12|
}
# Custom Formatter
Custom Formatter
Spew provides a custom formatter that implements the fmt.Formatter interface
so that it integrates cleanly with standard fmt package printing functions. The
@@ -174,7 +170,7 @@ standard fmt package for formatting. In addition, the custom formatter ignores
the width and precision arguments (however they will still work on the format
specifiers not handled by the custom formatter).
# Custom Formatter Usage
Custom Formatter Usage
The simplest way to make use of the spew custom formatter is to call one of the
convenience functions such as spew.Printf, spew.Println, or spew.Printf. The
@@ -188,17 +184,15 @@ functions have syntax you are most likely already familiar with:
See the Index for the full list convenience functions.
# Sample Formatter Output
Sample Formatter Output
Double pointer to a uint8:
%v: <**>5
%+v: <**>(0xf8400420d0->0xf8400420c8)5
%#v: (**uint8)5
%#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5
Pointer to circular struct with a uint8 field and a pointer to itself:
%v: <*>{1 <*><shown>}
%+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)<shown>}
%#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)<shown>}
@@ -207,7 +201,7 @@ Pointer to circular struct with a uint8 field and a pointer to itself:
See the Printf example for details on the setup of variables being shown
here.
# Errors
Errors
Since it is possible for custom Stringer/error interfaces to panic, spew
detects them and handles them internally by printing the panic information
@@ -488,15 +488,15 @@ pointer addresses used to indirect to the final value. It provides the
following features over the built-in printing facilities provided by the fmt
package:
- Pointers are dereferenced and followed
- Circular data structures are detected and handled properly
- Custom Stringer/error interfaces are optionally invoked, including
on unexported types
- Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
- Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
The configuration options are controlled by an exported package global,
spew.Config. See ConfigState for options documentation.
-1
View File
@@ -5,4 +5,3 @@ profile.out
.tmp/
.git-dist/
.vscode
build/tools/
-42
View File
@@ -110,48 +110,6 @@ func (r ReferenceName) IsTag() bool {
return strings.HasPrefix(string(r), refTagPrefix)
}
// IsSafe reports whether the reference name can be safely turned into a path
// under the .git directory, mirroring Git's refname_is_safe (refs.c). A name
// is safe when it is either:
//
// - under "refs/", non-empty after the prefix, containing no backslash and
// no empty, "." or ".." path component (so it cannot escape the refs/
// sub-tree, or alias another name, once turned into a path); or
// - a one-level pseudo-ref whose spelling is restricted to [A-Z_]
// (e.g. HEAD, ORIG_HEAD, FETCH_HEAD).
//
// Everything else — a lowercase or mixed one-level name such as "config" or
// "index", an absolute or drive-prefixed name, or a refs/ name that escapes —
// is unsafe, because it could resolve onto unrelated repository metadata.
func (r ReferenceName) IsSafe() bool {
s := string(r)
if s == "" {
return false
}
if rest, ok := strings.CutPrefix(s, refPrefix); ok {
// '\' is a path separator on Windows, so a refs/ name containing one
// could escape the sub-tree or alias another name once turned into a
// path; reject it outright (check_refname_format forbids '\' too).
if rest == "" || strings.Contains(rest, "\\") {
return false
}
for part := range strings.SplitSeq(rest, "/") {
if part == "" || part == "." || part == ".." {
return false
}
}
return true
}
for i := 0; i < len(s); i++ {
if (s[i] < 'A' || s[i] > 'Z') && s[i] != '_' {
return false
}
}
return true
}
func (r ReferenceName) String() string {
return string(r)
}
-57
View File
@@ -16,7 +16,6 @@ import (
"strings"
"time"
"github.com/go-git/go-git/v5/internal/pathutil"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/hash"
"github.com/go-git/go-git/v5/storage"
@@ -80,52 +79,8 @@ var (
// resolve outside the modules/ subtree, mirroring canonical Git's
// "ignoring suspicious submodule name" defence.
ErrModuleNameEscape = errors.New("submodule name escapes modules/ directory")
// ErrReferenceNameEscape is returned when a reference name would
// resolve outside its reference sub-tree once turned into a path
// under the .git directory (e.g. a name with a ".." component).
ErrReferenceNameEscape = errors.New("reference name escapes the reference storage")
)
// isPathSep reports whether r is a path separator in reference names.
// It treats both '/' and '\\' as separators to harden against cross-OS paths.
func isPathSep(r rune) bool { return r == '/' || r == '\\' }
// validReferenceName rejects reference names that cannot be safely turned into
// a path under the .git directory. A loose reference is stored verbatim at
// ".git/<name>", so a crafted name — for instance one advertised by a malicious
// remote — could climb out of its reference sub-tree and read, overwrite, or
// delete unrelated metadata such as .git/config.
//
// The storage-safety gate is plumbing.ReferenceName.IsSafe, mirroring Git's
// refname_is_safe: a name must be under refs/ without escaping it, or be a
// [A-Z_] pseudo-ref. This alone rejects absolute, drive-prefixed, escaping and
// single-level metadata names. On top of it, this adds filesystem-specific
// hardening that IsSafe's literal check does not cover: control characters, and
// components a case-insensitive/NTFS/HFS+ filesystem would fold back to "." or
// ".." (trailing dots/spaces, Alternate Data Streams, ignorable Unicode code
// points), delegated to pathutil.IsHFSDot and pathutil.IsNTFSDot with "." as
// the needle — as validSubmoduleName does — and run regardless of host OS.
func validReferenceName(name plumbing.ReferenceName) error {
if !name.IsSafe() {
return fmt.Errorf("%w: %q is not under refs/ nor a valid pseudo-ref", ErrReferenceNameEscape, string(name))
}
s := string(name)
for i := 0; i < len(s); i++ {
if s[i] < 0x20 || s[i] == 0x7f {
return fmt.Errorf("%w: %q", ErrReferenceNameEscape, s)
}
}
for _, part := range strings.FieldsFunc(s, isPathSep) {
// IsNTFSDot/IsHFSDot with a "." needle match ".." and its disguises
// but not a bare ".", so reject that component explicitly too.
if part == "." || pathutil.IsHFSDot(part, ".") || pathutil.IsNTFSDot(part, ".", "") {
return fmt.Errorf("%w: %q", ErrReferenceNameEscape, s)
}
}
return nil
}
// Options holds configuration for the storage.
type Options struct {
// ExclusiveAccess means that the filesystem is not modified externally
@@ -751,10 +706,6 @@ func (d *DotGit) checkReferenceAndTruncate(f billy.File, old *plumbing.Reference
}
func (d *DotGit) SetRef(r, old *plumbing.Reference) error {
if err := validReferenceName(r.Name()); err != nil {
return err
}
var content string
switch r.Type() {
case plumbing.SymbolicReference:
@@ -790,10 +741,6 @@ func (d *DotGit) Refs() ([]*plumbing.Reference, error) {
// Ref returns the reference for a given reference name.
func (d *DotGit) Ref(name plumbing.ReferenceName) (*plumbing.Reference, error) {
if err := validReferenceName(name); err != nil {
return nil, err
}
ref, err := d.readReferenceFile(".", name.String())
if err == nil {
return ref, nil
@@ -857,10 +804,6 @@ func (d *DotGit) packedRef(name plumbing.ReferenceName) (*plumbing.Reference, er
// RemoveRef removes a reference by name.
func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error {
if err := validReferenceName(name); err != nil {
return err
}
path := d.fs.Join(".", name.String())
_, err := d.fs.Stat(path)
if err == nil {
-67
View File
@@ -583,10 +583,6 @@ func (w *Worktree) checkoutChangeSubmodule(name string,
return err
}
if err := w.clearBlockingSymlinks(name); err != nil {
return err
}
if err := w.Filesystem.MkdirAll(name, mode); err != nil {
return err
}
@@ -630,70 +626,7 @@ func (w *Worktree) checkoutChangeRegularFile(name string,
return nil
}
// clearBlockingSymlinks removes a symlink that is in the way of
// materialising name, so the checkout writes a real entry in its place
// instead of following the link out of the worktree. Two cases:
//
// - a leading directory component that is a symlink (e.g. "s" while
// writing "s/config", where "s" links to ".git"): OpenFile/MkdirAll
// would traverse it, so the write would land under the link's target.
// - the final component itself being a symlink (e.g. writing "s" while
// "s" links to ".git/config"): OpenFile with O_TRUNC, or Symlink,
// would follow/replace through it and clobber the target.
//
// A symlink can never be a legitimate parent of, or the destination for,
// a tracked entry, so removing it is always correct. This mirrors upstream
// Git's forced checkout, which unlinks a blocking symlink in the leading
// path (create_directories) and unlinks an existing entry before
// write_entry.
// https://github.com/git/git/blob/v2.54.0/entry.c#L50
func (w *Worktree) clearBlockingSymlinks(name string) error {
var dirs []string
for dir := filepath.Dir(name); dir != "." && dir != "" && dir != string(filepath.Separator); dir = filepath.Dir(dir) {
dirs = append(dirs, dir)
}
// Leading components, shallowest-first: removing the shallowest symlink
// invalidates every component beneath it, so a single removal is enough.
for i := len(dirs) - 1; i >= 0; i-- {
fi, err := w.Filesystem.Lstat(dirs[i])
if err != nil {
// A missing component is created as a real directory by the
// checkout. Any other error means we cannot tell whether it is
// a symlink, so surface it instead of leaving a blocking link in
// place and failing later in a harder-to-diagnose way.
if os.IsNotExist(err) {
continue
}
return err
}
if fi.Mode()&os.ModeSymlink != 0 {
return w.Filesystem.Remove(dirs[i])
}
}
// Final component: an existing symlink here would be followed by the
// subsequent OpenFile/Symlink/MkdirAll, so replace it.
fi, err := w.Filesystem.Lstat(name)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
if fi.Mode()&os.ModeSymlink != 0 {
return w.Filesystem.Remove(name)
}
return nil
}
func (w *Worktree) checkoutFile(f *object.File) (err error) {
// checkoutFile is the materialisation boundary for tracked entries.
// Remove any blocking symlink first so the subsequent OpenFile or
// Symlink call writes the entry itself instead of following a planted
// final-component link in the underlying filesystem.
if err := w.clearBlockingSymlinks(f.Name); err != nil {
return err
}
mode, err := f.Mode.ToOSFileMode()
if err != nil {
return
+15 -100
View File
@@ -35,25 +35,10 @@ func defaultProtectNTFS() bool {
return true
}
// worktreeFilesystem wraps a billy.Filesystem and validates every path it
// is handed, so worktree operations cannot use dangerous paths at the
// boundary. Two layers apply:
//
// - validPath rejects dangerous path *strings*: .git and its HFS+/NTFS
// variants, "..", control characters, volume names.
// - validNoLeadingSymlink rejects paths whose leading directories
// already exist on disk as symlinks, so a write or delete cannot
// follow a planted link out of the tree.
//
// Both layers run on every mutating operation (validWritePath) and every
// read (validReadPath). Chroot additionally refuses a symlink as the final
// component, so a sub-filesystem such as a submodule worktree cannot be
// scoped to a redirected target.
//
// The wrapper intentionally stops at leading-component traversal. Callers
// that need final-component no-follow semantics for materialisation
// (checkoutFile) enforce that directly by removing the blocking symlink
// before opening the destination path.
// worktreeFilesystem wraps a billy.Filesystem and validates every path passed
// to a mutating operation. This prevents writing to, or deleting from,
// dangerous locations (e.g. .git/*, ../) regardless of which worktree
// code path triggers the operation.
type worktreeFilesystem struct {
billy.Filesystem
protectNTFS bool
@@ -65,7 +50,7 @@ func newWorktreeFilesystem(fs billy.Filesystem, protectNTFS, protectHFS bool) *w
}
func (sfs *worktreeFilesystem) Create(filename string) (billy.File, error) {
if err := sfs.validWritePath(filename); err != nil {
if err := sfs.validPath(filename); err != nil {
return nil, fmt.Errorf("create: %w", err)
}
return sfs.Filesystem.Create(filename)
@@ -79,7 +64,7 @@ func (sfs *worktreeFilesystem) Open(filename string) (billy.File, error) {
}
func (sfs *worktreeFilesystem) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) {
if err := sfs.validWritePath(filename); err != nil {
if err := sfs.validPath(filename); err != nil {
return nil, fmt.Errorf("openfile: %w", err)
}
return sfs.Filesystem.OpenFile(filename, flag, perm)
@@ -93,14 +78,14 @@ func (sfs *worktreeFilesystem) Stat(filename string) (os.FileInfo, error) {
}
func (sfs *worktreeFilesystem) Remove(filename string) error {
if err := sfs.validWritePath(filename); err != nil {
if err := sfs.validPath(filename); err != nil {
return fmt.Errorf("remove: %w", err)
}
return sfs.Filesystem.Remove(filename)
}
func (sfs *worktreeFilesystem) Rename(from, to string) error {
if err := sfs.validWritePath(from, to); err != nil {
if err := sfs.validPath(from, to); err != nil {
return fmt.Errorf("rename: %w", err)
}
return sfs.Filesystem.Rename(from, to)
@@ -121,7 +106,7 @@ func (sfs *worktreeFilesystem) Lstat(filename string) (os.FileInfo, error) {
}
func (sfs *worktreeFilesystem) Symlink(target, link string) error {
if err := sfs.validWritePath(link); err != nil {
if err := sfs.validPath(link); err != nil {
return fmt.Errorf("symlink: %w", err)
}
if err := sfs.validSymlinkName(link); err != nil {
@@ -146,7 +131,7 @@ func (sfs *worktreeFilesystem) MkdirAll(path string, perm os.FileMode) error {
if path == "" || path == "." || path == "/" {
return nil
}
if err := sfs.validWritePath(path); err != nil {
if err := sfs.validPath(path); err != nil {
return fmt.Errorf("mkdirall: %w", err)
}
return sfs.Filesystem.MkdirAll(path, perm)
@@ -160,39 +145,18 @@ func (sfs *worktreeFilesystem) Chroot(path string) (billy.Filesystem, error) {
if err := sfs.validReadPath(path); err != nil {
return nil, fmt.Errorf("chroot: %w", err)
}
// Chroot scopes a sub-filesystem to path, so the final component must
// be a real directory too: a symlink there would silently redirect the
// scope (e.g. a submodule worktree) to a target outside the tree. This
// is the "valid path, wrong target" case that validNoLeadingSymlink,
// which only inspects leading components, does not cover.
//
// A non-existent target is fine: Chroot creates it as a real
// directory. Any other Lstat error means we cannot prove the target
// is not a symlink, so fail closed rather than scope through it.
if fi, err := sfs.Filesystem.Lstat(path); err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("chroot: cannot stat %q: %w", path, err)
}
} else if fi.Mode()&os.ModeSymlink != 0 {
return nil, fmt.Errorf("chroot: invalid path %q: is a symlink", path)
}
return sfs.Filesystem.Chroot(path)
}
// validReadPath is like validWritePath but treats the empty string and "."
// as valid references to the worktree root. Read-side operations on the
// root (e.g. ReadDir(""), Lstat(".")) are legitimate. Mutating the root
// itself is not, so write-side operations reject it via validPath. Reads
// are still refused through a leading symlink, so the wrapper never
// follows a planted link even on the read surface.
// validReadPath is like validPath but treats the empty string and "." as
// valid references to the worktree root. Read-side operations on the root
// (e.g. ReadDir(""), Lstat(".")) are legitimate; mutating the root itself
// is not, so write-side operations continue to use validPath directly.
func (sfs *worktreeFilesystem) validReadPath(p string) error {
if p == "" || p == "." || p == "/" {
return nil
}
if err := sfs.validPath(p); err != nil {
return err
}
return sfs.validNoLeadingSymlink(p)
return sfs.validPath(p)
}
var errUnsupportedOperation = errors.New("unsupported operation")
@@ -268,55 +232,6 @@ func (sfs *worktreeFilesystem) validPath(paths ...string) error {
return nil
}
// validWritePath validates paths for mutating operations. It layers the
// filesystem-state check validNoLeadingSymlink on top of the string-only
// checks in validPath, so a write can neither name a dangerous path nor
// reach one by traversing an existing symlink. Every mutating method on
// the wrapper funnels through here, so the leading-symlink invariant holds
// for all worktree writers without each call site having to remember it.
func (sfs *worktreeFilesystem) validWritePath(paths ...string) error {
if err := sfs.validPath(paths...); err != nil {
return err
}
return sfs.validNoLeadingSymlink(paths...)
}
// validNoLeadingSymlink rejects paths whose leading directory components
// resolve through a symlink that already exists on the underlying
// filesystem. validPath guards the path string. This guards the on-disk
// state, so a write or delete cannot reach outside the worktree by
// traversing a symlink that a tree or an earlier step left in place.
//
// This is the fail-closed backstop for the whole class. Callers that want
// upstream's replace-and-continue behaviour (checkout) remove the blocking
// symlink first via clearBlockingSymlinks, so no symlink remains when the
// write reaches the wrapper. Callers that do not get a safe error,
// matching upstream Git refusing rather than following the link. See
// has_symlink_leading_path (symlinks.c) and the check_leading_path guard
// in unlink_entry (entry.c).
func (sfs *worktreeFilesystem) validNoLeadingSymlink(paths ...string) error {
for _, p := range paths {
for dir := filepath.Dir(p); dir != "." && dir != "" && dir != string(filepath.Separator); dir = filepath.Dir(dir) {
fi, err := sfs.Filesystem.Lstat(dir)
if err != nil {
// A missing ancestor is materialised as a real directory,
// so it cannot be a symlink and is safe to skip. Any other
// error (permission, I/O) means we cannot prove the
// component is not a symlink, so fail closed rather than
// let the operation traverse an unverified component.
if os.IsNotExist(err) {
continue
}
return fmt.Errorf("invalid path %q: cannot stat leading component %q: %w", p, dir, err)
}
if fi.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("invalid path %q: leading component %q is a symlink", p, dir)
}
}
}
return nil
}
// validSymlinkName checks the per-component name of a symlink for
// dotfile names that attackers can use to trick a checkout into
// writing a dangerous symlink. Each path component is compared
+1 -1
View File
@@ -371,7 +371,7 @@ func (w *Worktree) doAdd(path string, ignorePattern []gitignore.Pattern, skipSta
}
}
path = filepath.ToSlash(filepath.Clean(path))
path = filepath.Clean(path)
if err != nil || !fi.IsDir() {
added, h, err = w.doAddFile(idx, s, path, ignorePattern)
-1
View File
@@ -1,3 +1,2 @@
* -text
*.bin -text -diff
*.md text eol=lf
+700 -700
View File
File diff suppressed because it is too large Load Diff
+78 -78
View File
@@ -1,79 +1,79 @@
# Finite State Entropy
This package provides Finite State Entropy encoding and decoding.
Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS))
encoding provides a fast near-optimal symbol encoding/decoding
for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
## News
* Feb 2018: First implementation released. Consider this beta software for now.
# Usage
This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
You must provide input and will receive the output and maybe an error.
These error values can be returned:
| Error | Description |
|---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `(error)` | An internal error occurred. |
As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
# Performance
A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.
All compression functions are currently only running on the calling goroutine so only one core will be used per block.
The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be
beneficial to transpose all your input values down by 64.
With moderate block sizes around 64k speed are typically 200MB/s per core for compression and
around 300MB/s decompression speed.
The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s.
# Plans
At one point, more internals will be exposed to facilitate more "expert" usage of the components.
A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).
# Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
# Finite State Entropy
This package provides Finite State Entropy encoding and decoding.
Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS))
encoding provides a fast near-optimal symbol encoding/decoding
for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
## News
* Feb 2018: First implementation released. Consider this beta software for now.
# Usage
This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
You must provide input and will receive the output and maybe an error.
These error values can be returned:
| Error | Description |
|---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `(error)` | An internal error occurred. |
As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
# Performance
A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.
All compression functions are currently only running on the calling goroutine so only one core will be used per block.
The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be
beneficial to transpose all your input values down by 64.
With moderate block sizes around 64k speed are typically 200MB/s per core for compression and
around 300MB/s decompression speed.
The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s.
# Plans
At one point, more internals will be exposed to facilitate more "expert" usage of the components.
A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).
# Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
changes will likely not be accepted. If in doubt open an issue before writing the PR.
+89 -89
View File
@@ -1,89 +1,89 @@
# Huff0 entropy compression
This package provides Huff0 encoding and decoding as used in zstd.
[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders),
a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU
(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
## News
This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
This ensures that most functionality is well tested.
# Usage
This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and
[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
You must provide input and will receive the output and maybe an error.
These error values can be returned:
| Error | Description |
|---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) |
| `(error)` | An internal error occurred. |
As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.
## Tables and re-use
Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results.
The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy)
that controls this behaviour. See the documentation for details. This can be altered between each block.
Do however note that this information is *not* stored in the output block and it is up to the users of the package to
record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
based on the boolean reported back from the CompressXX call.
If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the
[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
## Decompressing
The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
This will initialize the decoding tables.
You can supply the complete block to `ReadTable` and it will return the data part of the block
which can be given to the decompressor.
Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X)
or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
# Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
changes will likely not be accepted. If in doubt open an issue before writing the PR.
# Huff0 entropy compression
This package provides Huff0 encoding and decoding as used in zstd.
[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders),
a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU
(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
## News
This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
This ensures that most functionality is well tested.
# Usage
This package provides a low level interface that allows to compress single independent blocks.
Each block is separate, and there is no built in integrity checks.
This means that the caller should keep track of block sizes and also do checksums if needed.
Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and
[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
You must provide input and will receive the output and maybe an error.
These error values can be returned:
| Error | Description |
|---------------------|-----------------------------------------------------------------------------|
| `<nil>` | Everything ok, output is returned |
| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) |
| `(error)` | An internal error occurred. |
As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object
that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
object can be used for both.
Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.
## Tables and re-use
Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results.
The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy)
that controls this behaviour. See the documentation for details. This can be altered between each block.
Do however note that this information is *not* stored in the output block and it is up to the users of the package to
record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
based on the boolean reported back from the CompressXX call.
If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the
[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
## Decompressing
The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
This will initialize the decoding tables.
You can supply the complete block to `ReadTable` and it will return the data part of the block
which can be given to the decompressor.
Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X)
or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
your input was likely corrupted.
It is important to note that a successful decoding does *not* mean your output matches your original input.
There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
# Contributing
Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
changes will likely not be accepted. If in doubt open an issue before writing the PR.
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.27@sha256:512690a5660563b57d37ecc31129e7f136e831db2aed24a1dbeb8ad7380dc0fa
FROM golang:1.26@sha256:f96cc555eb8db430159a3aa6797cd5bae561945b7b0fe7d0e284c63a3b291609
ENV GOOS=linux
ENV GOARCH=arm
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.27@sha256:512690a5660563b57d37ecc31129e7f136e831db2aed24a1dbeb8ad7380dc0fa
FROM golang:1.26@sha256:f96cc555eb8db430159a3aa6797cd5bae561945b7b0fe7d0e284c63a3b291609
ENV GOOS=linux
ENV GOARCH=arm64
@@ -24,4 +24,4 @@ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -8,14 +8,11 @@
//
// - unified_diff
//
// - context_diff
//
// Getting unified diffs was the main goal of the port. Keep in mind this code
// is mostly suitable to output text differences in a human friendly way, there
// are no guarantees generated diffs are consumable by patch(1).
//
// This package was adopted from [github.com/pmezard/go-difflib] which
// is no longer maintained.
//
// [github.com/pmezard/go-difflib]: https://github.com/pmezard/go-difflib
package difflib
import (
@@ -40,6 +37,13 @@ func max(a, b int) int {
return b
}
func calculateRatio(matches, length int) float64 {
if length > 0 {
return 2.0 * float64(matches) / float64(length)
}
return 1.0
}
type Match struct {
A int
B int
@@ -99,6 +103,14 @@ func NewMatcher(a, b []string) *SequenceMatcher {
return &m
}
func NewMatcherWithJunk(a, b []string, autoJunk bool,
isJunk func(string) bool) *SequenceMatcher {
m := SequenceMatcher{IsJunk: isJunk, autoJunk: autoJunk}
m.SetSeqs(a, b)
return &m
}
// Set two sequences to be compared.
func (m *SequenceMatcher) SetSeqs(a, b []string) {
m.SetSeq1(a)
@@ -187,15 +199,12 @@ func (m *SequenceMatcher) isBJunk(s string) bool {
// If IsJunk is not defined:
//
// Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where
//
// alo <= i <= i+k <= ahi
// blo <= j <= j+k <= bhi
//
// alo <= i <= i+k <= ahi
// blo <= j <= j+k <= bhi
// and for all (i',j',k') meeting those conditions,
//
// k >= k'
// i <= i'
// and if i == i', j <= j'
// k >= k'
// i <= i'
// and if i == i', j <= j'
//
// In other words, of all maximal matching blocks, return one that
// starts earliest in a, and of all those maximal matching blocks that
@@ -442,6 +451,66 @@ func (m *SequenceMatcher) GetGroupedOpCodes(n int) [][]OpCode {
return groups
}
// Return a measure of the sequences' similarity (float in [0,1]).
//
// Where T is the total number of elements in both sequences, and
// M is the number of matches, this is 2.0*M / T.
// Note that this is 1 if the sequences are identical, and 0 if
// they have nothing in common.
//
// .Ratio() is expensive to compute if you haven't already computed
// .GetMatchingBlocks() or .GetOpCodes(), in which case you may
// want to try .QuickRatio() or .RealQuickRation() first to get an
// upper bound.
func (m *SequenceMatcher) Ratio() float64 {
matches := 0
for _, m := range m.GetMatchingBlocks() {
matches += m.Size
}
return calculateRatio(matches, len(m.a)+len(m.b))
}
// Return an upper bound on ratio() relatively quickly.
//
// This isn't defined beyond that it is an upper bound on .Ratio(), and
// is faster to compute.
func (m *SequenceMatcher) QuickRatio() float64 {
// viewing a and b as multisets, set matches to the cardinality
// of their intersection; this counts the number of matches
// without regard to order, so is clearly an upper bound
if m.fullBCount == nil {
m.fullBCount = map[string]int{}
for _, s := range m.b {
m.fullBCount[s] = m.fullBCount[s] + 1
}
}
// avail[x] is the number of times x appears in 'b' less the
// number of times we've seen it in 'a' so far ... kinda
avail := map[string]int{}
matches := 0
for _, s := range m.a {
n, ok := avail[s]
if !ok {
n = m.fullBCount[s]
}
avail[s] = n - 1
if n > 0 {
matches += 1
}
}
return calculateRatio(matches, len(m.a)+len(m.b))
}
// Return an upper bound on ratio() very quickly.
//
// This isn't defined beyond that it is an upper bound on .Ratio(), and
// is faster to compute than either .Ratio() or .QuickRatio().
func (m *SequenceMatcher) RealQuickRatio() float64 {
la, lb := len(m.a), len(m.b)
return calculateRatio(min(la, lb), la+lb)
}
// Convert range to the "ed" format
func formatRangeUnified(start, stop int) string {
// Per the diff spec at http://www.unix.org/single_unix_specification/
@@ -583,6 +652,117 @@ func formatRangeContext(start, stop int) string {
return fmt.Sprintf("%d,%d", beginning, beginning+length-1)
}
type ContextDiff UnifiedDiff
// Compare two sequences of lines; generate the delta as a context diff.
//
// Context diffs are a compact way of showing line changes and a few
// lines of context. The number of context lines is set by diff.Context
// which defaults to three.
//
// By default, the diff control lines (those with *** or ---) are
// created with a trailing newline.
//
// For inputs that do not have trailing newlines, set the diff.Eol
// argument to "" so that the output will be uniformly newline free.
//
// The context diff format normally has a header for filenames and
// modification times. Any or all of these may be specified using
// strings for diff.FromFile, diff.ToFile, diff.FromDate, diff.ToDate.
// The modification times are normally expressed in the ISO 8601 format.
// If not specified, the strings default to blanks.
func WriteContextDiff(writer io.Writer, diff ContextDiff) error {
buf := bufio.NewWriter(writer)
defer buf.Flush()
var diffErr error
wf := func(format string, args ...interface{}) {
_, err := buf.WriteString(fmt.Sprintf(format, args...))
if diffErr == nil && err != nil {
diffErr = err
}
}
ws := func(s string) {
_, err := buf.WriteString(s)
if diffErr == nil && err != nil {
diffErr = err
}
}
if len(diff.Eol) == 0 {
diff.Eol = "\n"
}
prefix := map[byte]string{
'i': "+ ",
'd': "- ",
'r': "! ",
'e': " ",
}
started := false
m := NewMatcher(diff.A, diff.B)
for _, g := range m.GetGroupedOpCodes(diff.Context) {
if !started {
started = true
fromDate := ""
if len(diff.FromDate) > 0 {
fromDate = "\t" + diff.FromDate
}
toDate := ""
if len(diff.ToDate) > 0 {
toDate = "\t" + diff.ToDate
}
if diff.FromFile != "" || diff.ToFile != "" {
wf("*** %s%s%s", diff.FromFile, fromDate, diff.Eol)
wf("--- %s%s%s", diff.ToFile, toDate, diff.Eol)
}
}
first, last := g[0], g[len(g)-1]
ws("***************" + diff.Eol)
range1 := formatRangeContext(first.I1, last.I2)
wf("*** %s ****%s", range1, diff.Eol)
for _, c := range g {
if c.Tag == 'r' || c.Tag == 'd' {
for _, cc := range g {
if cc.Tag == 'i' {
continue
}
for _, line := range diff.A[cc.I1:cc.I2] {
ws(prefix[cc.Tag] + line)
}
}
break
}
}
range2 := formatRangeContext(first.J1, last.J2)
wf("--- %s ----%s", range2, diff.Eol)
for _, c := range g {
if c.Tag == 'r' || c.Tag == 'i' {
for _, cc := range g {
if cc.Tag == 'd' {
continue
}
for _, line := range diff.B[cc.J1:cc.J2] {
ws(prefix[cc.Tag] + line)
}
}
break
}
}
}
return diffErr
}
// Like WriteContextDiff but returns the diff a string.
func GetContextDiffString(diff ContextDiff) (string, error) {
w := &bytes.Buffer{}
err := WriteContextDiff(w, diff)
return string(w.Bytes()), err
}
// Split a string on "\n" while preserving them. The output can be used
// as input for UnifiedDiff and ContextDiff structures.
func SplitLines(s string) []string {
-15
View File
@@ -1,15 +0,0 @@
*
!.git/
!build/root.tgz
!cmd/
!config/
!internal/
!mod/
!pkg/
!regclient/
!scheme/
!types/
!vendor/
!go.*
!*.go
!Makefile
-5
View File
@@ -1,5 +0,0 @@
artifacts/
bin/
output/
vendor/
.regctl_conf_ci.json
-19
View File
@@ -1,19 +0,0 @@
# all lists use a `-`
MD004:
style: dash
# allow tabs in code blocks (for Go)
MD010:
code_blocks: false
# disable line length, prefer one sentence per line for PRs
MD013: false
# emphasis with underscore (`_emphasis_`)
MD049:
style: "underscore"
# bold with asterisk (`**bold**`)
MD050:
style: "asterisk"
-1
View File
@@ -1 +0,0 @@
GoVersionOverride = "1.26.3"
-53
View File
@@ -1,53 +0,0 @@
{"name":"docker-arg-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
{"name":"docker-arg-alpine-tag","key":"docker.io/library/alpine","version":"3.23.4"}
{"name":"docker-arg-ecr","key":"https://github.com/awslabs/amazon-ecr-credential-helper.git","version":"v0.12.0"}
{"name":"docker-arg-gcr","key":"https://github.com/GoogleCloudPlatform/docker-credential-gcr.git","version":"v2.1.32"}
{"name":"docker-arg-go-digest","key":"docker.io/library/golang:1.26.3-alpine","version":"sha256:91eda9776261207ea25fd06b5b7fed8d397dd2c0a283e77f2ab6e91bfa71079d"}
{"name":"docker-arg-go-tag","key":"docker.io/library/golang","version":"1.26.3"}
{"name":"docker-arg-lunajson","key":"https://github.com/grafi-tt/lunajson.git:master","version":"e3a9666eb1275741e887e29926b144f8daee3bef"}
{"name":"docker-arg-semver","key":"https://github.com/kikito/semver.lua.git:master","version":"a4b708ba243208d46e575da870af969dca46a94d"}
{"name":"gha-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
{"name":"gha-alpine-tag-base","key":"docker.io/library/alpine","version":"3"}
{"name":"gha-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.23.4"}
{"name":"gha-cosign-version","key":"https://github.com/sigstore/cosign.git","version":"v3.0.6"}
{"name":"gha-golang-matrix","key":"golang-matrix","version":"[\"1.25\", \"1.26\"]"}
{"name":"gha-golang-release","key":"golang-latest","version":"1.26"}
{"name":"gha-syft-version","key":"docker.io/anchore/syft","version":"v1.44.0"}
{"name":"gha-uses-commit","key":"https://github.com/actions/checkout.git:v6.0.2","version":"de0fac2e4500dabe0009e67214ff5f5447ce83dd"}
{"name":"gha-uses-commit","key":"https://github.com/actions/setup-go.git:v6.4.0","version":"4a3601121dd01d1626a1e23e37211e3254c1c06c"}
{"name":"gha-uses-commit","key":"https://github.com/actions/stale.git:v10.3.0","version":"eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899"}
{"name":"gha-uses-commit","key":"https://github.com/actions/upload-artifact.git:v7.0.1","version":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"}
{"name":"gha-uses-commit","key":"https://github.com/anchore/sbom-action.git:v0.24.0","version":"e22c389904149dbc22b58101806040fa8d37a610"}
{"name":"gha-uses-commit","key":"https://github.com/docker/build-push-action.git:v7.2.0","version":"f9f3042f7e2789586610d6e8b85c8f03e5195baf"}
{"name":"gha-uses-commit","key":"https://github.com/docker/login-action.git:v4.2.0","version":"650006c6eb7dba73a995cc03b0b2d7f5ca915bee"}
{"name":"gha-uses-commit","key":"https://github.com/docker/setup-buildx-action.git:v4.1.0","version":"d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5"}
{"name":"gha-uses-commit","key":"https://github.com/regclient/actions.git:main","version":"c70ad64367908075211b10dcd2ab9fad4bfa1816"}
{"name":"gha-uses-commit","key":"https://github.com/sigstore/cosign-installer.git:v4.1.2","version":"6f9f17788090df1f26f669e9d70d6ae9567deba6"}
{"name":"gha-uses-commit","key":"https://github.com/softprops/action-gh-release.git:v3.0.0","version":"b4309332981a82ec1c5618f44dd2e27cc8bfbfda"}
{"name":"gha-uses-semver","key":"https://github.com/actions/checkout.git","version":"v6.0.2"}
{"name":"gha-uses-semver","key":"https://github.com/actions/setup-go.git","version":"v6.4.0"}
{"name":"gha-uses-semver","key":"https://github.com/actions/stale.git","version":"v10.3.0"}
{"name":"gha-uses-semver","key":"https://github.com/actions/upload-artifact.git","version":"v7.0.1"}
{"name":"gha-uses-semver","key":"https://github.com/anchore/sbom-action.git","version":"v0.24.0"}
{"name":"gha-uses-semver","key":"https://github.com/docker/build-push-action.git","version":"v7.2.0"}
{"name":"gha-uses-semver","key":"https://github.com/docker/login-action.git","version":"v4.2.0"}
{"name":"gha-uses-semver","key":"https://github.com/docker/setup-buildx-action.git","version":"v4.1.0"}
{"name":"gha-uses-semver","key":"https://github.com/sigstore/cosign-installer.git","version":"v4.1.2"}
{"name":"gha-uses-semver","key":"https://github.com/softprops/action-gh-release.git","version":"v3.0.0"}
{"name":"go-mod-golang-release","key":"golang-oldest","version":"1.25.0"}
{"name":"makefile-ci-distribution","key":"docker.io/library/registry","version":"3.1.1"}
{"name":"makefile-ci-zot","key":"ghcr.io/project-zot/zot-linux-amd64","version":"v2.1.17"}
{"name":"makefile-go-vulncheck","key":"https://go.googlesource.com/vuln.git","version":"v1.3.0"}
{"name":"makefile-gofumpt","key":"https://github.com/mvdan/gofumpt.git","version":"v0.10.0"}
{"name":"makefile-gomajor","key":"https://github.com/icholy/gomajor.git","version":"v0.15.0"}
{"name":"makefile-gosec","key":"https://github.com/securego/gosec.git","version":"v2.26.1"}
{"name":"makefile-markdown-lint","key":"docker.io/davidanson/markdownlint-cli2","version":"v0.22.1"}
{"name":"makefile-osv-scanner","key":"https://github.com/google/osv-scanner.git","version":"v2.3.8"}
{"name":"makefile-staticcheck","key":"https://github.com/dominikh/go-tools.git","version":"v0.7.0"}
{"name":"makefile-syft-container-digest","key":"anchore/syft:v1.44.0","version":"sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6"}
{"name":"makefile-syft-container-tag","key":"anchore/syft","version":"v1.44.0"}
{"name":"makefile-syft-version","key":"docker.io/anchore/syft","version":"v1.44.0"}
{"name":"osv-golang-release","key":"docker.io/library/golang","version":"1.26.3"}
{"name":"shell-alpine-digest","key":"docker.io/library/alpine:3.23.4","version":"sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11"}
{"name":"shell-alpine-tag-base","key":"docker.io/library/alpine","version":"3"}
{"name":"shell-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.23.4"}
-346
View File
@@ -1,346 +0,0 @@
files:
"build/Dockerfile*":
processors:
- docker-arg-alpine-tag
- docker-arg-alpine-digest
- docker-arg-go-tag
- docker-arg-go-digest
- docker-arg-ecr
- docker-arg-gcr
- docker-arg-lunajson
- docker-arg-semver
"build/oci-image.sh":
processors:
- shell-alpine-tag-base
- shell-alpine-tag-comment
- shell-alpine-digest
".github/workflows/*.yml":
processors:
- gha-golang-matrix
- gha-golang-release
- gha-uses-vx
- gha-uses-semver
- gha-uses-commit
- gha-syft-version
- gha-cosign-version
- gha-alpine-tag-base
- gha-alpine-tag-comment
- gha-alpine-digest
"Makefile":
processors:
- makefile-gofumpt
- makefile-gomajor
- makefile-go-vulncheck
- makefile-markdown-lint
- makefile-gosec
- makefile-osv-scanner
- makefile-staticcheck
- makefile-syft-version
- makefile-syft-container-tag
- makefile-syft-container-digest
- makefile-ci-distribution
- makefile-ci-zot
"go.mod":
processors:
- go-mod-golang-release
".osv-scanner.toml":
processors:
- osv-golang-release
x-processor-tmpl:
git-commit: &git-commit
key: "{{ .SourceArgs.url }}:{{ .SourceArgs.ref }}"
scan: "regexp"
source: "git-commit"
filter:
expr: "^{{ .SourceArgs.ref }}$"
git-tag-semver: &git-tag-semver
key: "{{ .SourceArgs.url }}"
scan: "regexp"
source: "git-tag"
filter:
expr: '^v?\d+\.\d+\.\d+$'
sort:
method: "semver"
registry-digest: &registry-digest
key: "{{ .SourceArgs.image }}"
scan: "regexp"
source: "registry-digest"
registry-tag-semver: &registry-tag-semver
key: "{{ .SourceArgs.repo }}"
scan: "regexp"
source: "registry-tag"
filter:
expr: '^v?\d+\.\d+\.\d+$'
sort:
method: "semver"
processors:
docker-arg-alpine-tag:
<<: *registry-tag-semver
scanArgs:
regexp: '^ARG ALPINE_VER=(?P<Version>v?\d+\.\d+\.\d+)@(?P<SHA>sha256:[0-9a-f]+)\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
docker-arg-alpine-digest:
<<: *registry-digest
scanArgs:
regexp: '^ARG ALPINE_VER=(?P<Tag>v?\d+\.\d+\.\d+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
sourceArgs:
image: "docker.io/library/alpine:{{.ScanMatch.Tag}}"
docker-arg-go-tag:
<<: *registry-tag-semver
scanArgs:
regexp: '^ARG GO_VER=(?P<Version>[a-z0-9\-\.]+)-alpine@(?P<SHA>sha256:[0-9a-f]+)\s*$'
sourceArgs:
repo: "docker.io/library/golang"
docker-arg-go-digest:
<<: *registry-digest
scanArgs:
regexp: '^ARG GO_VER=(?P<Tag>[a-z0-9\-\.]+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
sourceArgs:
image: "docker.io/library/golang:{{.ScanMatch.Tag}}"
docker-arg-ecr:
<<: *git-tag-semver
scanArgs:
regexp: '^ARG ECR_HELPER_VER=(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
url: "https://github.com/awslabs/amazon-ecr-credential-helper.git"
# get the version for the ecr-login nested package in the repo
filter:
expr: '^ecr-login/v?\d+\.\d+\.\d+$'
# sort and output only the version number without the package name prefix
sort:
method: "semver"
template: '{{ index (split . "/") 1 }}'
template: '{{ index (split .Version "/") 1 }}'
docker-arg-gcr:
<<: *git-tag-semver
scanArgs:
regexp: '^ARG GCR_HELPER_VER=(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
url: "https://github.com/GoogleCloudPlatform/docker-credential-gcr.git"
docker-arg-lunajson:
<<: *git-commit
scanArgs:
regexp: '^ARG LUNAJSON_COMMIT=(?P<Version>[0-9a-f]+)\s*$'
sourceArgs:
url: "https://github.com/grafi-tt/lunajson.git"
ref: master
docker-arg-semver:
<<: *git-commit
scanArgs:
regexp: '^ARG SEMVER_COMMIT=(?P<Version>[0-9a-f]+)\s*$'
sourceArgs:
url: "https://github.com/kikito/semver.lua.git"
ref: master
gha-alpine-digest:
<<: *registry-digest
scanArgs:
regexp: '^\s*ALPINE_DIGEST: "(?P<Version>sha256:[0-9a-f]+)"\s*#\s*(?P<Tag>\d+\.\d+\.\d+)\s*$'
sourceArgs:
image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}"
gha-alpine-tag-base:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_NAME: "alpine:(?P<Version>v?\d+)"\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
# only return the major version number in the tag to support detecting a change in the base image
template: '{{ index ( split .Version "." ) 0 }}'
gha-alpine-tag-comment:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_DIGEST: "(?P<Digest>sha256:[0-9a-f]+)"\s*#\s*(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
gha-cosign-version:
<<: *git-tag-semver
scanArgs:
regexp: '^\s*cosign-release: "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
url: "https://github.com/sigstore/cosign.git"
filter:
expr: '^v?3\.\d+\.\d+$' # pin to v3, v4 will remove support for older clients
gha-golang-matrix:
<<: *registry-tag-semver
key: "golang-matrix"
scanArgs:
regexp: '^\s*gover: (?P<Version>\[["0-9, \.]+\])\s*$'
sourceArgs:
repo: "docker.io/library/golang"
filter:
expr: '^v?\d+\.\d+$'
template: '["{{ index .VerMap ( index .VerList 1 ) }}", "{{ index .VerMap ( index .VerList 0 ) }}"]'
gha-golang-release:
<<: *registry-tag-semver
key: "golang-latest"
scanArgs:
regexp: '^\s*RELEASE_GO_VER: "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
repo: "docker.io/library/golang"
filter:
expr: '^v?\d+\.\d+$'
gha-syft-version:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*syft-version: "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
repo: "docker.io/anchore/syft"
gha-uses-vx:
<<: *git-tag-semver
scanArgs:
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Commit>[0-9a-f]+)\s+#\s+(?P<Version>v?\d+)\s*$'
sourceArgs:
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
filter:
expr: '^v?\d+$'
gha-uses-semver:
<<: *git-tag-semver
scanArgs:
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Commit>[0-9a-f]+)\s+#\s+(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
gha-uses-commit:
<<: *git-commit
scanArgs:
regexp: '^\s+-?\s+uses: (?P<Repo>[^@/]+/[^@/]+)[^@]*@(?P<Version>[0-9a-f]+)\s+#\s+(?P<Ref>[\w\d\.]+)\s*$'
sourceArgs:
url: "https://github.com/{{ .ScanMatch.Repo }}.git"
ref: "{{ .ScanMatch.Ref }}"
go-mod-golang-release:
<<: *registry-tag-semver
key: "golang-oldest"
scanArgs:
regexp: '^go (?P<Version>[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/library/golang"
filter:
expr: '^\d+\.\d+$'
template: '{{ index .VerMap ( index .VerList 1 ) }}.0'
makefile-ci-distribution:
<<: *registry-tag-semver
scanArgs:
regexp: '^CI_DISTRIBUTION_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/library/registry"
makefile-ci-zot:
<<: *registry-tag-semver
scanArgs:
regexp: '^CI_ZOT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
repo: "ghcr.io/project-zot/zot-linux-amd64"
makefile-gofumpt:
<<: *git-tag-semver
scanArgs:
regexp: '^GOFUMPT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/mvdan/gofumpt.git"
makefile-gomajor:
<<: *git-tag-semver
scanArgs:
regexp: '^GOMAJOR_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/icholy/gomajor.git"
makefile-gosec:
<<: *git-tag-semver
scanArgs:
regexp: '^GOSEC_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/securego/gosec.git"
makefile-go-vulncheck:
<<: *git-tag-semver
scanArgs:
regexp: '^GO_VULNCHECK_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://go.googlesource.com/vuln.git"
makefile-markdown-lint:
<<: *registry-tag-semver
scanArgs:
regexp: '^MARKDOWN_LINT_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/davidanson/markdownlint-cli2"
makefile-osv-scanner:
<<: *git-tag-semver
scanArgs:
regexp: '^OSV_SCANNER_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/google/osv-scanner.git"
makefile-staticcheck:
<<: *git-tag-semver
scanArgs:
regexp: '^STATICCHECK_VER\?=(?P<Version>v?[0-9\.]+)\s*$'
sourceArgs:
url: "https://github.com/dominikh/go-tools.git"
filter:
# repo also has dated tags, ignore versions without a preceding "v"
expr: '^v\d+\.\d+\.\d+$'
makefile-syft-container-tag:
<<: *registry-tag-semver
scanArgs:
regexp: '^SYFT_CONTAINER\?=(?P<Repo>[^:]*):(?P<Version>v?[0-9\.]+)@(?P<Digest>sha256:[0-9a-f]+)\s*$'
sourceArgs:
repo: "{{ .ScanMatch.Repo }}"
makefile-syft-container-digest:
<<: *registry-digest
scanArgs:
regexp: '^SYFT_CONTAINER\?=(?P<Image>[^:]*):(?P<Tag>v?[0-9\.]+)@(?P<Version>sha256:[0-9a-f]+)\s*$'
sourceArgs:
image: "{{ .ScanMatch.Image }}:{{.ScanMatch.Tag}}"
makefile-syft-version:
<<: *registry-tag-semver
scanArgs:
regexp: '^SYFT_VERSION\?=(?P<Version>v[0-9\.]+)\s*$'
sourceArgs:
repo: "docker.io/anchore/syft"
osv-golang-release:
<<: *registry-tag-semver
scanArgs:
regexp: '^GoVersionOverride = "(?P<Version>v?[0-9\.]+)"\s*$'
sourceArgs:
repo: "docker.io/library/golang"
shell-alpine-tag-base:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_NAME="alpine:(?P<Version>v?\d+)"\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
# only return the major version number in the tag to support detecting a change in the base image
template: '{{ index ( split .Version "." ) 0 }}'
shell-alpine-tag-comment:
<<: *registry-tag-semver
scanArgs:
regexp: '^\s*ALPINE_DIGEST="(?P<Digest>sha256:[0-9a-f]+)"\s*#\s*(?P<Version>v?\d+\.\d+\.\d+)\s*$'
sourceArgs:
repo: "docker.io/library/alpine"
shell-alpine-digest:
<<: *registry-digest
scanArgs:
regexp: '^\s*ALPINE_DIGEST="(?P<Version>sha256:[0-9a-f]+)"\s*#\s*(?P<Tag>\d+\.\d+\.\d+)\s*$'
sourceArgs:
image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}"
scans:
regexp:
type: "regexp"
sources:
git-commit:
type: "git"
args:
type: "commit"
git-tag:
type: "git"
args:
type: "tag"
registry-digest:
type: "registry"
registry-tag:
type: "registry"
args:
type: "tag"
-134
View File
@@ -1,134 +0,0 @@
# Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
- Demonstrating empathy and kindness toward other people
- Being respectful of differing opinions, viewpoints, and experiences
- Giving and gracefully accepting constructive feedback
- Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
- Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior include:
- The use of sexualized language or imagery, and sexual attention or advances of
any kind
- Trolling, insulting or derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or email address,
without their explicit permission
- Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
<git@bmitch.net> or slack (I'm found on the CNCF, Docker, OCI, and OpenSSF
slacks).
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
[homepage]: https://www.contributor-covenant.org
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[Mozilla CoC]: https://github.com/mozilla/diversity
[FAQ]: https://www.contributor-covenant.org/faq
[translations]: https://www.contributor-covenant.org/translations
-80
View File
@@ -1,80 +0,0 @@
# Contributing
## Reporting security issues
Please see [SECURITY.md](security.md) for the process to report security issues.
## Reporting other issues
Please search for similar issues and if none are seen, report an issue at [github.com/regclient/regclient/issues](https://github.com/regclient/regclient/issues).
## Code style
This project attempts to follow these principles:
- Code is canonical Go, following styles and patterns commonly used by the Go community.
- Dependencies outside of the Go standard library should be minimized.
- Dependencies should be pinned to a specific digest and tracked by Go or version-check.
- Unit tests are strongly encouraged with a focus on test coverage of the successful path and common errors.
- Linters and other style formatting tools are used, please run `make all` before committing any changes.
## LLM Policy
This project expects all contributions to be developed by a human or created with a reproducible tool.
Developers using an AI/LLM tool to generate their contribution are expected to fully understand the entire contribution and the logic behind its design.
Contributions that appear to have been generated by an AI/LLM without a human review may result in a ban from future contributions to the project.
## Pull requests
PRs are welcome following the below guides:
- For anything beyond a minor fix, opening an issue is suggested to discuss possible solutions.
- Changes should be rebased on the main branch.
- Changes should be squashed to a single commit per logical change.
All changes must be signed (`git commit -s`) to indicate you agree to the [Developer Certificate or Origin](https://developercertificate.org/):
```text
Developer Certificate of Origin
Version 1.1
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
Everyone is permitted to copy and distribute verbatim copies of this
license document, but changing it is not allowed.
Developer's Certificate of Origin 1.1
By making a contribution to this project, I certify that:
(a) The contribution was created in whole or in part by me and I
have the right to submit it under the open source license
indicated in the file; or
(b) The contribution is based upon previous work that, to the best
of my knowledge, is covered under an appropriate open source
license and I have the right under that license to submit that
work with modifications, whether created in whole or in part
by me, under the same open source license (unless I am
permitted to submit under a different license), as indicated
in the file; or
(c) The contribution was provided directly to me by some other
person who certified (a), (b) or (c) and I have not modified
it.
(d) I understand and agree that this project and the contribution
are public and that a record of the contribution (including all
personal information I submit with it, including my sign-off) is
maintained indefinitely and may be redistributed consistent with
this project or the open source license(s) involved.
```
The sign-off will include the following message in your commit:
```text
Signed-off-by: Your Name <your-email@example.org>
```
This needs to be your real name, no aliases please.
-191
View File
@@ -1,191 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
Copyright 2020 The regclient Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-285
View File
@@ -1,285 +0,0 @@
COMMANDS?=regctl regsync regbot
BINARIES?=$(addprefix bin/,$(COMMANDS))
IMAGES?=$(addprefix docker-,$(COMMANDS))
ARTIFACT_PLATFORMS?=linux-amd64 linux-arm64 linux-ppc64le linux-s390x linux-riscv64 darwin-amd64 darwin-arm64 windows-amd64.exe freebsd-amd64
ARTIFACTS?=$(foreach cmd,$(addprefix artifacts/,$(COMMANDS)),$(addprefix $(cmd)-,$(ARTIFACT_PLATFORMS)))
IMAGE_PLATFORMS?=linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/riscv64
VCS_REPO?="https://github.com/regclient/regclient.git"
VCS_REF?=$(shell git rev-list -1 HEAD)
ifneq ($(shell git status --porcelain 2>/dev/null),)
VCS_REF := $(VCS_REF)-dirty
endif
VCS_VERSION?=$(shell vcs_describe="$$(git describe --all)"; \
vcs_version="(devel)"; \
if [ "$${vcs_describe}" != "$${vcs_describe#tags/}" ]; then \
vcs_version="$${vcs_describe#tags/}"; \
elif [ "$${vcs_describe}" != "$${vcs_describe#heads/}" ]; then \
vcs_version="$${vcs_describe#heads/}"; \
if [ "main" = "$${vcs_version}" ]; then vcs_version=edge; fi; \
fi; \
echo "$${vcs_version}" | sed -r 's#/+#-#g')
VCS_TAG?=$(shell git describe --tags --abbrev=0 2>/dev/null || true)
VCS_SEC?=$(shell git log -1 --format=%ct)
VCS_DATE?=$(shell date -d "@$(VCS_SEC)" +%Y-%m-%dT%H:%M:%SZ --utc)
LD_FLAGS?=-s -w -extldflags -static -buildid= -X \"github.com/regclient/regclient/internal/version.vcsTag=$(VCS_TAG)\"
GO_BUILD_FLAGS?=-trimpath -ldflags "$(LD_FLAGS)"
DOCKERFILE_EXT?=$(shell if docker build --help 2>/dev/null | grep -q -- '--progress'; then echo ".buildkit"; fi)
DOCKER_ARGS?=--build-arg "VCS_REF=$(VCS_REF)" --build-arg "VCS_VERSION=$(VCS_VERSION)" --build-arg "SOURCE_DATE_EPOCH=$(VCS_SEC)" --build-arg "BUILD_DATE=$(VCS_DATE)"
GOPATH?=$(shell go env GOPATH)
PWD:=$(shell pwd)
VER_BUMP?=$(shell command -v version-bump 2>/dev/null)
VER_BUMP_CONTAINER?=sudobmitch/version-bump:edge
ifeq "$(strip $(VER_BUMP))" ''
VER_BUMP=docker run --rm \
-v "$(shell pwd)/:$(shell pwd)/" -w "$(shell pwd)" \
-u "$(shell id -u):$(shell id -g)" \
$(VER_BUMP_CONTAINER)
endif
MARKDOWN_LINT_VER?=v0.22.1
GOFUMPT_VER?=v0.10.0
GOMAJOR_VER?=v0.15.0
GOSEC_VER?=v2.26.1
GO_VULNCHECK_VER?=v1.3.0
OSV_SCANNER_VER?=v2.3.8
SYFT?=$(shell command -v syft 2>/dev/null)
SYFT_CMD_VER:=$(shell [ -x "$(SYFT)" ] && echo "v$$($(SYFT) version | awk '/^Version: / {print $$2}')" || echo "0")
SYFT_VERSION?=v1.44.0
SYFT_CONTAINER?=anchore/syft:v1.44.0@sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6
ifneq "$(SYFT_CMD_VER)" "$(SYFT_VERSION)"
SYFT=docker run --rm \
-v "$(shell pwd)/:$(shell pwd)/" -w "$(shell pwd)" \
-u "$(shell id -u):$(shell id -g)" \
$(SYFT_CONTAINER)
endif
STATICCHECK_VER?=v0.7.0
CI_DISTRIBUTION_VER?=3.1.1
CI_ZOT_VER?=v2.1.17
.PHONY: .FORCE
.FORCE:
.PHONY: all
all: fmt gofumpt gofix goimports vet test lint binaries ## Full build of Go binaries (including fmt, vet, test, and lint)
.PHONY: fmt
fmt: ## go fmt
go fmt ./...
.PHONY: gofumpt
gofumpt: $(GOPATH)/bin/gofumpt ## gofumpt is a stricter alternative to go fmt
gofumpt -l -w .
.PHONY: gofix
gofix: ## go fix
go fix ./...
goimports: $(GOPATH)/bin/goimports
$(GOPATH)/bin/goimports -w -format-only -local github.com/regclient .
.PHONY: vet
vet: ## go vet
go vet ./...
.PHONY: test
test: ## go test
go test -cover -race ./...
.PHONY: lint
lint: lint-go lint-goimports lint-md lint-gosec ## Run all linting
.PHONY: lint-go
lint-go: $(GOPATH)/bin/gofumpt $(GOPATH)/bin/staticcheck .FORCE ## Run linting for Go
$(GOPATH)/bin/staticcheck -checks all ./...
$(GOPATH)/bin/gofumpt -l -d .
errors=$$(go fix -diff ./...); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi
lint-goimports: $(GOPATH)/bin/goimports
@if [ -n "$$($(GOPATH)/bin/goimports -l -format-only -local github.com/regclient .)" ]; then \
echo $(GOPATH)/bin/goimports -d -format-only -local github.com/regclient .; \
$(GOPATH)/bin/goimports -d -format-only -local github.com/regclient .; \
exit 1; \
fi
# excluding types/platform pending resultion to https://github.com/securego/gosec/issues/1116
.PHONY: lint-gosec
lint-gosec: $(GOPATH)/bin/gosec .FORCE ## Run gosec
$(GOPATH)/bin/gosec -terse -exclude-dir types/platform ./...
.PHONY: lint-md
lint-md: .FORCE ## Run linting for markdown
docker run --rm -v "$(PWD):/workdir:ro" davidanson/markdownlint-cli2:$(MARKDOWN_LINT_VER) \
"**/*.md" "#vendor"
.PHONY: vulnerability-scan
vulnerability-scan: osv-scanner vulncheck-go ## Run all vulnerability scanners
.PHONY: osv-scanner
osv-scanner: $(GOPATH)/bin/osv-scanner .FORCE ## Run OSV Scanner
$(GOPATH)/bin/osv-scanner scan --config .osv-scanner.toml -r --licenses="Apache-2.0,BSD-3-Clause,MIT,CC-BY-SA-4.0,UNKNOWN" .
.PHONY: vulncheck-go
vulncheck-go: $(GOPATH)/bin/govulncheck .FORCE ## Run govulncheck
$(GOPATH)/bin/govulncheck ./...
.PHONY: vendor
vendor: ## Vendor Go modules
go mod vendor
.PHONY: binaries
binaries: $(BINARIES) ## Build Go binaries
bin/%: .FORCE
CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o bin/$* ./cmd/$*
.PHONY: docker
docker: $(IMAGES) ## Build Docker images
docker-%: .FORCE
docker build -t regclient/$* -f build/Dockerfile.$*$(DOCKERFILE_EXT) $(DOCKER_ARGS) .
docker build -t regclient/$*:alpine -f build/Dockerfile.$*$(DOCKERFILE_EXT) --target release-alpine $(DOCKER_ARGS) .
.PHONY: oci-image
oci-image: $(addprefix oci-image-,$(COMMANDS)) ## Build reproducible images to an OCI Layout
oci-image-%: bin/regctl .FORCE
PATH="$(PWD)/bin:$(PATH)" build/oci-image.sh -r scratch -i "$*" -p "$(IMAGE_PLATFORMS)"
PATH="$(PWD)/bin:$(PATH)" build/oci-image.sh -r alpine -i "$*" -p "$(IMAGE_PLATFORMS)" -b "alpine:3"
.PHONY: test-docker
test-docker: $(addprefix test-docker-,$(COMMANDS)) ## Build multi-platform docker images (but do not tag)
test-docker-%:
docker buildx build --platform="$(IMAGE_PLATFORMS)" -f build/Dockerfile.$*.buildkit .
docker buildx build --platform="$(IMAGE_PLATFORMS)" -f build/Dockerfile.$*.buildkit --target release-alpine .
.PHONY: ci
ci: ci-distribution ci-zot ## Run CI tests against self hosted registries
.PHONY: ci-distribution
ci-distribution:
docker run --rm -d -p 5000 \
--label regclient-ci=true --name regclient-ci-distribution \
-e "REGISTRY_STORAGE_DELETE_ENABLED=true" \
docker.io/library/registry:$(CI_DISTRIBUTION_VER)
./build/ci-test.sh -t localhost:$$(docker port regclient-ci-distribution 5000 | head -1 | cut -f2 -d:)/test-ci
docker stop regclient-ci-distribution
.PHONY: ci-zot
ci-zot:
docker run --rm -d -p 5000 \
--label regclient-ci=true --name regclient-ci-zot \
-v "$$(pwd)/build/zot-config.json:/etc/zot/config.json:ro" \
ghcr.io/project-zot/zot-linux-amd64:$(CI_ZOT_VER)
./build/ci-test.sh -t localhost:$$(docker port regclient-ci-zot 5000 | head -1 | cut -f2 -d:)/test-ci
docker stop regclient-ci-zot
.PHONY: artifacts
artifacts: $(ARTIFACTS) ## Generate artifacts
.PHONY: artifact-pre
artifact-pre:
mkdir -p artifacts
artifacts/%: artifact-pre .FORCE
@set -e; \
target="$*"; \
command="$${target%%-*}"; \
platform_ext="$${target#*-}"; \
platform="$${platform_ext%.*}"; \
export GOOS="$${platform%%-*}"; \
export GOARCH="$${platform#*-}"; \
echo export GOOS=$${GOOS}; \
echo export GOARCH=$${GOARCH}; \
echo go build ${GO_BUILD_FLAGS} -o "$@" ./cmd/$${command}/; \
CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o "$@" ./cmd/$${command}/; \
$(SYFT) scan -q "file:$@" --source-name "$${command}" -o cyclonedx-json >"artifacts/$${command}-$${platform}.cyclonedx.json"; \
$(SYFT) scan -q "file:$@" --source-name "$${command}" -o spdx-json >"artifacts/$${command}-$${platform}.spdx.json"
.PHONY: clean
clean: ## delete generated content
[ ! -d artifacts ] || rm -r artifacts
[ ! -d bin ] || rm -r bin
[ ! -d output ] || rm -r output
[ ! -d vendor ] || rm -r vendor
.PHONY: plugin-user
plugin-user:
mkdir -p ${HOME}/.docker/cli-plugins/
cp docker-plugin/docker-regclient ${HOME}/.docker/cli-plugins/docker-regctl
.PHONY: plugin-host
plugin-host:
sudo cp docker-plugin/docker-regclient /usr/libexec/docker/cli-plugins/docker-regctl
.PHONY: util-golang-major
util-golang-major: $(GOPATH)/bin/gomajor ## check for major dependency updates
$(GOPATH)/bin/gomajor list
.PHONY: util-golang-update
util-golang-update: ## update go module versions
go get -u -t ./...
go mod tidy
[ ! -d vendor ] || go mod vendor
.PHONY: util-release-preview
util-release-preview: $(GOPATH)/bin/gorelease ## preview changes for next release
git checkout main
./.github/release.sh -d
gorelease
.PHONY: util-release-run
util-release-run: ## generate a new release
git checkout main
./.github/release.sh
.PHONY: util-version-check
util-version-check: ## check all dependencies for updates
$(VER_BUMP) check
.PHONY: util-version-update
util-version-update: ## update versions on all dependencies
$(VER_BUMP) update
$(GOPATH)/bin/gofumpt: .FORCE
@[ -f "$(GOPATH)/bin/gofumpt" ] \
&& [ "$$($(GOPATH)/bin/gofumpt -version | cut -f 1 -d ' ')" = "$(GOFUMPT_VER)" ] \
|| go install mvdan.cc/gofumpt@$(GOFUMPT_VER)
$(GOPATH)/bin/gomajor: .FORCE
@[ -f "$(GOPATH)/bin/gomajor" ] \
&& [ "$$($(GOPATH)/bin/gomajor version | grep '^version' | cut -f 2 -d ' ')" = "$(GOMAJOR_VER)" ] \
|| go install github.com/icholy/gomajor@$(GOMAJOR_VER)
$(GOPATH)/bin/goimports: .FORCE
@[ -f "$(GOPATH)/bin/goimports" ] && [ "$$(go version | cut -f3 -d' ')" = "$$(go version $(GOPATH)/bin/goimports | cut -f2 -d' ')" ] \
|| go install golang.org/x/tools/cmd/goimports@latest
$(GOPATH)/bin/gorelease: .FORCE
@[ -f "$(GOPATH)/bin/gorelease" ] && [ "$$(go version | cut -f3 -d' ')" = "$$(go version $(GOPATH)/bin/gorelease | cut -f2 -d' ')" ] \
|| go install golang.org/x/exp/cmd/gorelease@latest
$(GOPATH)/bin/gosec: .FORCE
@[ -f $(GOPATH)/bin/gosec ] \
&& [ "$$($(GOPATH)/bin/gosec -version | grep '^Version' | cut -f 2 -d ' ')" = "$(GOSEC_VER)" ] \
|| go install -ldflags '-X main.Version=$(GOSEC_VER) -X main.GitTag=$(GOSEC_VER)' \
github.com/securego/gosec/v2/cmd/gosec@$(GOSEC_VER)
$(GOPATH)/bin/staticcheck: .FORCE
@[ -f $(GOPATH)/bin/staticcheck ] \
&& [ "$$($(GOPATH)/bin/staticcheck -version | cut -f 3 -d ' ' | tr -d '()')" = "$(STATICCHECK_VER)" ] \
|| go install "honnef.co/go/tools/cmd/staticcheck@$(STATICCHECK_VER)"
$(GOPATH)/bin/govulncheck: .FORCE
@[ -f $(GOPATH)/bin/govulncheck ] \
&& [ $$(go version -m $(GOPATH)/bin/govulncheck | \
awk -F ' ' '{ if ($$1 == "mod" && $$2 == "golang.org/x/vuln") { printf "%s\n", $$3 } }') = "$(GO_VULNCHECK_VER)" ] \
|| CGO_ENABLED=0 go install "golang.org/x/vuln/cmd/govulncheck@$(GO_VULNCHECK_VER)"
$(GOPATH)/bin/osv-scanner: .FORCE
@[ -f $(GOPATH)/bin/osv-scanner ] \
&& [ "$$(osv-scanner --version | awk -F ': ' '{ if ($$1 == "osv-scanner version") { printf "%s\n", $$2 } }')" = "$(OSV_SCANNER_VER)" ] \
|| CGO_ENABLED=0 go install "github.com/google/osv-scanner/v2/cmd/osv-scanner@$(OSV_SCANNER_VER)"
.PHONY: help
help: # Display help
@awk -F ':|##' '/^[^\t].+?:.*?##/ { printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF }' $(MAKEFILE_LIST)
-116
View File
@@ -1,116 +0,0 @@
# regclient
[![Go Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/go.yml?branch=main&label=Go%20build)](https://github.com/regclient/regclient/actions/workflows/go.yml)
[![Docker Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/docker.yml?branch=main&label=Docker%20build)](https://github.com/regclient/regclient/actions/workflows/docker.yml)
[![Dependency Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/version-check.yml?branch=main&label=Dependency%20check)](https://github.com/regclient/regclient/actions/workflows/version-check.yml)
[![Vulnerability Workflow Status](https://img.shields.io/github/actions/workflow/status/regclient/regclient/vulnscans.yml?branch=main&label=Vulnerability%20check)](https://github.com/regclient/regclient/actions/workflows/vulnscans.yml)
[![Go Reference](https://pkg.go.dev/badge/github.com/regclient/regclient.svg)](https://pkg.go.dev/github.com/regclient/regclient)
![License](https://img.shields.io/github/license/regclient/regclient)
[![Go Report Card](https://goreportcard.com/badge/github.com/regclient/regclient)](https://goreportcard.com/report/github.com/regclient/regclient)
[![GitHub Downloads](https://img.shields.io/github/downloads/regclient/regclient/total?label=GitHub%20downloads)](https://github.com/regclient/regclient/releases)
regclient is a client interface to OCI conformant registries and content shipped with the OCI Image Layout.
It includes a Go library and several CLI commands.
## regclient Go Library Features
- Runs without a container runtime and without privileged access to the local host.
- Querying for a tag listing, repository listing, and remotely inspecting the contents of images.
- Efficiently copying and retagging images, only pulling layers when required, and without changing the image digest.
- Support for multi-platform images.
- Support for querying, creating, and copying OCI Artifacts, allowing arbitrary data to be stored in an OCI registry.
- Support for packaging OCI Artifacts with an Index of multiple artifacts, which can be used for platform specific artifacts.
- Support for querying OCI referrers, copying referrers, and pushing content with an OCI subject field, associating artifacts with other content on the registry.
- Support for the “digest tags” used by projects like sigstore/cosign, allowing the content to be included when copying images.
- Efficiently query for an image digest.
- Efficiently query for pull rate limits used by Docker Hub.
- Import and export content into OCI Layouts and Docker formatted tar files.
- Support OCI Layouts in all commands as a local disk equivalent of a repository.
- Support for deleting tags, manifests, and blobs.
- Ability to mutate existing images, including:
- Settings annotations or labels
- Deleting content from layers
- Changing timestamps for reproducibility
- Converting between Docker and OCI media types
- Replacing the base image layers
- Add or remove volumes and exposed ports
- Change digest algorithms
- Support for registry warning headers, which may be used to notify users of issues with the server or content they are using.
- Automatically import logins from the docker CLI, and registry certificates from the docker engine.
- Automatic retry, and fallback to a chunked blob push, when network issues are encountered.
The full Go references is available on [pkg.go.dev](https://pkg.go.dev/github.com/regclient/regclient).
## regctl Features
`regctl` is a CLI interface to the `regclient` library.
In addition to the features listed for `regclient`, `regctl` adds the following abilities:
- Generating multi-platform manifests from multiple images that may have been separately built.
- Repackage a multi-platform image with only the requested platforms.
- Push and pull arbitrary OCI artifacts.
- Recursively list all content associated with an image.
- Extract files from a layer or image.
- Compare images, showing the differences between manifests, the config, and layers.
- Formatted output using Go templates.
The project website includes [usage instructions](https://regclient.org/usage/regctl/) and a [CLI reference](https://regclient.org/cli/regctl/).
## regsync features
`regsync` is an image mirroring tool.
It will copy images between two locations with the following additional features:
- Ability to run on a cron schedule, one time synchronization, or only report stale images.
- Uses a yaml configuration.
- Each source may be an entire registry (not recommended), a repository, or a single image, with the ability to filter repositories and tags.
- Support for multi-platform images, OCI referrers, “digest tags”, and copying to or from an OCI Layout (for maintaining a mirror over an air-gap).
- Ability to mirror multiple images concurrently.
- Support for copying a single platform from multi-platform images.
- Ability to backup an existing image before overwriting the tag.
- Ability to postpone mirror step when rate limit (used by Docker Hub) is below a threshold.
- Can use user’s docker configuration for user credentials and registry certificates.
The project website includes [usage instructions](https://regclient.org/usage/regsync/) and a [CLI reference](https://regclient.org/cli/regsync/).
## regbot features
`regbot` is a scripting tool on top of the `regclient` API with the following features:
- Ability to run on a cron schedule, one time execution, or test with a dry-run mode.
- Uses a yaml configuration.
- Scripts are written in Lua and executed directly in Go.
- Built-in functions include:
- Repository list
- Tag list
- Image manifest (either head or get, and optional resolving multi-platform reference)
- Image config (this includes the creation time, labels, and other details shown in a docker image inspect)
- Image rate limit and a wait function to delay the script when rate limit remaining is below a threshold
- Image copy
- Manifest delete
- Tag delete
The project website includes [usage instructions](https://regclient.org/usage/regbot/) and a [CLI reference](https://regclient.org/cli/regbot/).
## Development Status
This project is using v0 version numbers due to Go's backwards compatibility requirements of a v1 release.
The library and commands are stable for external use.
Minor version updates may contain breaking changes, however effort is made to first deprecate and provide warnings to give users time to move off of older APIs and commands.
## Installing
See the [installation instructions](https://regclient.org/install/) on the project website for the various ways to download or build CLI binaries.
## Usage
See the [project documentation](https://regclient.org/usage/).
## Contributors
<a href="https://github.com/regclient/regclient/graphs/contributors">
<img src="https://contrib.rocks/image?repo=regclient/regclient" alt="contributor list"/>
</a>
<!-- markdownlint-disable-file MD033 -->
-5
View File
@@ -1,5 +0,0 @@
# Reporting security issues
Please report security issues directly in GitHub at <https://github.com/regclient/regclient/security/advisories/new> or alternatively email <git@bmitch.net>.
We will typically respond within 7 working days of your report. If the issue is confirmed as a vulnerability, we will open a Security Advisory and acknowledge your contributions as part of it. This project follows a 90 day disclosure timeline.
-283
View File
@@ -1,283 +0,0 @@
package regclient
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"log/slog"
"time"
"github.com/regclient/regclient/internal/pqueue"
"github.com/regclient/regclient/internal/reqmeta"
"github.com/regclient/regclient/scheme"
"github.com/regclient/regclient/types"
"github.com/regclient/regclient/types/blob"
"github.com/regclient/regclient/types/descriptor"
"github.com/regclient/regclient/types/errs"
"github.com/regclient/regclient/types/ref"
"github.com/regclient/regclient/types/warning"
)
const blobCBFreq = time.Millisecond * 100
type blobOpt struct {
callback func(kind types.CallbackKind, instance string, state types.CallbackState, cur, total int64)
readerHook func(*blob.BReader) (*blob.BReader, error)
}
// BlobOpts define options for the Image* commands.
type BlobOpts func(*blobOpt)
// BlobWithCallback provides progress data to a callback function.
func BlobWithCallback(callback func(kind types.CallbackKind, instance string, state types.CallbackState, cur, total int64)) BlobOpts {
return func(opts *blobOpt) {
opts.callback = callback
}
}
// BlobWithReaderHook is called in [RegClient.BlobCopy] with the blob source.
// The returned [blob.BReader] is pushed to the target.
// If the hook returns an error, the copy will fail.
func BlobWithReaderHook(hook func(*blob.BReader) (*blob.BReader, error)) BlobOpts {
return func(opts *blobOpt) {
opts.readerHook = hook
}
}
// BlobCopy copies a blob between two locations.
// If the blob already exists in the target, the copy is skipped.
// A server side cross repository blob mount is attempted.
func (rc *RegClient) BlobCopy(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor, opts ...BlobOpts) error {
if !refSrc.IsSetRepo() {
return fmt.Errorf("refSrc is not set: %s%.0w", refSrc.CommonName(), errs.ErrInvalidReference)
}
if !refTgt.IsSetRepo() {
return fmt.Errorf("refTgt is not set: %s%.0w", refTgt.CommonName(), errs.ErrInvalidReference)
}
var opt blobOpt
for _, optFn := range opts {
optFn(&opt)
}
// dedup warnings
if w := warning.FromContext(ctx); w == nil {
ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()})
}
tDesc := d
tDesc.URLs = []string{} // ignore URLs when pushing to target
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackStarted, 0, d.Size)
}
// for the same repository, there's nothing to copy
if ref.EqualRepository(refSrc, refTgt) {
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
}
rc.slog.Debug("Blob copy skipped, same repo",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("digest", string(d.Digest)))
return nil
}
// check if layer already exists
if _, err := rc.BlobHead(ctx, refTgt, tDesc); err == nil {
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
}
rc.slog.Debug("Blob copy skipped, already exists",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("digest", string(d.Digest)))
return nil
}
// acquire throttle for both src and tgt to avoid deadlocks
tList := []*pqueue.Queue[reqmeta.Data]{}
schemeSrcAPI, err := rc.schemeGet(refSrc.Scheme)
if err != nil {
return err
}
schemeTgtAPI, err := rc.schemeGet(refTgt.Scheme)
if err != nil {
return err
}
if tSrc, ok := schemeSrcAPI.(scheme.Throttler); ok {
tList = append(tList, tSrc.Throttle(refSrc, false)...)
}
if tTgt, ok := schemeTgtAPI.(scheme.Throttler); ok {
tList = append(tList, tTgt.Throttle(refTgt, true)...)
}
if len(tList) > 0 {
ctxMulti, done, err := pqueue.AcquireMulti[reqmeta.Data](ctx, reqmeta.Data{Kind: reqmeta.Blob, Size: d.Size}, tList...)
if err != nil {
return err
}
if done != nil {
defer done()
}
ctx = ctxMulti
}
// try mounting blob from the source repo is the registry is the same
if ref.EqualRegistry(refSrc, refTgt) {
err := rc.BlobMount(ctx, refSrc, refTgt, d)
if err == nil {
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackSkipped, 0, d.Size)
}
rc.slog.Debug("Blob copy performed server side with registry mount",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("digest", string(d.Digest)))
return nil
}
rc.slog.Warn("Failed to mount blob",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("err", err.Error()))
}
// fast options failed, download layer from source and push to target
blobIO, err := rc.BlobGet(ctx, refSrc, d)
if err != nil {
if !errors.Is(err, context.Canceled) {
rc.slog.Warn("Failed to retrieve blob",
slog.String("src", refSrc.Reference),
slog.String("digest", string(d.Digest)),
slog.String("err", err.Error()))
}
return err
}
if opt.callback != nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackStarted, 0, d.Size)
ticker := time.NewTicker(blobCBFreq)
done := make(chan bool)
defer func() {
close(done)
ticker.Stop()
if ctx.Err() == nil {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackFinished, d.Size, d.Size)
}
}()
go func() {
for {
select {
case <-done:
return
case <-ticker.C:
offset, err := blobIO.Seek(0, io.SeekCurrent)
if err == nil && offset > 0 {
opt.callback(types.CallbackBlob, d.Digest.String(), types.CallbackActive, offset, d.Size)
}
}
}
}()
}
if opt.readerHook != nil {
blobIO, err = opt.readerHook(blobIO)
if err != nil {
rc.slog.Warn("Failed to apply reader hook to blob",
slog.String("src", refSrc.Reference),
slog.String("err", err.Error()))
return err
}
}
defer blobIO.Close()
if _, err := rc.BlobPut(ctx, refTgt, blobIO.GetDescriptor(), blobIO); err != nil {
if !errors.Is(err, context.Canceled) {
rc.slog.Warn("Failed to push blob",
slog.String("src", refSrc.Reference),
slog.String("tgt", refTgt.Reference),
slog.String("err", err.Error()))
}
return err
}
return nil
}
// BlobDelete removes a blob from the registry.
// This method should only be used to repair a damaged registry.
// Typically a server side garbage collection should be used to purge unused blobs.
func (rc *RegClient) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error {
if !r.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return err
}
return schemeAPI.BlobDelete(ctx, r, d)
}
// BlobGet retrieves a blob, returning a reader.
// This reader must be closed to free up resources that limit concurrent pulls.
func (rc *RegClient) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
data, err := d.GetData()
if err == nil {
return blob.NewReader(blob.WithDesc(d), blob.WithRef(r), blob.WithReader(bytes.NewReader(data))), nil
}
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
return schemeAPI.BlobGet(ctx, r, d)
}
// BlobGetOCIConfig retrieves an OCI config from a blob, automatically extracting the JSON.
func (rc *RegClient) BlobGetOCIConfig(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.OCIConfig, error) {
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
b, err := rc.BlobGet(ctx, r, d)
if err != nil {
return nil, err
}
return b.ToOCIConfig()
}
// BlobHead is used to verify if a blob exists and is accessible.
func (rc *RegClient) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) {
if !r.IsSetRepo() {
return nil, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return nil, err
}
return schemeAPI.BlobHead(ctx, r, d)
}
// BlobMount attempts to perform a server side copy/mount of the blob between repositories.
func (rc *RegClient) BlobMount(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, d descriptor.Descriptor) error {
if !refSrc.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", refSrc.CommonName(), errs.ErrInvalidReference)
}
if !refTgt.IsSetRepo() {
return fmt.Errorf("ref is not set: %s%.0w", refTgt.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(refSrc.Scheme)
if err != nil {
return err
}
return schemeAPI.BlobMount(ctx, refSrc, refTgt, d)
}
// BlobPut uploads a blob to a repository.
// Descriptor is optional, leave size and digest to zero value if unknown.
// Reader must also be an [io.Seeker] to support chunked upload fallback.
//
// This will attempt an anonymous blob mount first which some registries may support.
// It will then try doing a full put of the blob without chunking (most widely supported).
// If the full put fails, it will fall back to a chunked upload (useful for flaky networks).
func (rc *RegClient) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error) {
if !r.IsSetRepo() {
return descriptor.Descriptor{}, fmt.Errorf("ref is not set: %s%.0w", r.CommonName(), errs.ErrInvalidReference)
}
schemeAPI, err := rc.schemeGet(r.Scheme)
if err != nil {
return descriptor.Descriptor{}, err
}
return schemeAPI.BlobPut(ctx, r, d, rdr)
}
-100
View File
@@ -1,100 +0,0 @@
package config
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"strings"
)
// credHelper wraps a command that manages user credentials.
type credHelper struct {
prog string
env map[string]string
}
func newCredHelper(prog string, env map[string]string) *credHelper {
return &credHelper{prog: prog, env: env}
}
func (ch *credHelper) run(arg string, input io.Reader) ([]byte, error) {
//#nosec G204 only untrusted arg is a hostname which the executed command should not trust
cmd := exec.Command(ch.prog, arg)
cmd.Env = os.Environ()
if ch.env != nil {
for k, v := range ch.env {
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
}
}
cmd.Stderr = os.Stderr
cmd.Stdin = input
return cmd.Output()
}
type credStore struct {
ServerURL string `json:"ServerURL"`
Username string `json:"Username"`
Secret string `json:"Secret"` //#nosec G117 exported struct intentionally holds secrets
}
// get requests a credential from the helper for a given host.
func (ch *credHelper) get(host *Host) error {
hostname := host.Hostname
if host.CredHost != "" {
hostname = host.CredHost
}
hostIn := strings.NewReader(hostname)
credOut := credStore{
Username: host.User,
Secret: host.Pass,
}
outB, err := ch.run("get", hostIn)
if err != nil {
outS := strings.TrimSpace(string(outB))
return fmt.Errorf("error getting credentials, output: %s, error: %w", outS, err)
}
err = json.NewDecoder(bytes.NewReader(outB)).Decode(&credOut)
if err != nil {
return fmt.Errorf("error reading credentials: %w", err)
}
if credOut.Username == tokenUser {
host.User = ""
host.Pass = ""
host.Token = credOut.Secret
} else {
host.User = credOut.Username
host.Pass = credOut.Secret
host.Token = ""
}
return nil
}
// list returns a list of hosts supported by the credential helper.
func (ch *credHelper) list() ([]Host, error) {
credList := map[string]string{}
outB, err := ch.run("list", bytes.NewReader([]byte{}))
if err != nil {
outS := strings.TrimSpace(string(outB))
return nil, fmt.Errorf("error getting credential list, output: %s, error: %w", outS, err)
}
err = json.NewDecoder(bytes.NewReader(outB)).Decode(&credList)
if err != nil {
return nil, fmt.Errorf("error reading credential list: %w", err)
}
hostList := []Host{}
for host, user := range credList {
if !HostValidate(host) {
continue
}
h := HostNewName(host)
h.User = user
h.CredHelper = ch.prog
hostList = append(hostList, *h)
}
return hostList, nil
}
// TODO: store method not implemented
-210
View File
@@ -1,210 +0,0 @@
package config
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"strings"
"github.com/regclient/regclient/internal/conffile"
"github.com/regclient/regclient/types/errs"
)
const (
// dockerEnv is the environment variable used to look for Docker's config.json.
dockerEnv = "DOCKER_CONFIG"
// dockerEnvConfig is used to inject the config as an environment variable.
dockerEnvConfig = "DOCKER_AUTH_CONFIG"
// dockerDir is the directory name for Docker's config (inside the users home directory).
dockerDir = ".docker"
// dockerConfFile is the name of Docker's config file.
dockerConfFile = "config.json"
// dockerHelperPre is the prefix of docker credential helpers.
dockerHelperPre = "docker-credential-"
)
// dockerConfig is used to parse the ~/.docker/config.json
type dockerConfig struct {
AuthConfigs map[string]dockerAuthConfig `json:"auths"`
HTTPHeaders map[string]string `json:"HttpHeaders,omitempty"`
DetachKeys string `json:"detachKeys,omitempty"`
CredentialsStore string `json:"credsStore,omitempty"`
CredentialHelpers map[string]string `json:"credHelpers,omitempty"`
Proxies map[string]dockerProxyConfig `json:"proxies,omitempty"`
}
// dockerProxyConfig contains proxy configuration settings
type dockerProxyConfig struct {
HTTPProxy string `json:"httpProxy,omitempty"`
HTTPSProxy string `json:"httpsProxy,omitempty"`
NoProxy string `json:"noProxy,omitempty"`
FTPProxy string `json:"ftpProxy,omitempty"`
AllProxy string `json:"allProxy,omitempty"`
}
// dockerAuthConfig contains the auths
type dockerAuthConfig struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"` //#nosec G117 exported struct intentionally holds secrets
Auth string `json:"auth,omitempty"`
ServerAddress string `json:"serveraddress,omitempty"`
// IdentityToken is used to authenticate the user and get
// an access token for the registry.
IdentityToken string `json:"identitytoken,omitempty"`
// RegistryToken is a bearer token to be sent to a registry
RegistryToken string `json:"registrytoken,omitempty"`
}
// DockerLoad returns a slice of hosts from the users docker config.
// This will search for the config.json in either the DOCKER_CONFIG identified directory or the default .docker directory.
// It also includes hosts extracted from the DOCKER_AUTH_CONFIG variable.
// If the config file is missing and no value is injected using an environment variable, an empty list is returned.
func DockerLoad() ([]Host, error) {
hosts := []Host{}
errList := []error{}
// load from a file
cf := conffile.New(
conffile.WithHomeDir(dockerDir, dockerConfFile, true),
conffile.WithEnvDir(dockerEnv, dockerConfFile),
)
rdr, err := cf.Open()
if err != nil && !errors.Is(err, fs.ErrNotExist) {
errList = append(errList, err)
} else if err == nil {
defer rdr.Close()
hostsFile, err := dockerParse(rdr)
if err != nil {
errList = append(errList, err)
} else {
hosts = append(hosts, hostsFile...)
}
}
// load from an env var
hostsEnv, err := DockerLoadEnv(dockerEnvConfig)
if err != nil && !errors.Is(err, errs.ErrNotFound) {
errList = append(errList, err)
} else if err == nil {
hosts = append(hosts, hostsEnv...)
}
// return the concatenated result, only wrapping an error list if necessary
if len(errList) == 1 {
return hosts, errList[0]
} else {
return hosts, errors.Join(errList...)
}
}
// DockerLoadFile returns a slice of hosts from a named docker config file.
func DockerLoadFile(fname string) ([]Host, error) {
//#nosec G304 scoping file operations to a directory is not yet a feature of regclient.
rdr, err := os.Open(fname)
if err != nil && errors.Is(err, fs.ErrNotExist) {
return []Host{}, nil
} else if err != nil {
return nil, err
}
defer rdr.Close()
return dockerParse(rdr)
}
// DockerLoadEnv returns a slice of hosts extracted from the config injected in an environment variable.
func DockerLoadEnv(envName string) ([]Host, error) {
envVal := os.Getenv(envName)
if envVal == "" {
return []Host{}, errs.ErrNotFound
}
return dockerParse(strings.NewReader(envVal))
}
// dockerParse parses a docker config into a slice of Hosts.
func dockerParse(rdr io.Reader) ([]Host, error) {
dc := dockerConfig{}
if err := json.NewDecoder(rdr).Decode(&dc); err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
hosts := []Host{}
for name, auth := range dc.AuthConfigs {
if !HostValidate(name) {
continue
}
h, err := dockerAuthToHost(name, dc, auth)
if err != nil {
continue
}
hosts = append(hosts, h)
}
// also include default entries for credential helpers
for name, helper := range dc.CredentialHelpers {
if !HostValidate(name) {
continue
}
h := HostNewName(name)
h.CredHelper = dockerHelperPre + helper
if _, ok := dc.AuthConfigs[name]; ok {
continue // skip fields with auth config
}
hosts = append(hosts, *h)
}
// add credStore entries
if dc.CredentialsStore != "" {
ch := newCredHelper(dockerHelperPre+dc.CredentialsStore, map[string]string{})
csHosts, err := ch.list()
if err == nil {
hosts = append(hosts, csHosts...)
}
}
return hosts, nil
}
// dockerAuthToHost parses an auth entry from a docker config into a Host.
func dockerAuthToHost(name string, conf dockerConfig, auth dockerAuthConfig) (Host, error) {
helper := ""
if conf.CredentialHelpers != nil && conf.CredentialHelpers[name] != "" {
helper = dockerHelperPre + conf.CredentialHelpers[name]
}
// parse base64 auth into user/pass
if auth.Auth != "" {
var err error
auth.Username, auth.Password, err = decodeAuth(auth.Auth)
if err != nil {
return Host{}, err
}
}
if (auth.Username == "" || auth.Password == "") && auth.IdentityToken == "" && helper == "" {
return Host{}, fmt.Errorf("no credentials found for %s", name)
}
h := HostNewName(name)
// ignore unknown names
if h.Name != DockerRegistry && !strings.HasSuffix(strings.TrimSuffix(name, "/"), h.Name) {
return Host{}, fmt.Errorf("rejecting entry with repository: %s", name)
}
h.User = auth.Username
h.Pass = auth.Password
h.Token = auth.IdentityToken
h.CredHelper = helper
return *h, nil
}
// decodeAuth extracts a base64 encoded user:pass into the username and password.
func decodeAuth(authStr string) (string, string, error) {
if authStr == "" {
return "", "", nil
}
decoded, err := base64.StdEncoding.DecodeString(authStr)
if err != nil {
return "", "", err
}
userPass := strings.SplitN(string(decoded), ":", 2)
if len(userPass) != 2 {
return "", "", fmt.Errorf("invalid auth configuration file")
}
return userPass[0], strings.Trim(userPass[1], "\x00"), nil
}
-521
View File
@@ -1,521 +0,0 @@
// Package config is used for all regclient configuration settings.
package config
import (
"encoding/json"
"fmt"
"io"
"log/slog"
"maps"
"slices"
"strings"
"time"
"github.com/regclient/regclient/internal/timejson"
)
// TLSConf specifies whether TLS is enabled and verified for a host.
type TLSConf int
const (
// TLSUndefined indicates TLS is not passed, defaults to Enabled.
TLSUndefined TLSConf = iota
// TLSEnabled uses TLS (https) for the connection.
TLSEnabled
// TLSInsecure uses TLS but does not verify CA.
TLSInsecure
// TLSDisabled does not use TLS (http).
TLSDisabled
)
const (
// DockerRegistry is the name resolved in docker images on Hub.
DockerRegistry = "docker.io"
// DockerRegistryAuth is the name provided in docker's config for Hub.
DockerRegistryAuth = "https://index.docker.io/v1/"
// DockerRegistryDNS is the host to connect to for Hub.
DockerRegistryDNS = "registry-1.docker.io"
// defaultExpire is the default time to expire a credential and force re-authentication.
defaultExpire = time.Hour * 1
// defaultCredHelperRetry is the time to refresh a credential from a failed credential helper command.
defaultCredHelperRetry = time.Second * 5
// defaultConcurrent is the default number of concurrent registry connections.
defaultConcurrent = 3
// defaultReqPerSec is the default maximum frequency to send requests to a registry.
defaultReqPerSec = 0
// tokenUser is the username returned by credential helpers that indicates the password is an identity token.
tokenUser = "<token>"
)
// MarshalJSON converts TLSConf to a json string using MarshalText.
func (t TLSConf) MarshalJSON() ([]byte, error) {
s, err := t.MarshalText()
if err != nil {
return []byte(""), err
}
return json.Marshal(string(s))
}
// MarshalText converts TLSConf to a string.
func (t TLSConf) MarshalText() ([]byte, error) {
var s string
switch t {
default:
s = ""
case TLSEnabled:
s = "enabled"
case TLSInsecure:
s = "insecure"
case TLSDisabled:
s = "disabled"
}
return []byte(s), nil
}
// UnmarshalJSON converts TLSConf from a json string.
func (t *TLSConf) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err != nil {
return err
}
return t.UnmarshalText([]byte(s))
}
// UnmarshalText converts TLSConf from a string.
func (t *TLSConf) UnmarshalText(b []byte) error {
switch strings.ToLower(string(b)) {
default:
return fmt.Errorf("unknown TLS value \"%s\"", b)
case "":
*t = TLSUndefined
case "enabled":
*t = TLSEnabled
case "insecure":
*t = TLSInsecure
case "disabled":
*t = TLSDisabled
}
return nil
}
// Host defines settings for connecting to a registry.
type Host struct {
Name string `json:"-" yaml:"registry,omitempty"` // Name of the registry (required) (yaml configs pass this as a field, json provides this from the object key)
TLS TLSConf `json:"tls,omitempty" yaml:"tls"` // TLS setting: enabled (default), disabled, insecure
RegCert string `json:"regcert,omitempty" yaml:"regcert"` // public pem cert of registry
ClientCert string `json:"clientCert,omitempty" yaml:"clientCert"` // public pem cert for client (mTLS)
ClientKey string `json:"clientKey,omitempty" yaml:"clientKey"` //#nosec G117 private pem cert for client (mTLS)
Hostname string `json:"hostname,omitempty" yaml:"hostname"` // hostname of registry, default is the registry name
User string `json:"user,omitempty" yaml:"user"` // username, not used with credHelper
Pass string `json:"pass,omitempty" yaml:"pass"` //#nosec G117 password, not used with credHelper
Token string `json:"token,omitempty" yaml:"token"` // token, experimental for specific APIs
CredHelper string `json:"credHelper,omitempty" yaml:"credHelper"` // credential helper command for requesting logins
CredExpire timejson.Duration `json:"credExpire,omitempty" yaml:"credExpire"` // time until credential expires
CredHost string `json:"credHost,omitempty" yaml:"credHost"` // used when a helper hostname doesn't match Hostname
PathPrefix string `json:"pathPrefix,omitempty" yaml:"pathPrefix"` // used for mirrors defined within a repository namespace
Mirrors []string `json:"mirrors,omitempty" yaml:"mirrors"` // list of other Host Names to use as mirrors
Priority uint `json:"priority,omitempty" yaml:"priority"` // priority when sorting mirrors, higher priority attempted first
RepoAuth bool `json:"repoAuth,omitempty" yaml:"repoAuth"` // tracks a separate auth per repo
API string `json:"api,omitempty" yaml:"api"` // Deprecated: registry API to use
APIOpts map[string]string `json:"apiOpts,omitempty" yaml:"apiOpts"` // options for APIs
BlobChunk int64 `json:"blobChunk,omitempty" yaml:"blobChunk"` // size of each blob chunk
BlobMax int64 `json:"blobMax,omitempty" yaml:"blobMax"` // threshold to switch to chunked upload, -1 to disable, 0 for regclient.blobMaxPut
ReqPerSec float64 `json:"reqPerSec,omitempty" yaml:"reqPerSec"` // requests per second
ReqConcurrent int64 `json:"reqConcurrent,omitempty" yaml:"reqConcurrent"` // concurrent requests, default is defaultConcurrent(3)
Scheme string `json:"scheme,omitempty" yaml:"scheme"` // Deprecated: use TLS instead
credRefresh time.Time `json:"-" yaml:"-"` // internal use, when to refresh credentials
}
// Cred defines a user credential for accessing a registry.
type Cred struct {
User, Password, Token string //#nosec G117 exported struct intentionally holds secrets
}
// HostNew creates a default Host entry.
func HostNew() *Host {
h := Host{
TLS: TLSEnabled,
APIOpts: map[string]string{},
ReqConcurrent: int64(defaultConcurrent),
ReqPerSec: float64(defaultReqPerSec),
}
return &h
}
// HostNewName creates a default Host with a hostname.
func HostNewName(name string) *Host {
return HostNewDefName(nil, name)
}
// HostNewDefName creates a host using provided defaults and hostname.
func HostNewDefName(def *Host, name string) *Host {
var h Host
if def == nil {
h = *HostNew()
} else {
h = *def
// configure required defaults
if h.TLS == TLSUndefined {
h.TLS = TLSEnabled
}
if h.APIOpts == nil {
h.APIOpts = map[string]string{}
}
if h.ReqConcurrent == 0 {
h.ReqConcurrent = int64(defaultConcurrent)
}
if h.ReqPerSec == 0 {
h.ReqPerSec = float64(defaultReqPerSec)
}
// copy any fields that are not passed by value
if len(h.APIOpts) > 0 {
orig := h.APIOpts
h.APIOpts = map[string]string{}
maps.Copy(h.APIOpts, orig)
}
if h.Mirrors != nil {
orig := h.Mirrors
h.Mirrors = make([]string, len(orig))
copy(h.Mirrors, orig)
}
}
// configure host
scheme, registry, _ := parseName(name)
if scheme == "http" {
h.TLS = TLSDisabled
}
// Docker Hub is a special case
if registry == DockerRegistry {
h.Name = DockerRegistry
h.Hostname = DockerRegistryDNS
h.CredHost = DockerRegistryAuth
return &h
}
h.Name = registry
h.Hostname = registry
if name != registry {
h.CredHost = name
}
return &h
}
// HostValidate returns true if the scheme is missing or a known value, and the path is not set.
func HostValidate(name string) bool {
scheme, _, path := parseName(name)
return path == "" && (scheme == "https" || scheme == "http")
}
// GetCred returns the credential, fetching from a credential helper if needed.
func (host *Host) GetCred() Cred {
// refresh from credHelper if needed
if host.CredHelper != "" && (host.credRefresh.IsZero() || time.Now().After(host.credRefresh)) {
host.refreshHelper()
}
return Cred{User: host.User, Password: host.Pass, Token: host.Token}
}
func (host *Host) refreshHelper() {
if host.CredHelper == "" {
return
}
if host.CredExpire <= 0 {
host.CredExpire = timejson.Duration(defaultExpire)
}
// run a cred helper, calling get method
ch := newCredHelper(host.CredHelper, map[string]string{})
err := ch.get(host)
if err != nil {
host.credRefresh = time.Now().Add(defaultCredHelperRetry)
} else {
host.credRefresh = time.Now().Add(time.Duration(host.CredExpire))
}
}
// IsZero returns true if the struct is set to the zero value or the result of [HostNew].
func (host Host) IsZero() bool {
if (host.TLS != TLSUndefined && host.TLS != TLSEnabled) ||
host.RegCert != "" ||
host.ClientCert != "" ||
host.ClientKey != "" ||
(host.Hostname != "" && host.Hostname != host.Name) ||
host.User != "" ||
host.Pass != "" ||
host.Token != "" ||
host.CredHelper != "" ||
host.CredExpire != 0 ||
host.CredHost != "" ||
host.PathPrefix != "" ||
len(host.Mirrors) != 0 ||
host.Priority != 0 ||
host.RepoAuth ||
len(host.APIOpts) != 0 ||
host.BlobChunk != 0 ||
host.BlobMax != 0 ||
(host.ReqPerSec != 0 && host.ReqPerSec != float64(defaultReqPerSec)) ||
(host.ReqConcurrent != 0 && host.ReqConcurrent != int64(defaultConcurrent)) ||
!host.credRefresh.IsZero() {
return false
}
return true
}
// Merge adds fields from a new config host entry.
func (host *Host) Merge(newHost Host, log *slog.Logger) error {
name := newHost.Name
if name == "" {
name = host.Name
}
if log == nil {
log = slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{}))
}
// merge the existing and new config host
if host.Name == "" {
// only set the name if it's not initialized, this shouldn't normally change
host.Name = newHost.Name
}
if newHost.CredHelper == "" && (newHost.Pass != "" || host.Token != "") {
// unset existing cred helper for user/pass or token
host.CredHelper = ""
host.CredExpire = 0
}
if newHost.CredHelper != "" && newHost.User == "" && newHost.Pass == "" && newHost.Token == "" {
// unset existing user/pass/token for cred helper
host.User = ""
host.Pass = ""
host.Token = ""
}
if newHost.User != "" {
if host.User != "" && host.User != newHost.User {
log.Warn("Changing login user for registry",
slog.String("orig", host.User),
slog.String("new", newHost.User),
slog.String("host", name))
}
host.User = newHost.User
}
if newHost.Pass != "" {
if host.Pass != "" && host.Pass != newHost.Pass {
log.Warn("Changing login password for registry",
slog.String("host", name))
}
host.Pass = newHost.Pass
}
if newHost.Token != "" {
if host.Token != "" && host.Token != newHost.Token {
log.Warn("Changing login token for registry",
slog.String("host", name))
}
host.Token = newHost.Token
}
if newHost.CredHelper != "" {
if host.CredHelper != "" && host.CredHelper != newHost.CredHelper {
log.Warn("Changing credential helper for registry",
slog.String("host", name),
slog.String("orig", host.CredHelper),
slog.String("new", newHost.CredHelper))
}
host.CredHelper = newHost.CredHelper
}
if newHost.CredExpire != 0 {
if host.CredExpire != 0 && host.CredExpire != newHost.CredExpire {
log.Warn("Changing credential expire for registry",
slog.String("host", name),
slog.Any("orig", host.CredExpire),
slog.Any("new", newHost.CredExpire))
}
host.CredExpire = newHost.CredExpire
}
if newHost.CredHost != "" {
if host.CredHost != "" && host.CredHost != newHost.CredHost {
log.Warn("Changing credential host for registry",
slog.String("host", name),
slog.String("orig", host.CredHost),
slog.String("new", newHost.CredHost))
}
host.CredHost = newHost.CredHost
}
if newHost.TLS != TLSUndefined {
if host.TLS != TLSUndefined && host.TLS != newHost.TLS {
tlsOrig, _ := host.TLS.MarshalText()
tlsNew, _ := newHost.TLS.MarshalText()
log.Warn("Changing TLS settings for registry",
slog.String("orig", string(tlsOrig)),
slog.String("new", string(tlsNew)),
slog.String("host", name))
}
host.TLS = newHost.TLS
}
if newHost.RegCert != "" {
if host.RegCert != "" && host.RegCert != newHost.RegCert {
log.Warn("Changing certificate settings for registry",
slog.String("orig", host.RegCert),
slog.String("new", newHost.RegCert),
slog.String("host", name))
}
host.RegCert = newHost.RegCert
}
if newHost.ClientCert != "" {
if host.ClientCert != "" && host.ClientCert != newHost.ClientCert {
log.Warn("Changing client certificate settings for registry",
slog.String("orig", host.ClientCert),
slog.String("new", newHost.ClientCert),
slog.String("host", name))
}
host.ClientCert = newHost.ClientCert
}
if newHost.ClientKey != "" {
if host.ClientKey != "" && host.ClientKey != newHost.ClientKey {
log.Warn("Changing client certificate key settings for registry",
slog.String("host", name))
}
host.ClientKey = newHost.ClientKey
}
if newHost.Hostname != "" {
if host.Hostname != "" && host.Hostname != newHost.Hostname {
log.Warn("Changing hostname settings for registry",
slog.String("orig", host.Hostname),
slog.String("new", newHost.Hostname),
slog.String("host", name))
}
host.Hostname = newHost.Hostname
}
if newHost.PathPrefix != "" {
newHost.PathPrefix = strings.Trim(newHost.PathPrefix, "/") // leading and trailing / are not needed
if host.PathPrefix != "" && host.PathPrefix != newHost.PathPrefix {
log.Warn("Changing path prefix settings for registry",
slog.String("orig", host.PathPrefix),
slog.String("new", newHost.PathPrefix),
slog.String("host", name))
}
host.PathPrefix = newHost.PathPrefix
}
if len(newHost.Mirrors) > 0 {
if len(host.Mirrors) > 0 && !slices.Equal(host.Mirrors, newHost.Mirrors) {
log.Warn("Changing mirror settings for registry",
slog.Any("orig", host.Mirrors),
slog.Any("new", newHost.Mirrors),
slog.String("host", name))
}
host.Mirrors = newHost.Mirrors
}
if newHost.Priority != 0 {
if host.Priority != 0 && host.Priority != newHost.Priority {
log.Warn("Changing priority settings for registry",
slog.Uint64("orig", uint64(host.Priority)),
slog.Uint64("new", uint64(newHost.Priority)),
slog.String("host", name))
}
host.Priority = newHost.Priority
}
if newHost.RepoAuth {
host.RepoAuth = newHost.RepoAuth
}
// TODO: eventually delete
if newHost.API != "" {
log.Warn("API field has been deprecated",
slog.String("api", newHost.API),
slog.String("host", name))
}
if len(newHost.APIOpts) > 0 {
if len(host.APIOpts) > 0 {
merged := maps.Clone(host.APIOpts)
for k, v := range newHost.APIOpts {
if host.APIOpts[k] != "" && host.APIOpts[k] != v {
log.Warn("Changing APIOpts setting for registry",
slog.String("orig", host.APIOpts[k]),
slog.String("new", newHost.APIOpts[k]),
slog.String("opt", k),
slog.String("host", name))
}
merged[k] = v
}
host.APIOpts = merged
} else {
host.APIOpts = newHost.APIOpts
}
}
if newHost.BlobChunk > 0 {
if host.BlobChunk != 0 && host.BlobChunk != newHost.BlobChunk {
log.Warn("Changing blobChunk settings for registry",
slog.Int64("orig", host.BlobChunk),
slog.Int64("new", newHost.BlobChunk),
slog.String("host", name))
}
host.BlobChunk = newHost.BlobChunk
}
if newHost.BlobMax != 0 {
if host.BlobMax != 0 && host.BlobMax != newHost.BlobMax {
log.Warn("Changing blobMax settings for registry",
slog.Int64("orig", host.BlobMax),
slog.Int64("new", newHost.BlobMax),
slog.String("host", name))
}
host.BlobMax = newHost.BlobMax
}
if newHost.ReqPerSec != 0 {
if host.ReqPerSec != 0 && host.ReqPerSec != newHost.ReqPerSec {
log.Warn("Changing reqPerSec settings for registry",
slog.Float64("orig", host.ReqPerSec),
slog.Float64("new", newHost.ReqPerSec),
slog.String("host", name))
}
host.ReqPerSec = newHost.ReqPerSec
}
if newHost.ReqConcurrent > 0 {
if host.ReqConcurrent != 0 && host.ReqConcurrent != newHost.ReqConcurrent {
log.Warn("Changing reqPerSec settings for registry",
slog.Int64("orig", host.ReqConcurrent),
slog.Int64("new", newHost.ReqConcurrent),
slog.String("host", name))
}
host.ReqConcurrent = newHost.ReqConcurrent
}
return nil
}
// parseName splits a registry into the scheme, hostname, and repository/path.
func parseName(name string) (string, string, string) {
scheme := "https"
path := ""
// Docker Hub is a special case
if name == DockerRegistryAuth || name == DockerRegistryDNS || name == DockerRegistry {
return scheme, DockerRegistry, ""
}
// handle http/https prefix
i := strings.Index(name, "://")
if i > 0 {
scheme = name[:i]
name = name[i+3:]
}
// trim any repository path
i = strings.Index(name, "/")
if i > 0 {
path = name[i+1:]
name = name[:i]
}
return scheme, name, path
}
-1905
View File
File diff suppressed because it is too large Load Diff
-923
View File
@@ -1,923 +0,0 @@
// Package auth is used for HTTP authentication
package auth
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"slices"
"strings"
"sync"
"time"
"github.com/regclient/regclient/internal/regnet"
"github.com/regclient/regclient/types/errs"
)
type charLU byte
var charLUs [256]charLU
var defaultClientID = "regclient"
// minTokenLife tokens are required to last at least 60 seconds to support older docker clients
var minTokenLife = 60
// tokenBuffer is used to renew a token before it expires to account for time to process requests on the server
var tokenBuffer = time.Second * 5
const (
isSpace charLU = 1 << iota
isToken
)
func init() {
for c := range 256 {
charLUs[c] = 0
if strings.ContainsRune(" \t\r\n", rune(c)) {
charLUs[c] |= isSpace
}
if (rune('a') <= rune(c) && rune(c) <= rune('z')) || (rune('A') <= rune(c) && rune(c) <= rune('Z') || (rune('0') <= rune(c) && rune(c) <= rune('9')) || strings.ContainsRune("-._~+/", rune(c))) {
charLUs[c] |= isToken
}
}
}
// CredsFn is passed to lookup credentials for a given hostname, response is a username and password or empty strings
type CredsFn func(host string) Cred
// Cred is returned by the CredsFn.
// If Token is provided and auth method is bearer, it will attempt to use it as a refresh token.
// Else if user and password are provided, they are attempted with all auth methods.
// Else if neither are provided and auth method is bearer, an anonymous login is attempted.
type Cred struct {
//#nosec G117 exported struct intentionally holds secrets
User, Password string // clear text username and password
Token string // refresh token only used for bearer auth
}
// challenge is the extracted contents of the WWW-Authenticate header.
type challenge struct {
authType string
params map[string]string
}
// handler handles a challenge for a host to return an auth header
type handler interface {
AddScope(scope string) error
ProcessChallenge(challenge) error
UpdateRequest(*http.Request) error
}
// handlerBuild is used to make a new handler for a specific authType and URL
type handlerBuild func(client *http.Client, clientID, host string, credFn CredsFn, slog *slog.Logger) handler
// Opts configures options for NewAuth
type Opts func(*Auth)
// Auth is used to handle authentication requests.
type Auth struct {
httpClient *http.Client
clientID string
credsFn CredsFn
hbs map[string]handlerBuild // handler builders based on authType
hs map[string]map[string]handler // handlers based on url and authType
authTypes []string
slog *slog.Logger
mu sync.Mutex
}
// NewAuth creates a new Auth
func NewAuth(opts ...Opts) *Auth {
a := &Auth{
httpClient: &http.Client{},
clientID: defaultClientID,
credsFn: DefaultCredsFn,
hbs: map[string]handlerBuild{},
hs: map[string]map[string]handler{},
authTypes: []string{},
slog: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
}
for _, opt := range opts {
opt(a)
}
if len(a.authTypes) == 0 {
a.addDefaultHandlers()
}
return a
}
// WithCreds provides a user/pass lookup for a url
func WithCreds(f CredsFn) Opts {
return func(a *Auth) {
if f != nil {
a.credsFn = f
}
}
}
// WithHTTPClient uses a specific http client with requests
func WithHTTPClient(h *http.Client) Opts {
return func(a *Auth) {
if h != nil {
a.httpClient = h
}
}
}
// WithClientID uses a client ID with request headers
func WithClientID(clientID string) Opts {
return func(a *Auth) {
a.clientID = clientID
}
}
// WithHandler includes a handler for a specific auth type
func WithHandler(authType string, hb handlerBuild) Opts {
return func(a *Auth) {
lcat := strings.ToLower(authType)
a.hbs[lcat] = hb
a.authTypes = append(a.authTypes, lcat)
}
}
// WithDefaultHandlers includes a Basic and Bearer handler, this is automatically added with "WithHandler" is not called
func WithDefaultHandlers() Opts {
return func(a *Auth) {
a.addDefaultHandlers()
}
}
// WithLog injects a Logger
func WithLog(slog *slog.Logger) Opts {
return func(a *Auth) {
a.slog = slog
}
}
// AddScope extends an existing auth with additional scopes.
// This is used to pre-populate scopes with the Docker convention rather than
// depend on the registry to respond with the correct http status and headers.
func (a *Auth) AddScope(host, scope string) error {
a.mu.Lock()
defer a.mu.Unlock()
success := false
if a.hs[host] == nil {
return errs.ErrNoNewChallenge
}
for _, at := range a.authTypes {
if a.hs[host][at] != nil {
err := a.hs[host][at].AddScope(scope)
if err == nil {
success = true
} else if err != errs.ErrNoNewChallenge {
return err
}
}
}
if !success {
return errs.ErrNoNewChallenge
}
a.slog.Debug("Auth scope added",
slog.String("host", host),
slog.String("scope", scope))
return nil
}
// HandleResponse parses the 401 response, extracting the WWW-Authenticate
// header and verifying the requirement is different from what was included in
// the last request
func (a *Auth) HandleResponse(resp *http.Response) error {
a.mu.Lock()
defer a.mu.Unlock()
// verify response is an access denied
if resp.StatusCode != http.StatusUnauthorized {
return errs.ErrUnsupported
}
// extract host and auth header
host := resp.Request.URL.Host
cl, err := ParseAuthHeaders(resp.Header.Values("WWW-Authenticate"))
if err != nil {
return err
}
a.slog.Debug("Auth request parsed",
slog.Any("challenge", cl))
if len(cl) < 1 {
return errs.ErrEmptyChallenge
}
goodChallenge := false
// loop over the received challenge(s)
for _, c := range cl {
if _, ok := a.hbs[c.authType]; !ok {
a.slog.Warn("Unsupported auth type",
slog.String("authtype", c.authType))
continue
}
// setup a handler for the host and auth type
if _, ok := a.hs[host]; !ok {
a.hs[host] = map[string]handler{}
}
if _, ok := a.hs[host][c.authType]; !ok {
h := a.hbs[c.authType](a.httpClient, a.clientID, host, a.credsFn, a.slog)
if h == nil {
continue
}
a.hs[host][c.authType] = h
}
// process the challenge with that handler
err := a.hs[host][c.authType].ProcessChallenge(c)
if err == nil {
goodChallenge = true
} else if err == errs.ErrNoNewChallenge {
// handle race condition when another request updates the challenge
// detect that by seeing the current auth header is different
prevAH := resp.Request.Header.Get("Authorization")
err := a.hs[host][c.authType].UpdateRequest(resp.Request)
if err == nil && prevAH != resp.Request.Header.Get("Authorization") {
goodChallenge = true
}
} else {
return err
}
}
if !goodChallenge {
return errs.ErrHTTPUnauthorized
}
return nil
}
// UpdateRequest adds Authorization headers to a request
func (a *Auth) UpdateRequest(req *http.Request) error {
a.mu.Lock()
defer a.mu.Unlock()
host := req.URL.Host
if a.hs[host] == nil {
return nil
}
var err error
for _, at := range a.authTypes {
if a.hs[host][at] != nil {
err = a.hs[host][at].UpdateRequest(req)
if err != nil {
a.slog.Debug("Failed to generate auth",
slog.String("err", err.Error()),
slog.String("host", host),
slog.String("authtype", at))
continue
}
break
}
}
if err != nil {
return err
}
return nil
}
func (a *Auth) addDefaultHandlers() {
if _, ok := a.hbs["basic"]; !ok {
a.hbs["basic"] = NewBasicHandler
a.authTypes = append(a.authTypes, "basic")
}
if _, ok := a.hbs["bearer"]; !ok {
a.hbs["bearer"] = NewBearerHandler
a.authTypes = append(a.authTypes, "bearer")
}
}
// DefaultCredsFn is used to return no credentials when auth is not configured with a CredsFn
// This avoids the need to check for nil pointers
func DefaultCredsFn(h string) Cred {
return Cred{}
}
// ParseAuthHeaders extracts the scheme and realm from WWW-Authenticate headers
func ParseAuthHeaders(ahl []string) ([]challenge, error) {
var cl []challenge
for _, ah := range ahl {
c, err := parseAuthHeader(ah)
if err != nil {
return nil, fmt.Errorf("failed to parse challenge header: %s, %w", ah, err)
}
cl = append(cl, c...)
}
return cl, nil
}
// parseAuthHeader parses a single header line for WWW-Authenticate
// Example values:
// Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push"
// Basic realm="GitHub Package Registry"
func parseAuthHeader(ah string) ([]challenge, error) {
var cl []challenge
var c *challenge
curElement := []byte{}
curKey := ""
stateElement := "string"
stateSyntax := "start"
for _, b := range []byte(ah) {
switch stateElement {
case "string":
// string: ignore leading space, enter quote only if first character, handle escapes, handle valid tokens, else end
if charLUs[b]&isToken != 0 {
// add valid tokens to element
curElement = append(curElement, b)
} else if charLUs[b]&isSpace != 0 && len(curElement) == 0 {
// ignore leading spaces
} else if b == '"' && len(curElement) == 0 {
stateElement = "quote"
} else if b == '\\' {
stateElement = "escape_string"
} else {
stateElement = "end"
}
case "quote":
// quote: handle escapes, handle closing quote (quote_end to read next character), all other tokens are valid
if b == '\\' {
stateElement = "escape_quote"
} else if b == '"' {
stateElement = "quote_end"
} else {
curElement = append(curElement, b)
}
case "quote_end":
// any character after the close quote is the end of the element
stateElement = "end"
case "escape_string":
// escape_string: handle any character and return to string state
curElement = append(curElement, b)
stateElement = "string"
case "escape_quote":
// escape_quote: handle any character and return to quote state
curElement = append(curElement, b)
stateElement = "quote"
case "end":
// finished parsing element, continue to processing element according to the current state
default:
return nil, fmt.Errorf("unhandled element case: %w", errs.ErrParsingFailed)
}
if stateElement != "end" {
// continue parsing the element until it ends
continue
}
// syntax looks at each string within the overall challenge syntax
switch stateSyntax {
case "start":
// start: (start of auth_type) read auth_type and space (end_auth_type) or auth_type and comma (start)
if charLUs[b]&isSpace != 0 && len(curElement) > 0 {
stateSyntax = "end_auth_type"
} else if b == ',' && len(curElement) > 0 {
// state remains at start
} else {
return nil, fmt.Errorf("start element did not end with a space or comma: %w", errs.ErrParsingFailed)
}
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
cl = append(cl, *c)
case "start_or_param":
// start_or_param: (after param_value) read auth_type and space (end_auth_type) or param_key and equals (param_value)
if charLUs[b]&isSpace != 0 && len(curElement) > 0 {
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
cl = append(cl, *c)
stateSyntax = "end_auth_type"
} else if b == '=' && len(curElement) > 0 {
curKey = strings.ToLower(string(curElement))
stateSyntax = "param_value"
} else {
return nil, fmt.Errorf("expected auth type or param: %w", errs.ErrParsingFailed)
}
case "end_auth_type":
// end_auth_type: (after reading auth_type) read param_key and equals (param_value) or just a comma (start)
if b == '=' && len(curElement) > 0 {
curKey = strings.ToLower(string(curElement))
stateSyntax = "param_value"
} else if b == ',' && len(curElement) == 0 {
// ignore white space between end of auth_type and comma
stateSyntax = "start"
} else {
return nil, fmt.Errorf("expected param or comma: %w", errs.ErrParsingFailed)
}
case "param_value":
// param_value: (after param_key) read param_value and comma (start_or_param)
if b == ',' {
c.params[curKey] = string(curElement)
stateSyntax = "start_or_param"
curKey = ""
} else {
return nil, fmt.Errorf("expected param value: %w", errs.ErrParsingFailed)
}
default:
return nil, fmt.Errorf("unhandled syntax case: %w", errs.ErrParsingFailed)
}
// reset element state
stateElement = "string"
curElement = []byte{}
}
// at end of parsing, if the element is not empty, process according to syntax state:
if len(curElement) > 0 {
// ensure this is not within an unclosed quote or partial escape
if stateElement != "string" && stateElement != "quote_end" {
return nil, fmt.Errorf("eol element in state %s: %w", stateElement, errs.ErrParsingFailed)
}
switch stateSyntax {
case "start", "start_or_param":
// add a new auth type if a string is seen at the start, before any equals
c = &challenge{authType: strings.ToLower(string(curElement)), params: map[string]string{}}
cl = append(cl, *c)
case "param_value":
// add the last param key=val
c.params[curKey] = string(curElement)
case "end_auth_type":
// missing equals for param
return nil, fmt.Errorf("eol at param without value: %w", errs.ErrParsingFailed)
}
}
return cl, nil
}
// basicHandler supports Basic auth type requests
type basicHandler struct {
realm string
host string
credsFn CredsFn
}
// NewBasicHandler creates a new BasicHandler
func NewBasicHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
return &basicHandler{
realm: "",
host: host,
credsFn: credsFn,
}
}
// AddScope is not valid for BasicHandler
func (b *basicHandler) AddScope(scope string) error {
return errs.ErrNoNewChallenge
}
// ProcessChallenge for BasicHandler is a noop
func (b *basicHandler) ProcessChallenge(c challenge) error {
if _, ok := c.params["realm"]; !ok {
return errs.ErrInvalidChallenge
}
if b.realm != c.params["realm"] {
b.realm = c.params["realm"]
return nil
}
return errs.ErrNoNewChallenge
}
// UpdateRequest for BasicHandler generates base64 encoded user/pass for a host
func (b *basicHandler) UpdateRequest(req *http.Request) error {
cred := b.credsFn(b.host)
if cred.User == "" || cred.Password == "" {
return fmt.Errorf("no credentials available: %w", errs.ErrHTTPUnauthorized)
}
req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
base64.StdEncoding.EncodeToString([]byte(cred.User+":"+cred.Password))))
return nil
}
// bearerHandler supports Bearer auth type requests
type bearerHandler struct {
client *http.Client
clientID string
realm, service string
host string
credsFn CredsFn
scopes []string
tokenURL *url.URL
token bearerToken
slog *slog.Logger
}
// bearerToken is the json response to the Bearer request
type bearerToken struct {
Token string `json:"token"`
AccessToken string `json:"access_token"` //#nosec G117 exported struct intentionally holds secrets
ExpiresIn int `json:"expires_in"`
IssuedAt time.Time `json:"issued_at"`
RefreshToken string `json:"refresh_token"` //#nosec G117 exported struct intentionally holds secrets
Scope string `json:"scope"`
}
// NewBearerHandler creates a new BearerHandler
func NewBearerHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
return &bearerHandler{
client: client,
clientID: clientID,
host: host,
credsFn: credsFn,
realm: "",
service: "",
scopes: []string{},
slog: slog,
}
}
// AddScope appends a new scope if it doesn't already exist
func (b *bearerHandler) AddScope(scope string) error {
if b.scopeExists(scope) {
if b.token.Token == "" || !b.isExpired() {
return errs.ErrNoNewChallenge
}
return nil
}
b.addScope(scope)
return nil
}
func (b *bearerHandler) addScope(scope string) {
if !b.tryExtendExistingScope(scope) {
b.scopes = append(b.scopes, scope)
}
// delete old token
b.token.Token = ""
}
var knownActions = []string{"pull", "push", "delete"}
// tryExtendExistingScope extends an existing scope if both the new scope and the current scope contain only knownActions.
// It returns true if actions are added or are already present. Otherwise, it returns false,
// indicating that the new scope should be appended to b.scopes instead.
func (b *bearerHandler) tryExtendExistingScope(scope string) bool {
repo, actions, ok := parseScope(scope)
if !ok {
return false
}
scopePrefix := "repository:" + repo + ":"
for i, cur := range b.scopes {
if !strings.HasPrefix(cur, scopePrefix) {
continue
}
_, curActions, curOk := parseScope(cur)
if !curOk {
continue
}
for _, a := range actions {
if !slices.Contains(curActions, a) {
curActions = append(curActions, a)
}
}
b.scopes[i] = scopePrefix + strings.Join(curActions, ",")
return true
}
return false
}
// parseScope splits a scope into the repo and slice of actions.
// Unknown actions in the scope will set bool to false.
func parseScope(scope string) (string, []string, bool) {
scopeSplit := strings.SplitN(scope, ":", 3)
if scopeSplit[0] != "repository" || len(scopeSplit) < 3 {
return "", nil, false
}
actionSplit := strings.Split(scopeSplit[2], ",")
for _, a := range actionSplit {
if !slices.Contains(knownActions, a) {
return "", nil, false
}
}
return scopeSplit[1], actionSplit, true
}
// ProcessChallenge handles WWW-Authenticate header for bearer tokens
// Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push"
func (b *bearerHandler) ProcessChallenge(c challenge) error {
if _, ok := c.params["realm"]; !ok {
return errs.ErrInvalidChallenge
}
if _, ok := c.params["service"]; !ok {
c.params["service"] = ""
}
if _, ok := c.params["scope"]; !ok {
c.params["scope"] = ""
}
existingScope := b.scopeExists(c.params["scope"])
if b.realm == c.params["realm"] && b.service == c.params["service"] && existingScope && (b.token.Token == "" || !b.isExpired()) {
return errs.ErrNoNewChallenge
}
if b.realm == "" {
b.realm = c.params["realm"]
} else if b.realm != c.params["realm"] {
return errs.ErrInvalidChallenge
}
if b.service == "" {
b.service = c.params["service"]
} else if b.service != c.params["service"] {
return errs.ErrInvalidChallenge
}
if !existingScope {
b.addScope(c.params["scope"])
}
return nil
}
// UpdateRequest for BearerHandler adds a bearer token to the request.
func (b *bearerHandler) UpdateRequest(req *http.Request) error {
// handle relative realm values
if b.tokenURL == nil {
u, err := req.URL.Parse(b.realm)
if err != nil {
return err
}
b.tokenURL = u
}
// verify tokenURL is allowed for request URL
if err := regnet.AllowRedirect(*req.URL, *b.tokenURL); err != nil {
return err
}
// if unexpired token already exists, return it
if b.token.Token != "" && !b.isExpired() {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
return nil
}
// attempt to post if a refresh token is available or token auth is being used
cred := b.credsFn(b.host)
if b.token.RefreshToken != "" || cred.Token != "" {
if err := b.tryPost(cred); err == nil {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
return nil
} else if err != errs.ErrHTTPUnauthorized {
return fmt.Errorf("failed to request auth token (post): %w%.0w", err, errs.ErrHTTPUnauthorized)
}
}
// attempt a get (with basic auth if user/pass available)
if err := b.tryGet(cred); err == nil {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.token.Token))
return nil
} else if err != errs.ErrHTTPUnauthorized {
return fmt.Errorf("failed to request auth token (get): %w%.0w", err, errs.ErrHTTPUnauthorized)
}
return errs.ErrHTTPUnauthorized
}
// isExpired returns true when token issue date is either 0, token has expired,
// or will expire within buffer time
func (b *bearerHandler) isExpired() bool {
if b.token.IssuedAt.IsZero() {
return true
}
expireSec := b.token.IssuedAt.Add(time.Duration(b.token.ExpiresIn) * time.Second)
expireSec = expireSec.Add(tokenBuffer * -1)
return time.Now().After(expireSec)
}
// tryGet requests a new token with a GET request
func (b *bearerHandler) tryGet(cred Cred) error {
//#nosec G704 inputs follow specification
req, err := http.NewRequest("GET", b.tokenURL.String(), nil)
if err != nil {
return err
}
reqParams := req.URL.Query()
reqParams.Add("client_id", b.clientID)
// Note, an offline_token should not be requested by default due to broken OAuth2 implementations returning an invalid token
if b.service != "" {
reqParams.Add("service", b.service)
}
for _, s := range b.scopes {
reqParams.Add("scope", s)
}
if cred.User != "" && cred.Password != "" {
reqParams.Add("account", cred.User)
req.SetBasicAuth(cred.User, cred.Password)
}
req.Header.Add("User-Agent", b.clientID)
req.URL.RawQuery = reqParams.Encode()
//#nosec G704 inputs follow specification
resp, err := b.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return b.validateResponse(resp)
}
// tryPost requests a new token via a POST request
func (b *bearerHandler) tryPost(cred Cred) error {
form := url.Values{}
if len(b.scopes) > 0 {
form.Set("scope", strings.Join(b.scopes, " "))
}
if b.service != "" {
form.Set("service", b.service)
}
form.Set("client_id", b.clientID)
if b.token.RefreshToken != "" {
form.Set("grant_type", "refresh_token")
form.Set("refresh_token", b.token.RefreshToken)
} else if cred.Token != "" {
form.Set("grant_type", "refresh_token")
form.Set("refresh_token", cred.Token)
} else if cred.User != "" && cred.Password != "" {
form.Set("grant_type", "password")
form.Set("username", cred.User)
form.Set("password", cred.Password)
}
//#nosec G704 inputs are user controlled or follow specification
req, err := http.NewRequest("POST", b.tokenURL.String(), strings.NewReader(form.Encode()))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
req.Header.Add("User-Agent", b.clientID)
//#nosec G704 inputs are user controlled or follow specification
resp, err := b.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return b.validateResponse(resp)
}
// scopeExists check if the scope already exists within the list of scopes
func (b *bearerHandler) scopeExists(search string) bool {
if search == "" {
return true
}
searchRepo, searchActions, searchOk := parseScope(search)
if !searchOk {
return slices.Contains(b.scopes, search)
}
scopePrefix := "repository:" + searchRepo + ":"
for _, scope := range b.scopes {
if scope == search {
return true
}
if !strings.HasPrefix(scope, scopePrefix) {
continue
}
_, actions, ok := parseScope(scope)
if !ok {
continue
}
for _, sa := range searchActions {
if !slices.Contains(actions, sa) {
return false
}
}
return true
}
return false
}
// validateResponse extracts the returned token
func (b *bearerHandler) validateResponse(resp *http.Response) error {
if resp.StatusCode != 200 {
return errs.ErrHTTPUnauthorized
}
// decode response and if successful, update token
decoder := json.NewDecoder(resp.Body)
decoded := bearerToken{}
if err := decoder.Decode(&decoded); err != nil {
return err
}
b.token = decoded
if b.token.ExpiresIn < minTokenLife {
b.token.ExpiresIn = minTokenLife
}
// If token is already expired, it was sent with a zero value or
// there may be a clock skew between the client and auth server.
// Also handle cases of remote time in the future.
// But if remote time is slightly in the past, leave as is so token
// expires here before the server.
if b.isExpired() || b.token.IssuedAt.After(time.Now()) {
b.token.IssuedAt = time.Now().UTC()
}
// AccessToken and Token should be the same and we use Token elsewhere
if b.token.AccessToken != "" {
b.token.Token = b.token.AccessToken
}
return nil
}
// jwtHubHandler supports JWT auth type requests.
type jwtHubHandler struct {
client *http.Client
clientID string
realm string
host string
credsFn CredsFn
jwt string
}
type jwtHubPost struct {
User string `json:"username"`
Pass string `json:"password"` //#nosec G117 exported struct intentionally holds secrets
}
type jwtHubResp struct {
Detail string `json:"detail"`
Token string `json:"token"`
RefreshToken string `json:"refresh_token"` //#nosec G117 exported struct intentionally holds secrets
}
// NewJWTHubHandler creates a new JWTHandler for Docker Hub.
func NewJWTHubHandler(client *http.Client, clientID, host string, credsFn CredsFn, slog *slog.Logger) handler {
// JWT handler is only tested against Hub, and the API is Hub specific
if host == "hub.docker.com" {
return &jwtHubHandler{
client: client,
clientID: clientID,
host: host,
credsFn: credsFn,
realm: "https://hub.docker.com/v2/users/login",
}
}
return nil
}
// AddScope is not valid for JWTHubHandler
func (j *jwtHubHandler) AddScope(scope string) error {
return errs.ErrNoNewChallenge
}
// ProcessChallenge handles WWW-Authenticate header for JWT auth on Docker Hub
func (j *jwtHubHandler) ProcessChallenge(c challenge) error {
cred := j.credsFn(j.host)
// use token if provided
if cred.Token != "" {
j.jwt = cred.Token
return nil
}
// send a login request to hub
bodyBytes, err := json.Marshal(jwtHubPost{
User: cred.User,
Pass: cred.Password, //#nosec G117 field name follows spec
})
if err != nil {
return err
}
req, err := http.NewRequest("POST", j.realm, bytes.NewReader(bodyBytes))
if err != nil {
return err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
req.Header.Add("User-Agent", j.clientID)
//#nosec G704 inputs are user controlled or follow specification requirements
resp, err := j.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != 200 || resp.StatusCode >= 300 {
return errs.ErrHTTPUnauthorized
}
var bodyParsed jwtHubResp
err = json.Unmarshal(body, &bodyParsed)
if err != nil {
return err
}
j.jwt = bodyParsed.Token
return nil
}
// UpdateRequest for JWTHubHandler adds JWT header
func (j *jwtHubHandler) UpdateRequest(req *http.Request) error {
if len(j.jwt) > 0 {
req.Header.Set("Authorization", fmt.Sprintf("JWT %s", j.jwt))
return nil
}
return errs.ErrHTTPUnauthorized
}
-48
View File
@@ -1,48 +0,0 @@
package auth
import (
"github.com/regclient/regclient/types/errs"
)
var (
// ErrEmptyChallenge indicates an issue with the received challenge in the WWW-Authenticate header
//
// Deprecated: replace with [errs.ErrEmptyChallenge].
//go:fix inline
ErrEmptyChallenge = errs.ErrEmptyChallenge
// ErrInvalidChallenge indicates an issue with the received challenge in the WWW-Authenticate header
//
// Deprecated: replace with [errs.ErrInvalidChallenge].
//go:fix inline
ErrInvalidChallenge = errs.ErrInvalidChallenge
// ErrNoNewChallenge indicates a challenge update did not result in any change
//
// Deprecated: replace with [errs.ErrNoNewChallenge].
//go:fix inline
ErrNoNewChallenge = errs.ErrNoNewChallenge
// ErrNotFound indicates no credentials found for basic auth
//
// Deprecated: replace with [errs.ErrNotFound].
//go:fix inline
ErrNotFound = errs.ErrNotFound
// ErrNotImplemented returned when method has not been implemented yet
//
// Deprecated: replace with [errs.ErrNotImplemented].
//go:fix inline
ErrNotImplemented = errs.ErrNotImplemented
// ErrParseFailure indicates the WWW-Authenticate header could not be parsed
//
// Deprecated: replace with [errs.ErrParseFailure].
//go:fix inline
ErrParseFailure = errs.ErrParsingFailed
// ErrUnauthorized request was not authorized
//
// Deprecated: replace with [errs.ErrUnauthorized].
//go:fix inline
ErrUnauthorized = errs.ErrHTTPUnauthorized
// ErrUnsupported indicates the request was unsupported
//
// Deprecated: replace with [errs.ErrUnsupported].
//go:fix inline
ErrUnsupported = errs.ErrUnsupported
)
-181
View File
@@ -1,181 +0,0 @@
//go:build go1.18
// Package cache is used to store values with limits.
// Items are automatically pruned when too many entries are stored, or values become stale.
package cache
import (
"sort"
"sync"
"time"
"github.com/regclient/regclient/types/errs"
)
type Cache[k comparable, v any] struct {
mu sync.Mutex
minAge time.Duration
maxAge time.Duration
minCount int
maxCount int
timer *time.Timer
entries map[k]*Entry[v]
}
type Entry[v any] struct {
used time.Time
value v
}
type sortKeys[k comparable] struct {
keys []k
lessFn func(a, b k) bool
}
type conf struct {
minAge time.Duration
maxCount int
}
type cacheOpts func(*conf)
func WithAge(age time.Duration) cacheOpts {
return func(c *conf) {
c.minAge = age
}
}
func WithCount(count int) cacheOpts {
return func(c *conf) {
c.maxCount = count
}
}
func New[k comparable, v any](opts ...cacheOpts) Cache[k, v] {
c := conf{}
for _, opt := range opts {
opt(&c)
}
maxAge := c.minAge + (c.minAge / 10)
minCount := 0
if c.maxCount > 0 {
minCount = int(float64(c.maxCount) * 0.9)
}
return Cache[k, v]{
minAge: c.minAge,
maxAge: maxAge,
minCount: minCount,
maxCount: c.maxCount,
entries: map[k]*Entry[v]{},
}
}
func (c *Cache[k, v]) Delete(key k) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
delete(c.entries, key)
if len(c.entries) == 0 && c.timer != nil {
c.timer.Stop()
c.timer = nil
}
}
func (c *Cache[k, v]) Set(key k, val v) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
c.entries[key] = &Entry[v]{
used: time.Now(),
value: val,
}
if len(c.entries) > c.maxCount {
c.pruneLocked()
} else if c.timer == nil {
// prune resets the timer, so this is only needed if the prune wasn't triggered
c.timer = time.AfterFunc(c.maxAge, c.prune)
}
}
func (c *Cache[k, v]) Get(key k) (v, error) {
if c == nil {
var val v
return val, errs.ErrNotFound
}
c.mu.Lock()
defer c.mu.Unlock()
if e, ok := c.entries[key]; ok {
if e.used.Add(c.minAge).Before(time.Now()) {
// entry expired
go c.prune()
} else {
c.entries[key].used = time.Now()
return e.value, nil
}
}
var val v
return val, errs.ErrNotFound
}
func (c *Cache[k, v]) prune() {
c.mu.Lock()
defer c.mu.Unlock()
c.pruneLocked()
}
func (c *Cache[k, v]) pruneLocked() {
// sort key list by last used date
keyList := make([]k, 0, len(c.entries))
for key := range c.entries {
keyList = append(keyList, key)
}
sk := sortKeys[k]{
keys: keyList,
lessFn: func(a, b k) bool {
return c.entries[a].used.Before(c.entries[b].used)
},
}
sort.Sort(&sk)
// prune entries
now := time.Now()
cutoff := now.Add(c.minAge * -1)
nextTime := now
delCount := len(keyList) - c.minCount
for i, key := range keyList {
if i < delCount || c.entries[key].used.Before(cutoff) {
delete(c.entries, key)
} else {
nextTime = c.entries[key].used
break
}
}
// set next timer
if len(c.entries) > 0 {
dur := nextTime.Sub(now) + c.maxAge
if c.timer == nil {
// this shouldn't be possible
c.timer = time.AfterFunc(dur, c.prune)
} else {
c.timer.Reset(dur)
}
} else if c.timer != nil {
c.timer.Stop()
c.timer = nil
}
}
func (sk *sortKeys[k]) Len() int {
return len(sk.keys)
}
func (sk *sortKeys[k]) Less(i, j int) bool {
return sk.lessFn(sk.keys[i], sk.keys[j])
}
func (sk *sortKeys[k]) Swap(i, j int) {
sk.keys[i], sk.keys[j] = sk.keys[j], sk.keys[i]
}
-188
View File
@@ -1,188 +0,0 @@
// Package conffile wraps the read and write of configuration files
package conffile
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"os/user"
"path/filepath"
)
type File struct {
perms int
fullname string
}
type Opt func(*File)
// New returns a new File.
// The last successful option determines the filename.
func New(opts ...Opt) *File {
f := File{perms: 0o600}
for _, fn := range opts {
fn(&f)
}
if f.fullname == "" {
return nil
}
return &f
}
// WithAppDir determines the filename from the XDG or Windows specification.
// By default, this is based in $HOME/.config on Linux and %APPDATA% on Windows.
// If the file does not exist, this will set the filename only if "force" is true.
func WithAppDir(unixDir, winDir, name string, force bool) Opt {
var dir string
if winDir == "" {
dir = unixDir
} else {
dir = osString(unixDir, winDir)
}
return func(f *File) {
fullname := filepath.Join(appDir(), dir, name)
if force || exists(fullname) {
f.fullname = fullname
}
}
}
// WithDirName determines the filename from a subdirectory in the user's HOME.
//
// Deprecated: Replace with [WithHomeDir]
//
//go:fix inline
func WithDirName(dir, name string) Opt {
return WithHomeDir(dir, name, true)
}
// WithEnvFile sets the fullname to the environment value if defined.
func WithEnvFile(envVar string) Opt {
return func(f *File) {
val := os.Getenv(envVar)
if val != "" {
f.fullname = val
}
}
}
// WithEnvDir sets the fullname to the environment value + filename if the environment variable is defined.
func WithEnvDir(envVar, name string) Opt {
return func(f *File) {
val := os.Getenv(envVar)
if val != "" {
f.fullname = filepath.Join(val, name)
}
}
}
// WithFullname specifies the filename.
// This will always set the filename even if the file does not exist.
func WithFullname(fullname string) Opt {
return func(f *File) {
f.fullname = fullname
}
}
// WithHomeDir determines the filename from a subdirectory in the user's HOME
// e.g. dir=".app", name="config.json", sets the fullname to "$HOME/.app/config.json".
// If the file does not exist, this will set the filename only if "force" is true.
func WithHomeDir(dir, name string, force bool) Opt {
return func(f *File) {
filename := filepath.Join(homeDir(), dir, name)
if force || exists(filename) {
f.fullname = filename
}
}
}
// WithPerms specifies the permissions to create a file with (default 0600).
func WithPerms(perms int) Opt {
return func(f *File) {
f.perms = perms
}
}
func (f *File) Name() string {
return f.fullname
}
func (f *File) Open() (io.ReadCloser, error) {
return os.Open(f.fullname)
}
func (f *File) Write(rdr io.Reader) error {
// create temp file/open
dir := filepath.Dir(f.fullname)
if err := os.MkdirAll(dir, 0o700); err != nil {
return err
}
tmp, err := os.CreateTemp(dir, filepath.Base(f.fullname))
if err != nil {
return err
}
tmpStat, err := tmp.Stat()
if err != nil {
return err
}
tmpName := tmpStat.Name()
tmpFullname := filepath.Join(dir, tmpName)
defer os.Remove(tmpFullname)
// copy from rdr to temp file
_, err = io.Copy(tmp, rdr)
errC := tmp.Close()
if err != nil {
return fmt.Errorf("failed to write config: %w", err)
}
if errC != nil {
return fmt.Errorf("failed to close config: %w", errC)
}
// adjust file ownership/permissions
mode := os.FileMode(0o600)
uid := os.Getuid()
gid := os.Getgid()
// adjust defaults based on existing file if available
stat, err := os.Stat(f.fullname)
if err == nil {
// adjust mode to existing file
if stat.Mode().IsRegular() {
mode = stat.Mode()
}
uid, gid, _ = getFileOwner(stat)
} else if !errors.Is(err, fs.ErrNotExist) {
return err
}
// update mode and owner of temp file
//#nosec G703 tempfile location is user controlled
if err := os.Chmod(tmpFullname, mode); err != nil {
return err
}
if uid > 0 && gid > 0 {
//#nosec G703 tempfile location is user controlled
_ = os.Chown(tmpFullname, uid, gid)
}
// move temp file to target filename
//#nosec G703 tempfile location is user controlled
return os.Rename(tmpFullname, f.fullname)
}
func exists(name string) bool {
_, err := os.Stat(name)
return err == nil
}
func homeDir() string {
home := os.Getenv(homeEnv)
if home == "" {
u, err := user.Current()
if err == nil {
home = u.HomeDir
}
}
return home
}
@@ -1,37 +0,0 @@
//go:build !windows
package conffile
import (
"io/fs"
"os"
"path/filepath"
"syscall"
)
const (
appDirEnv = "XDG_CONFIG_HOME"
homeEnv = "HOME"
)
func appDir() string {
appDir := os.Getenv(appDirEnv)
if appDir == "" {
home := homeDir()
appDir = filepath.Join(home, ".config")
}
return appDir
}
func getFileOwner(stat fs.FileInfo) (int, int, error) {
var uid, gid int
if sysstat, ok := stat.Sys().(*syscall.Stat_t); ok {
uid = int(sysstat.Uid)
gid = int(sysstat.Gid)
}
return uid, gid, nil
}
func osString(unix, _ string) string {
return unix
}
@@ -1,31 +0,0 @@
//go:build windows
package conffile
import (
"io/fs"
"os"
"path/filepath"
)
const (
appDirEnv = "APPDATA"
homeEnv = "USERPROFILE"
)
func appDir() string {
appDir := os.Getenv(appDirEnv)
if appDir == "" {
home := homeDir()
appDir = filepath.Join(home, "AppData")
}
return appDir
}
func getFileOwner(_ fs.FileInfo) (int, int, error) {
return 0, 0, nil
}
func osString(_, win string) string {
return win
}
-198
View File
@@ -1,198 +0,0 @@
// Package httplink parses the Link header from HTTP responses according to RFC5988
package httplink
import (
"fmt"
"strings"
"github.com/regclient/regclient/types/errs"
)
type (
Links []Link
Link struct {
URI string
Param map[string]string
}
)
type charLU byte
var charLUs [256]charLU
const (
isSpace charLU = 1 << iota
isToken
isAlphaNum
)
func init() {
for c := range 256 {
charLUs[c] = 0
if strings.ContainsRune(" \t\r\n", rune(c)) {
charLUs[c] |= isSpace
}
if (rune('a') <= rune(c) && rune(c) <= rune('z')) || (rune('A') <= rune(c) && rune(c) <= rune('Z') || (rune('0') <= rune(c) && rune(c) <= rune('9'))) {
charLUs[c] |= isAlphaNum | isToken
}
if strings.ContainsRune("!#$%&'()*+-./:<=>?@[]^_`{|}~", rune(c)) {
charLUs[c] |= isToken
}
}
}
// Parse reads "Link" http headers into an array of Link structs.
// Header array should be the output of resp.Header.Values("link").
func Parse(headers []string) (Links, error) {
links := []Link{}
for _, h := range headers {
state := "init"
var ub, pnb, pvb []byte
parms := map[string]string{}
endLink := func() {
links = append(links, Link{
URI: string(ub),
Param: parms,
})
// reset state
ub, pnb, pvb = []byte{}, []byte{}, []byte{}
parms = map[string]string{}
}
endParm := func() {
if _, ok := parms[string(pnb)]; !ok {
parms[string(pnb)] = string(pvb)
}
// reset parm
pnb, pvb = []byte{}, []byte{}
}
for i, b := range []byte(h) {
switch state {
case "init":
if b == '<' {
state = "uriQuoted"
} else if charLUs[b]&isToken != 0 {
state = "uri"
ub = append(ub, b)
} else if charLUs[b]&isSpace != 0 || b == ',' {
// noop
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "uri":
// parse tokens until space or comma
if charLUs[b]&isToken != 0 {
ub = append(ub, b)
} else if charLUs[b]&isSpace != 0 {
state = "fieldSep"
} else if b == ';' {
state = "parmName"
} else if b == ',' {
state = "init"
endLink()
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "uriQuoted":
// parse tokens until quote
if b == '>' {
state = "fieldSep"
} else {
ub = append(ub, b)
}
case "fieldSep":
if b == ';' {
state = "parmName"
} else if b == ',' {
state = "init"
endLink()
} else if charLUs[b]&isSpace != 0 {
// noop
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "parmName":
if len(pnb) > 0 && b == '=' {
state = "parmValue"
} else if len(pnb) > 0 && b == '*' {
state = "parmNameStar"
} else if charLUs[b]&isAlphaNum != 0 {
pnb = append(pnb, b)
} else if len(pnb) == 0 && charLUs[b]&isSpace != 0 {
// noop
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "parmNameStar":
if b == '=' {
state = "parmValue"
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
case "parmValue":
if len(pvb) == 0 {
if charLUs[b]&isToken != 0 {
pvb = append(pvb, b)
} else if b == '"' {
state = "parmValueQuoted"
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
} else {
if charLUs[b]&isToken != 0 {
pvb = append(pvb, b)
} else if charLUs[b]&isSpace != 0 {
state = "fieldSep"
endParm()
} else if b == ';' {
state = "parmName"
endParm()
} else if b == ',' {
state = "init"
endParm()
endLink()
} else {
// unknown character
return nil, fmt.Errorf("unknown character in position %d of %s: %w", i, h, errs.ErrParsingFailed)
}
}
case "parmValueQuoted":
if b == '"' {
state = "fieldSep"
endParm()
} else {
pvb = append(pvb, b)
}
}
}
// check for valid state at end of header
switch state {
case "parmValue":
endParm()
endLink()
case "uri", "fieldSep":
endLink()
case "init":
// noop
default:
return nil, fmt.Errorf("unexpected end state %s for header %s: %w", state, h, errs.ErrParsingFailed)
}
}
return links, nil
}
// Get returns a link with a specific parm value, e.g. rel="next"
func (links Links) Get(parm, val string) (Link, error) {
for _, link := range links {
if link.Param != nil && link.Param[parm] == val {
return link, nil
}
}
return Link{}, errs.ErrNotFound
}
-29
View File
@@ -1,29 +0,0 @@
// Package limitread provides a reader that will error if the limit is ever exceeded
package limitread
import (
"fmt"
"io"
"github.com/regclient/regclient/types/errs"
)
type LimitRead struct {
Reader io.Reader
Limit int64
}
func (lr *LimitRead) Read(p []byte) (int, error) {
if lr.Limit < 0 {
return 0, fmt.Errorf("read limit exceeded%.0w", errs.ErrSizeLimitExceeded)
}
if int64(len(p)) > lr.Limit+1 {
p = p[0 : lr.Limit+1]
}
n, err := lr.Reader.Read(p)
lr.Limit -= int64(n)
if lr.Limit < 0 {
return n, fmt.Errorf("read limit exceeded%.0w", errs.ErrSizeLimitExceeded)
}
return n, err
}
-257
View File
@@ -1,257 +0,0 @@
// Package pqueue implements a priority queue.
package pqueue
import (
"context"
"fmt"
"slices"
"sync"
)
type Queue[T any] struct {
mu sync.Mutex
max int
next func(queued, active []*T) int
active []*T
queued []*T
wait []*chan struct{}
}
// Opts is used to configure a new priority queue.
type Opts[T any] struct {
Max int // maximum concurrent entries, defaults to 1.
Next func(queued, active []*T) int // function to lookup index of next queued entry to release, defaults to oldest entry.
}
// New creates a new priority queue.
func New[T any](opts Opts[T]) *Queue[T] {
if opts.Max <= 0 {
opts.Max = 1
}
return &Queue[T]{
max: opts.Max,
next: opts.Next,
}
}
// Acquire adds a new entry to the queue and returns once it is ready.
// The returned function must be called when the queued job completes to release the next entry.
// If there is any error, the returned function will be nil.
func (q *Queue[T]) Acquire(ctx context.Context, e T) (func(), error) {
if q == nil {
return func() {}, nil
}
found, err := q.checkContext(ctx)
if err != nil {
return nil, err
}
if found {
return func() {}, nil
}
q.mu.Lock()
if len(q.active)+len(q.queued) < q.max {
q.active = append(q.active, &e)
q.mu.Unlock()
return q.releaseFn(&e), nil
}
// limit reached, add to queue and wait
w := make(chan struct{}, 1)
q.queued = append(q.queued, &e)
q.wait = append(q.wait, &w)
q.mu.Unlock()
// wait on both context and queue
select {
case <-ctx.Done():
// context abort, remove queued entry
q.mu.Lock()
if i := slices.Index(q.queued, &e); i >= 0 {
q.queued = slices.Delete(q.queued, i, i+1)
q.wait = slices.Delete(q.wait, i, i+1)
q.mu.Unlock()
return nil, ctx.Err()
}
q.mu.Unlock()
// queued entry found, assume race condition with context and entry being released, release next entry
q.release(&e)
return nil, ctx.Err()
case <-w:
return q.releaseFn(&e), nil
}
}
// TryAcquire attempts to add an entry on to the list of active entries.
// If the returned function is nil, the queue was not available.
// If the returned function is not nil, it must be called when the job is complete to release the next entry.
func (q *Queue[T]) TryAcquire(ctx context.Context, e T) (func(), error) {
if q == nil {
return func() {}, nil
}
found, err := q.checkContext(ctx)
if err != nil {
return nil, err
}
if found {
return func() {}, nil
}
q.mu.Lock()
defer q.mu.Unlock()
if len(q.active)+len(q.queued) < q.max {
q.active = append(q.active, &e)
return q.releaseFn(&e), nil
}
return nil, nil
}
// release next entry or noop.
func (q *Queue[T]) release(prev *T) {
q.mu.Lock()
defer q.mu.Unlock()
// remove prev entry from active list
if i := slices.Index(q.active, prev); i >= 0 {
q.active = slices.Delete(q.active, i, i+1)
}
// skip checks when at limit or nothing queued
if len(q.queued) == 0 {
if len(q.active) == 0 {
// free up slices if this was the last active entry
q.active = nil
q.queued = nil
q.wait = nil
}
return
}
if len(q.active) >= q.max {
return
}
i := 0
if q.next != nil && len(q.queued) > 1 {
i = q.next(q.queued, q.active)
// validate response
i = max(min(i, len(q.queued)-1), 0)
}
// release queued entry, move to active list, and remove from queued/wait lists
close(*q.wait[i])
q.active = append(q.active, q.queued[i])
q.queued = slices.Delete(q.queued, i, i+1)
q.wait = slices.Delete(q.wait, i, i+1)
}
// releaseFn is a convenience wrapper around [release].
func (q *Queue[T]) releaseFn(prev *T) func() {
return func() {
q.release(prev)
}
}
// TODO: is there a way to make a different context key for each generic type?
type ctxType int
var ctxKey ctxType
type valMulti[T any] struct {
qList []*Queue[T]
}
// AcquireMulti is used to simultaneously lock multiple queues without the risk of deadlock.
// The returned context needs to be used on calls to [Acquire] or [TryAcquire] which will immediately succeed since the resource is already acquired.
// Attempting to acquire other resources with [Acquire], [TryAcquire], or [AcquireMulti] using the returned context and will fail for being outside of the transaction.
// The returned function must be called to release the resources.
// The returned function is not thread safe, ensure no other simultaneous calls to [Acquire] or [TryAcquire] using the returned context have finished before it is called.
func AcquireMulti[T any](ctx context.Context, e T, qList ...*Queue[T]) (context.Context, func(), error) {
// verify context not already holding locks
qCtx := ctx.Value(ctxKey)
if qCtx != nil {
if qCtxVal, ok := qCtx.(*valMulti[T]); !ok || qCtxVal.qList != nil {
return ctx, nil, fmt.Errorf("context already used by another AcquireMulti request")
}
}
// delete nil entries
for i := len(qList) - 1; i >= 0; i-- {
if qList[i] == nil {
qList = slices.Delete(qList, i, i+1)
}
}
// empty/nil list is a noop
if len(qList) == 0 {
return ctx, func() {}, nil
}
// dedup entries from the list
for i := len(qList) - 2; i >= 0; i-- {
for j := len(qList) - 1; j > i; j-- {
if qList[i] == qList[j] {
qList[j] = qList[len(qList)-1]
qList = qList[:len(qList)-1]
}
}
}
// Loop through queues to acquire, waiting on the first, and attempting the remaining.
// If any of the remaining entries cannot be immediately acquired, reset and make it the new queue to wait on.
lockI := 0
doneList := make([]func(), len(qList))
for {
acquired := true
i := 0
done, err := qList[lockI].Acquire(ctx, e)
if err != nil {
return ctx, nil, err
}
doneList[lockI] = done
for i < len(qList) {
if i != lockI {
doneList[i], err = qList[i].TryAcquire(ctx, e)
if doneList[i] == nil || err != nil {
acquired = false
break
}
}
i++
}
if err == nil && acquired {
break
}
// cleanup on failed attempt
if lockI > i {
doneList[lockI]()
}
// track blocking index for a retry
lockI = i
for i > 0 {
i--
doneList[i]()
}
// abort on errors
if err != nil {
return ctx, nil, err
}
}
// success, update context
ctxVal := valMulti[T]{qList: qList}
newCtx := context.WithValue(ctx, ctxKey, &ctxVal)
cleanup := func() {
ctxVal.qList = nil
// dequeue in reverse order to minimize chance of another AcquireMulti being freed and immediately blocking on the next queue
for i := len(doneList) - 1; i >= 0; i-- {
doneList[i]()
}
}
return newCtx, cleanup, nil
}
func (q *Queue[T]) checkContext(ctx context.Context) (bool, error) {
qCtx := ctx.Value(ctxKey)
if qCtx == nil {
return false, nil
}
qCtxVal, ok := qCtx.(*valMulti[T])
if !ok {
return false, nil // another type is using the context, treat it as unset
}
if qCtxVal.qList == nil {
return false, nil
}
if slices.Contains(qCtxVal.qList, q) {
// instance already locked
return true, nil
}
return true, fmt.Errorf("cannot acquire new locks during a transaction")
}

Some files were not shown because too many files have changed in this diff Show More