Compare commits

...

14 Commits

Author SHA1 Message Date
7c0d883135 chore: new release 2021-10-06 08:48:23 +02:00
e78ced41fb fix: use freifunk DNS resolver
Closes coop-cloud/organising#180.
2021-10-06 08:47:01 +02:00
e9113500d8 feat: allow to override STACK_NAME 2021-10-05 20:40:16 +02:00
7368cabc49 fix: format output correctly 2021-10-05 20:24:52 +02:00
f75e264811 fix: ensure dirs are created
Also use debug logging for help.

Closes coop-cloud/organising#183.
Closes coop-cloud/organising#183.
2021-10-05 20:24:41 +02:00
8bfd76fd04 feat: generate versions for catalogue also
Closes coop-cloud/organising#179.
2021-10-05 20:14:00 +02:00
1cb5e3509d fix: add compose.yml before commiting with recipe release; reset parts of tag according to semver when releasing 2021-10-05 16:36:15 +02:00
3cd2399cca fix: ignore WIP stuff and sort [ci skip] 2021-10-05 11:56:26 +02:00
11c4651a3b fix: don't crash when there is a more serious upgrade available 2021-10-05 09:55:25 +00:00
49f90674f2 fix: --major/minor/patch is the most serious upgrade you want to do 2021-10-05 09:55:25 +00:00
74a70edb03 feat: upgrade an app with no user input with --minor/major/patch flag 2021-10-05 09:55:25 +00:00
6fc5c31347 WIP: #172 upgrade --major/minor/patch placeholder 2021-10-05 09:55:25 +00:00
c616907b71 feat: teach recipe sync to understand new versions
Closes coop-cloud/organising#177.
2021-10-05 10:28:09 +02:00
a58cea3e0a docs: dont assume that yet [ci skip] 2021-10-02 23:30:18 +02:00
14 changed files with 339 additions and 87 deletions

View File

@ -31,9 +31,10 @@ changelog:
sort: asc
filters:
exclude:
- "^WIP:"
- "^chore:"
- "^docs:"
- "^refactor:"
- "^style:"
- "^test:"
- "^tests:"
- "^chore:"

View File

@ -31,7 +31,10 @@ var appDeployCommand = &cli.Command{
for k, v := range abraShEnv {
app.Env[k] = v
}
if _, exists := app.Env["STACK_NAME"]; !exists {
app.Env["STACK_NAME"] = app.StackName()
}
composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
if err != nil {
@ -39,7 +42,7 @@ var appDeployCommand = &cli.Command{
}
deployOpts := stack.Deploy{
Composefiles: composeFiles,
Namespace: app.StackName(),
Namespace: app.Env["STACK_NAME"],
Prune: false,
ResolveImage: stack.ResolveImageAlways,
}

View File

@ -2,6 +2,7 @@ package catalogue
import (
"encoding/json"
"fmt"
"io/ioutil"
"path"
@ -49,7 +50,18 @@ var catalogueGenerateCommand = &cli.Command{
Aliases: []string{"g"},
Usage: "Generate a new copy of the catalogue",
ArgsUsage: "[<recipe>]",
BashComplete: func(c *cli.Context) {},
BashComplete: func(c *cli.Context) {
catl, err := catalogue.ReadRecipeCatalogue()
if err != nil {
logrus.Warn(err)
}
if c.NArg() > 0 {
return
}
for name := range catl {
fmt.Println(name)
}
},
Action: func(c *cli.Context) error {
recipeName := c.Args().First()
@ -60,18 +72,18 @@ var catalogueGenerateCommand = &cli.Command{
logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos))
bar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
ch := make(chan string, len(repos))
for _, repoMeta := range repos {
go func(rm catalogue.RepoMeta) {
if recipeName != "" && recipeName != rm.Name {
ch <- rm.Name
bar.Add(1)
retrieveBar.Add(1)
return
}
if _, exists := CatalogueSkipList[rm.Name]; exists {
ch <- rm.Name
bar.Add(1)
retrieveBar.Add(1)
return
}
@ -86,7 +98,7 @@ var catalogueGenerateCommand = &cli.Command{
}
ch <- rm.Name
bar.Add(1)
retrieveBar.Add(1)
}(repoMeta)
}
@ -95,14 +107,23 @@ var catalogueGenerateCommand = &cli.Command{
}
catl := make(catalogue.RecipeCatalogue)
catlBar := formatter.CreateProgressbar(len(repos), "generating catalogue...")
for _, recipeMeta := range repos {
if recipeName != "" && recipeName != recipeMeta.Name {
catlBar.Add(1)
continue
}
if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
catlBar.Add(1)
continue
}
versions, err := catalogue.GetRecipeVersions(recipeMeta.Name)
if err != nil {
logrus.Fatal(err)
}
catl[recipeMeta.Name] = catalogue.RecipeMeta{
Name: recipeMeta.Name,
Repository: recipeMeta.CloneURL,
@ -110,10 +131,11 @@ var catalogueGenerateCommand = &cli.Command{
DefaultBranch: recipeMeta.DefaultBranch,
Description: recipeMeta.Description,
Website: recipeMeta.Website,
// Versions: ..., // FIXME: once the new versions work goes down
Versions: versions,
// Category: ..., // FIXME: once we sort out the machine-readable catalogue interface
// Features: ..., // FIXME: once we figure out the machine-readable catalogue interface
}
catlBar.Add(1)
}
recipesJSON, err := json.MarshalIndent(catl, "", " ")
@ -125,7 +147,7 @@ var catalogueGenerateCommand = &cli.Command{
logrus.Fatal(err)
}
logrus.Debugf("generated new recipe catalogue in '%s'", config.APPS_JSON)
logrus.Infof("generated new recipe catalogue in '%s'", config.APPS_JSON)
return nil
},

View File

@ -4,6 +4,7 @@ package cli
import (
"fmt"
"os"
"path"
"coopcloud.tech/abra/cli/app"
"coopcloud.tech/abra/cli/catalogue"
@ -82,17 +83,30 @@ func RunApp(version, commit string) {
logrus.SetOutput(os.Stderr)
logrus.AddHook(logrusStack.StandardHook())
}
return nil
paths := []string{
config.ABRA_DIR,
path.Join(config.ABRA_DIR, "servers"),
path.Join(config.ABRA_DIR, "apps"),
path.Join(config.ABRA_DIR, "vendor"),
}
if err := os.Mkdir(config.ABRA_DIR, 0755); err != nil {
for _, path := range paths {
if err := os.Mkdir(path, 0755); err != nil {
if !os.IsExist(err) {
logrus.Fatal(err)
}
logrus.Debugf("'%s' already created, moving on...", path)
continue
}
logrus.Debugf("'%s' is missing, creating...", path)
}
logrus.Debugf("abra version '%s', commit '%s'", version, commit)
return nil
}
if err := app.Run(os.Args); err != nil {
logrus.Fatal(err)
}

View File

@ -36,7 +36,7 @@ var DryFlag = &cli.BoolFlag{
var CommitMessage string
var CommitMessageFlag = &cli.StringFlag{
Name: "commit-message",
Usage: "commit message",
Usage: "commit message. Implies --commit",
Aliases: []string{"cm"},
Destination: &CommitMessage,
}
@ -44,6 +44,7 @@ var CommitMessageFlag = &cli.StringFlag{
var Commit bool
var CommitFlag = &cli.BoolFlag{
Name: "commit",
Usage: "add compose.yml to staging area and commit changes",
Value: false,
Aliases: []string{"c"},
Destination: &Commit,
@ -118,6 +119,12 @@ or a rollback of an app.
}
survey.AskOne(prompt, &CommitMessage)
}
err = commitWorktree.AddGlob("compose.**yml")
if err != nil {
logrus.Fatal(err)
}
logrus.Debug("staged compose.**yml for commit")
_, err = commitWorktree.Commit(CommitMessage, &git.CommitOptions{})
if err != nil {
logrus.Fatal(err)
@ -214,12 +221,15 @@ or a rollback of an app.
if err != nil {
logrus.Fatal(err)
}
newTag.Patch = "0"
newTag.Minor = strconv.Itoa(now + 1)
} else if Major {
now, err := strconv.Atoi(newTag.Major)
if err != nil {
logrus.Fatal(err)
}
newTag.Patch = "0"
newTag.Minor = "0"
newTag.Major = strconv.Itoa(now + 1)
}
newTagString = newTag.String()

View File

@ -4,61 +4,87 @@ import (
"fmt"
"coopcloud.tech/abra/cli/internal"
"coopcloud.tech/abra/pkg/client"
"github.com/docker/distribution/reference"
"coopcloud.tech/abra/pkg/catalogue"
"github.com/AlecAivazis/survey/v2"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
)
var recipeSyncCommand = &cli.Command{
Name: "sync",
Usage: "Generate new recipe labels",
Usage: "Ensure recipe version labels are up-to-date",
Aliases: []string{"s"},
Description: `
This command will generate labels for each service which correspond to the
following format:
This command will generate labels for the main recipe service (i.e. the service
named "app", by convention) which corresponds to the following format:
coop-cloud.${STACK_NAME}.${SERVICE_NAME}.version=${IMAGE_TAG}-${IMAGE_DIGEST}
coop-cloud.${STACK_NAME}.version=${RECIPE_TAG}
The <recipe> configuration will be updated on the local file system. These
labels are consumed by abra in other command invocations and used to determine
the versioning metadata of up-and-running containers are.
The ${RECIPE_TAG} is determined by the recipe maintainer and is retrieved by
this command by asking for the list of git tags on the local git repository.
The <recipe> configuration will be updated on the local file system.
`,
ArgsUsage: "<recipe>",
BashComplete: func(c *cli.Context) {
catl, err := catalogue.ReadRecipeCatalogue()
if err != nil {
logrus.Warn(err)
}
if c.NArg() > 0 {
return
}
for name := range catl {
fmt.Println(name)
}
},
Action: func(c *cli.Context) error {
recipe := internal.ValidateRecipe(c)
mainService := "app"
var services []string
hasAppService := false
for _, service := range recipe.Config.Services {
services = append(services, service.Name)
if service.Name == "app" {
hasAppService = true
logrus.Debugf("detected app service in '%s'", recipe.Name)
}
}
if !hasAppService {
logrus.Fatal(fmt.Sprintf("no 'app' service defined in '%s'", recipe.Name))
logrus.Warnf("no 'app' service defined in '%s'", recipe.Name)
var chosenService string
prompt := &survey.Select{
Message: fmt.Sprintf("what is the main service name for '%s'?", recipe.Name),
Options: services,
}
if err := survey.AskOne(prompt, &chosenService); err != nil {
logrus.Fatal(err)
}
mainService = chosenService
}
for _, service := range recipe.Config.Services {
img, err := reference.ParseNormalizedNamed(service.Image)
logrus.Debugf("selecting '%s' as the service to sync version labels", mainService)
tags, err := recipe.Tags()
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("detected image '%s' for service '%s'", img, service.Name)
digest, err := client.GetTagDigest(img)
if err != nil {
if len(tags) == 0 {
logrus.Fatalf("no tags detected for '%s'", recipe.Name)
}
latestTag := tags[len(tags)-1]
logrus.Infof("choosing '%s' as latest tag for recipe '%s'", latestTag, recipe.Name)
label := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", latestTag)
if err := recipe.UpdateLabel(mainService, label); err != nil {
logrus.Fatal(err)
}
logrus.Debugf("retrieved digest '%s' for '%s'", digest, img)
tag := img.(reference.NamedTagged).Tag()
label := fmt.Sprintf("coop-cloud.${STACK_NAME}.%s.version=%s-%s", service.Name, tag, digest)
if err := recipe.UpdateLabel(service.Name, label); err != nil {
logrus.Fatal(err)
}
logrus.Debugf("added label '%s' to service '%s'", label, service.Name)
}
logrus.Infof("added label '%s' to service '%s'", label, mainService)
return nil
},

View File

@ -27,14 +27,24 @@ update the relevant compose file tags on the local file system.
Some image tags cannot be parsed because they do not follow some sort of
semver-like convention. In this case, all possible tags will be listed and it
is up to the end-user to decide.
This is step 1 of upgrading a recipe. Step 2 is running "abra recipe sync
<recipe>".
`,
ArgsUsage: "<recipe>",
Flags: []cli.Flag{
PatchFlag,
MinorFlag,
MajorFlag,
},
Action: func(c *cli.Context) error {
recipe := internal.ValidateRecipe(c)
bumpType := btoi(Major)*4 + btoi(Minor)*2 + btoi(Patch)
if bumpType != 0 {
// a bitwise check if the number is a power of 2
if (bumpType & (bumpType - 1)) != 0 {
logrus.Fatal("you can only use one of: --major, --minor, --patch.")
}
}
for _, service := range recipe.Config.Services {
catlVersions, err := catalogue.VersionsOfService(recipe.Name, service.Name)
if err != nil {
@ -107,7 +117,24 @@ This is step 1 of upgrading a recipe. Step 2 is running "abra recipe sync
}
logrus.Debugf("detected compatible upgradable tags '%s' for '%s'", compatibleStrings, service.Name)
var upgradeTag string
if bumpType != 0 {
for _, upTag := range compatible {
upElement, err := tag.UpgradeElement(upTag)
if err != nil {
return err
}
delta := tagcmp.UpgradeType(upElement)
if delta <= bumpType {
upgradeTag = upTag.String()
break
}
}
if upgradeTag == "" {
logrus.Warnf("not upgrading from '%s' to '%s' for '%s', because the upgrade type is more serious than what user wants.", tag.String(), compatible[0].String(), image)
continue
}
} else {
msg := fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
tag := img.(reference.NamedTagged).Tag()
@ -119,7 +146,6 @@ This is step 1 of upgrading a recipe. Step 2 is running "abra recipe sync
}
}
var upgradeTag string
prompt := &survey.Select{
Message: msg,
Options: compatibleStrings,
@ -127,11 +153,12 @@ This is step 1 of upgrading a recipe. Step 2 is running "abra recipe sync
if err := survey.AskOne(prompt, &upgradeTag); err != nil {
logrus.Fatal(err)
}
}
if err := recipe.UpdateTag(image, upgradeTag); err != nil {
logrus.Fatal(err)
}
logrus.Debugf("tag updated from '%s' to '%s' for '%s'", image, upgradeTag, recipe.Name)
logrus.Infof("tag upgraded from '%s' to '%s' for '%s'", tag.String(), upgradeTag, image)
}
return nil

View File

@ -35,6 +35,9 @@ later for more advanced use cases.
return err
}
// https://www.privacy-handbuch.de/handbuch_93d.htm
freifunkDNS := "5.1.66.255:53"
resolver := &net.Resolver{
PreferGo: false,
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
@ -42,10 +45,10 @@ later for more advanced use cases.
Timeout: time.Millisecond * time.Duration(10000),
}
// comrade librehosters DNS resolver https://snopyta.org/service/dns/
return d.DialContext(ctx, "udp", "95.216.24.230:53")
return d.DialContext(ctx, "udp", freifunkDNS)
},
}
logrus.Debugf("created DNS resolver via 95.216.24.230")
logrus.Debugf("created DNS resolver via '%s'", freifunkDNS)
ctx := context.Background()
ips, err := resolver.LookupIPAddr(ctx, domainName)

2
go.mod
View File

@ -3,7 +3,7 @@ module coopcloud.tech/abra
go 1.17
require (
coopcloud.tech/tagcmp v0.0.0-20210906102006-2a8edd82d75d
coopcloud.tech/tagcmp v0.0.0-20211003080922-7b06d1c16182
github.com/AlecAivazis/survey/v2 v2.3.1
github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4
github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4

4
go.sum
View File

@ -23,6 +23,10 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
coopcloud.tech/tagcmp v0.0.0-20210906102006-2a8edd82d75d h1:5jeUIiToqQ7vTlLeycdGp4Ezurd6/RTNl5K38usHtoo=
coopcloud.tech/tagcmp v0.0.0-20210906102006-2a8edd82d75d/go.mod h1:ESVm0wQKcbcFi06jItF3rI7enf4Jt2PvbkWpDDHk1DQ=
coopcloud.tech/tagcmp v0.0.0-20211003074705-03d2daced95c h1:7kCjnhjrOevcJeA/koCyyv20E6AglvqC7biGbLzyrbU=
coopcloud.tech/tagcmp v0.0.0-20211003074705-03d2daced95c/go.mod h1:ESVm0wQKcbcFi06jItF3rI7enf4Jt2PvbkWpDDHk1DQ=
coopcloud.tech/tagcmp v0.0.0-20211003080922-7b06d1c16182 h1:VGFzudsoGXGRaw5eJE3rErHHUDsmuIJpQkdfKJgrNs4=
coopcloud.tech/tagcmp v0.0.0-20211003080922-7b06d1c16182/go.mod h1:ESVm0wQKcbcFi06jItF3rI7enf4Jt2PvbkWpDDHk1DQ=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/AlecAivazis/survey/v2 v2.3.1 h1:lzkuHA60pER7L4eYL8qQJor4bUWlJe4V0gqAT19tdOA=
github.com/AlecAivazis/survey/v2 v2.3.1/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg=

View File

@ -9,12 +9,17 @@ import (
"io/ioutil"
"net/http"
"os"
"path"
"strings"
"time"
"coopcloud.tech/abra/pkg/client"
"coopcloud.tech/abra/pkg/config"
"coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/web"
"github.com/docker/distribution/reference"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/sirupsen/logrus"
)
@ -56,6 +61,9 @@ type serviceMeta struct {
Tag string `json:"tag"`
}
// RecipeVersions are the versions associated with a recipe.
type RecipeVersions []map[tag]map[service]serviceMeta
// RecipeMeta represents metadata for a recipe in the abra catalogue.
type RecipeMeta struct {
Category string `json:"category"`
@ -65,7 +73,7 @@ type RecipeMeta struct {
Icon string `json:"icon"`
Name string `json:"name"`
Repository string `json:"repository"`
Versions []map[tag]map[service]serviceMeta `json:"versions"`
Versions RecipeVersions `json:"versions"`
Website string `json:"website"`
}
@ -365,3 +373,110 @@ func ReadReposMetadata() (RepoCatalogue, error) {
return reposMeta, nil
}
// GetRecipeVersions retrieves all recipe versions.
func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
versions := RecipeVersions{}
recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
logrus.Debugf("attempting to open git repository in '%s'", recipeDir)
repo, err := git.PlainOpen(recipeDir)
if err != nil {
return versions, err
}
worktree, err := repo.Worktree()
if err != nil {
logrus.Fatal(err)
}
gitTags, err := repo.Tags()
if err != nil {
return versions, err
}
if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")
logrus.Debugf("processing '%s' for '%s'", tag, recipeName)
checkOutOpts := &git.CheckoutOptions{
Create: false,
Force: true,
Keep: false,
Branch: plumbing.ReferenceName(ref.Name()),
}
if err := worktree.Checkout(checkOutOpts); err != nil {
logrus.Debugf("failed to check out '%s' in '%s'", tag, recipeDir)
return err
}
logrus.Debugf("successfully checked out '%s' in '%s'", ref.Name(), recipeDir)
recipe, err := recipe.Get(recipeName)
if err != nil {
return err
}
versionMeta := make(map[string]serviceMeta)
for _, service := range recipe.Config.Services {
img, err := reference.ParseNormalizedNamed(service.Image)
if err != nil {
return err
}
path := reference.Path(img)
if strings.Contains(path, "library") {
path = strings.Split(path, "/")[1]
}
digest, err := client.GetTagDigest(img)
if err != nil {
return err
}
versionMeta[service.Name] = serviceMeta{
Digest: digest,
Image: path,
Tag: img.(reference.NamedTagged).Tag(),
}
logrus.Debugf("collecting digest: '%s', image: '%s', tag: '%s'", digest, path, tag)
}
versions = append(versions, map[string]map[string]serviceMeta{tag: versionMeta})
return nil
}); err != nil {
return versions, err
}
branch := "master"
if _, err := repo.Branch("master"); err != nil {
if _, err := repo.Branch("main"); err != nil {
logrus.Debugf("failed to select branch in '%s'", recipeDir)
logrus.Fatal(err)
}
branch = "main"
}
refName := fmt.Sprintf("refs/heads/%s", branch)
checkOutOpts := &git.CheckoutOptions{
Create: false,
Force: true,
Keep: false,
Branch: plumbing.ReferenceName(refName),
}
if err := worktree.Checkout(checkOutOpts); err != nil {
logrus.Debugf("failed to check out '%s' in '%s'", branch, recipeDir)
logrus.Fatal(err)
}
logrus.Debugf("switched back to '%s' in '%s'", branch, recipeDir)
logrus.Debugf("collected '%s' for '%s'", versions, recipeName)
return versions, nil
}

View File

@ -139,7 +139,7 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
}
}
logrus.Debugf("collecting metadata from '%v' servers: '%s'", len(servers), servers)
logrus.Debugf("collecting metadata from '%v' servers: '%s'", len(servers), strings.Join(servers, ", "))
for _, server := range servers {
serverDir := path.Join(ABRA_SERVER_FOLDER, server)

View File

@ -41,6 +41,33 @@ func (r Recipe) UpdateTag(image, tag string) error {
return nil
}
// Tags list the recipe tags
func (r Recipe) Tags() ([]string, error) {
var tags []string
recipeDir := path.Join(config.ABRA_DIR, "apps", r.Name)
repo, err := git.PlainOpen(recipeDir)
if err != nil {
return tags, err
}
gitTags, err := repo.Tags()
if err != nil {
return tags, err
}
if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
tags = append(tags, strings.TrimPrefix(string(ref.Name()), "refs/tags/"))
return nil
}); err != nil {
return tags, err
}
logrus.Debugf("detected '%s' as tags for recipe '%s'", strings.Join(tags, ", "), r.Name)
return tags, nil
}
// Get retrieves a recipe.
func Get(recipeName string) (Recipe, error) {
if err := EnsureExists(recipeName); err != nil {

View File

@ -1,6 +1,6 @@
#!/usr/bin/env bash
ABRA_VERSION="0.1.6-alpha"
ABRA_VERSION="0.1.7-alpha"
ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
function show_banner {