From 7b0fb50e7f08158190b8d8e1d6ea39b325e7cfce Mon Sep 17 00:00:00 2001
From: decentral1se <hi@decentral1.se>
Date: Sat, 5 Jun 2021 07:55:05 +0200
Subject: [PATCH] Abstract common functions into a library
---
bin/__pycache__/abralib.cpython-39.pyc | Bin 0 -> 2782 bytes
bin/abralib.py | 103 +++++++++++++++++++++++
bin/app-json.py | 109 +++----------------------
bin/clone-all-apps.py | 16 ++++
bin/renovate-ls-apps.py | 72 +---------------
5 files changed, 132 insertions(+), 168 deletions(-)
create mode 100644 bin/__pycache__/abralib.cpython-39.pyc
create mode 100644 bin/abralib.py
create mode 100755 bin/clone-all-apps.py
diff --git a/bin/__pycache__/abralib.cpython-39.pyc b/bin/__pycache__/abralib.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..23cc42cb0d8e8654cca93db002370aaebc0a99dd
GIT binary patch
literal 2782
zcmZWr&2JmW6`z@1F3BZDQD1iCIL=0C1273m2PlxXaNVY|YzIkg#ZnSvkzl*r8If!4
z?yP2pva}$5sC-R<o&y6i&`W`!r`-B~^swkHx%ix$PI)t=T*X;p-@Nzc&70Zxe(w##
zg#{PEv%d77zwQ9{y%sJ$76^ZWSN;HmA|FvqNrF@2lhpF9)b?$xbyiYI9p6c-e$~i!
zQcLT8J#~H8$d#m#HvOjIon#?x`7Og&lf|^{w+&xQmeOT^+3@vbC0+GbF=8%tPq4qn
znr!VGL>o`Z2|m5_4EdL7(|={IUC>&89i%I>*@v(CSDDRT1MM|tognaY?euj<o?-BQ
zi7wLi6YP6*i7r1u{u^|KuEMxM*LG0na`D#AC=`r(V-+W{ikbBGxbOyX)_=Q~AA2$q
zajs<VIYb0?tg>=z>(8IwDI0@O#?dCv_Tph#A2RhZ9}b!5kg^huXe`P~x(@@$4m*ma
zlBLUz@-U-g30AnYw?j38u_ANW6_{o;V$puU$0{Fd*@k6$w7_>k9`}nzzZVWf*bnnu
zc9wiw3q@Viu2vPUVTZ6qTr>?of|VH)g*ynN{c+wMa8<M-&hseYW7<<J8u?Dd)mUUj
zoAPL%iEfm4Da#XnT&&GCG|p&u5GvW-kN9D+0&3ZN$xD?Tg=wBJzdj5FWm%jJeU}S3
zOe{6;<j3Z?OCqFkl(4R3jCM0lS<#jXPS=%kbiVO|d=!eb8ws90<b$F;|L#MUQ7-z&
z2e%`lb1{D^FltbL1h4!X5QQe_1P}4lnqYBtf;30*1jl$viYp3FtQbwmKk?u3DWL$7
zQ|lS}9PJ<*Df@VW(G(zF^i<{4q1FTe2yIoJ*eCcK^widXQZ-sRf$?+n1)|Q>er#v%
zY`l)Hg9boVEmj|lGcWYyh$V>^@ic|B^dQ+<E-GWS*ZpnLx*f&|08epGfc2U4t|!As
z-g68eaj#Ps`m00(Na;`(L9AfC0x*{Kt)qzLD&|>fC45*`m=Ii)7R-t!*mo+W6{S?P
z%-Vg3=1`Wlp>=k?^j}E43iD_0^hZ2peb|9zN`%RKlFb7yMby`2pfi`m11N^F7QjgZ
z@clL{mCHa7afyu`Tq6Wu!7X^jYoLwv)cJFsqD@f9?|@J|g*2T)Z_xpMYU*@|rqEZW
zgxXW9xH}<&8jH%rI=4_@O`Vy18PO`OeNA-oreuOo?TPU+v8eq%npy`N6Fg69<@nz(
zl8PXyb(2*07fFQ_&y(7iC)JE5sSQYKvv}Qmz#swaka>dT+)JRYy*<HG?@p{(=oRme
zl*;Ar`~6|8df`~{jHhwbD|p7BY~%jndwnPh+1J(6HAUFhj^tTm?igyt+ZAmHQL(lw
zj<pEr*kdXhd3iWwUg7*vkAHZs!O_8fRmCZTo>8n^0Mm;@E`pIp(khIl9cR$k7J&8b
zhZ3*@jY&aGi4{<ZHBByQa*m*}NXm*io?jD8<~)<kxApGgini0}nUSl&l`ZH~fiVxX
z_uqj|UIT(Go3sHU@B@h0xD60-u}hp8SbhZ-kw*OaV*^uz*MQ}Jz$m2f36^a|^k}XN
z@&s!97F78Z>iQP?>boz=lt8@`WgQ~%n}ZF#1}Yac03Higf2-`%3bikQ$)N^X)VW~k
zX#c8uK?7@b#Vbd{jn==0i%0&~_;QVJ13H*(Xgk<f@6x<5^Gjb2U6|R@7JzE8cxN*K
zcz7Z7iA~<+xd-x?db*|S+&{k1^?N@Hlx~8$vUcE=It4=r%2%s8BowV9pm-e($v?<}
z4I7ct&cc-WRcN#Uq}F5(w4`^v#QDp~bZ=;EL*L48eg>X@gclVXfY^)$aMD1Zu6HBh
zQJBb^z?>VO;l9vGFI=bq&)9os*elkx2yG+-9Cy2>ZYJKN$m=G%9c=nk@y4Aollrpo
z(lE}>jbrKE-TL5`xsr<|<5XXhfOun8)IZH>sPw9$sb^=U(hW80t8kFAvH9`+ds{of
z_6NIng`+hK54N`N?*zN|gPo5)+%~l#JeV+jOX!G}R-CDhUAq5=vv^Zm{92O@O<o57
z&qMPGm`D!@A~ooYEz-m_OvthZzoBg?fHq#n;yT#e>)_j+H_K`ez||TAeieH95I#xh
z3=2yWT@)qZ(j3oxhF|5fm&1Pu^O-_dOj*@3bZSut$%o4(VMpc`DP1`p<bp>K;n|I1
z$zy3MUDx&jL|(~RYpCD7|CcQjEx#5HBu~bQiFfrGy4q>=i%-lX+yucM++IOYIywA(
z%9P*O*?jQf_U?;m4YwdL@%3vG{vbi$OlCpWMQ=X+LL5y*O(&_{NI4xR?51uD@+}}X
dS%uKGadY+~O$h5UwpUuX1)PIj%U!I!@)KpE(pdli
literal 0
HcmV?d00001
diff --git a/bin/abralib.py b/bin/abralib.py
new file mode 100644
index 00000000000..e7722b73807
--- /dev/null
+++ b/bin/abralib.py
@@ -0,0 +1,103 @@
+"""Shared utilities for bin/*.py scripts."""
+
+from logging import DEBUG, basicConfig, getLogger
+from os import chdir, mkdir
+from os.path import exists, expanduser
+from pathlib import Path
+from shlex import split
+from subprocess import check_output
+from sys import exit
+
+from requests import get
+
+HOME_PATH = expanduser("~/")
+CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
+REPOS_TO_SKIP = (
+ "abra",
+ "abra-apps",
+ "abra-gandi",
+ "abra-hetzner",
+ "backup-bot",
+ "coopcloud.tech",
+ "coturn",
+ "docker-cp-deploy",
+ "docker-dind-bats-kcov",
+ "docs.coopcloud.tech",
+ "example",
+ "gardening",
+ "organising",
+ "pyabra",
+ "radicle-seed-node",
+ "stack-ssh-deploy",
+ "swarm-cronjob",
+)
+YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq")
+SCRIPT_PATH = Path(__file__).absolute().parent
+
+log = getLogger(__name__)
+basicConfig()
+log.setLevel(DEBUG)
+
+
+def _run_cmd(cmd, shell=False, **kwargs):
+ """Run a shell command."""
+ args = [split(cmd)]
+
+ if shell:
+ args = [cmd]
+ kwargs = {"shell": shell}
+
+ try:
+ return check_output(*args, **kwargs).decode("utf-8").strip()
+ except Exception as exception:
+ log.error(f"Failed to run {cmd}, saw {str(exception)}")
+ exit(1)
+
+
+def get_repos_json():
+ """ Retrieve repo list from Gitea """
+
+ url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
+
+ log.info(f"Retrieving {url}")
+
+ repos = []
+ response = True
+ page = 1
+
+ try:
+ while response:
+ log.info(f"Trying to fetch page {page}")
+ response = get(url + f"?page={page}", timeout=10).json()
+ repos.extend(response)
+ page += 1
+
+ return repos
+ except Exception as exception:
+ log.error(f"Failed to retrieve {url}, saw {str(exception)}")
+ exit(1)
+
+
+def clone_all_apps(repos_json):
+ """Clone all Co-op Cloud apps to ~/.abra/apps."""
+ if not exists(CLONES_PATH):
+ mkdir(CLONES_PATH)
+
+ repos = [[p["name"], p["ssh_url"]] for p in repos_json]
+
+ for name, url in repos:
+ if name in REPOS_TO_SKIP:
+ continue
+
+ if not exists(f"{CLONES_PATH}/{name}"):
+ log.info(f"Retrieving {url}")
+ _run_cmd(f"git clone {url} {CLONES_PATH}/{name}")
+
+ chdir(f"{CLONES_PATH}/{name}")
+ if not int(_run_cmd("git branch --list | wc -l", shell=True)):
+ log.info(f"Guessing main branch is HEAD for {name}")
+ _run_cmd("git checkout main")
+ else:
+ log.info(f"Updating {name}")
+ chdir(f"{CLONES_PATH}/{name}")
+ _run_cmd("git fetch -a")
diff --git a/bin/app-json.py b/bin/app-json.py
index eb75dab5058..ba056ac5727 100755
--- a/bin/app-json.py
+++ b/bin/app-json.py
@@ -7,84 +7,24 @@
# https://apps.coopcloud.tech
from json import dump
-from logging import DEBUG, basicConfig, getLogger
-from os import chdir, listdir, mkdir
-from os.path import basename, exists, expanduser
-from pathlib import Path
+from os import chdir, listdir
+from os.path import basename
from re import findall, search
-from shlex import split
-from subprocess import DEVNULL, check_output
-from sys import exit
+from subprocess import DEVNULL
from requests import get
-HOME_PATH = expanduser("~/")
-CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
-YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq")
-SCRIPT_PATH = Path(__file__).absolute().parent
-REPOS_TO_SKIP = (
- "abra",
- "abra-apps",
- "abra-gandi",
- "abra-hetzner",
- "backup-bot",
- "coopcloud.tech",
- "coturn",
- "docker-cp-deploy",
- "docker-dind-bats-kcov",
- "docs.coopcloud.tech",
- "example",
- "gardening",
- "organising",
- "pyabra",
- "radicle-seed-node",
- "stack-ssh-deploy",
- "swarm-cronjob",
+from abralib import (
+ CLONES_PATH,
+ REPOS_TO_SKIP,
+ SCRIPT_PATH,
+ YQ_PATH,
+ _run_cmd,
+ clone_all_apps,
+ get_repos_json,
+ log,
)
-log = getLogger(__name__)
-basicConfig()
-log.setLevel(DEBUG)
-
-
-def _run_cmd(cmd, shell=False, **kwargs):
- """Run a shell command."""
- args = [split(cmd)]
-
- if shell:
- args = [cmd]
- kwargs = {"shell": shell}
-
- try:
- return check_output(*args, **kwargs).decode("utf-8").strip()
- except Exception as exception:
- log.error(f"Failed to run {cmd}, saw {str(exception)}")
- exit(1)
-
-
-def get_repos_json():
- """ Retrieve repo list from Gitea """
-
- url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
-
- log.info(f"Retrieving {url}")
-
- repos = []
- response = True
- page = 1
-
- try:
- while response:
- log.info(f"Trying to fetch page {page}")
- response = get(url + f"?page={page}", timeout=10).json()
- repos.extend(response)
- page += 1
-
- return repos
- except Exception as exception:
- log.error(f"Failed to retrieve {url}, saw {str(exception)}")
- exit(1)
-
def get_published_apps_json():
"""Retrieve already published apps json."""
@@ -99,31 +39,6 @@ def get_published_apps_json():
return {}
-def clone_all_apps(repos_json):
- """Clone all Co-op Cloud apps to ~/.abra/apps."""
- if not exists(CLONES_PATH):
- mkdir(CLONES_PATH)
-
- repos = [[p["name"], p["ssh_url"]] for p in repos_json]
-
- for name, url in repos:
- if name in REPOS_TO_SKIP:
- continue
-
- if not exists(f"{CLONES_PATH}/{name}"):
- log.info(f"Retrieving {url}")
- _run_cmd(f"git clone {url} {CLONES_PATH}/{name}")
-
- chdir(f"{CLONES_PATH}/{name}")
- if not int(_run_cmd("git branch --list | wc -l", shell=True)):
- log.info(f"Guessing main branch is HEAD for {name}")
- _run_cmd("git checkout main")
- else:
- log.info(f"Updating {name}")
- chdir(f"{CLONES_PATH}/{name}")
- _run_cmd("git fetch -a")
-
-
def generate_apps_json(repos_json):
"""Generate the abra-apps.json application versions file."""
apps_json = {}
diff --git a/bin/clone-all-apps.py b/bin/clone-all-apps.py
new file mode 100755
index 00000000000..b9ec1f8ca4f
--- /dev/null
+++ b/bin/clone-all-apps.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+# Usage: ./clone-all-apps.py
+#
+# Clone all available apps into ~/.abra/apps using ssh:// URLs
+
+from abralib import clone_all_apps, get_repos_json
+
+
+def main():
+ """Run the script."""
+ repos_json = get_repos_json()
+ clone_all_apps(repos_json)
+
+
+main()
diff --git a/bin/renovate-ls-apps.py b/bin/renovate-ls-apps.py
index 19839cb84e7..1b602acb08a 100755
--- a/bin/renovate-ls-apps.py
+++ b/bin/renovate-ls-apps.py
@@ -4,77 +4,7 @@
#
# Output list of apps for Renovate bot configuration
-from logging import DEBUG, basicConfig, getLogger
-from os.path import expanduser
-from pathlib import Path
-from shlex import split
-from subprocess import check_output
-from sys import exit
-
-from requests import get
-
-HOME_PATH = expanduser("~/")
-CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
-YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq")
-SCRIPT_PATH = Path(__file__).absolute().parent
-REPOS_TO_SKIP = (
- "abra",
- "abra-apps",
- "abra-gandi",
- "abra-hetzner",
- "backup-bot",
- "coopcloud.tech",
- "coturn",
- "docker-cp-deploy",
- "docker-dind-bats-kcov",
- "docs.coopcloud.tech",
- "example",
- "gardening",
- "organising",
- "pyabra",
- "radicle-seed-node",
- "stack-ssh-deploy",
- "swarm-cronjob",
-)
-
-log = getLogger(__name__)
-basicConfig()
-log.setLevel(DEBUG)
-
-
-def _run_cmd(cmd, shell=False, **kwargs):
- """Run a shell command."""
- args = [split(cmd)]
-
- if shell:
- args = [cmd]
- kwargs = {"shell": shell}
-
- try:
- return check_output(*args, **kwargs).decode("utf-8").strip()
- except Exception as exception:
- log.error(f"Failed to run {cmd}, saw {str(exception)}")
- exit(1)
-
-
-def get_repos_json():
- """ Retrieve repo list from Gitea """
- url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
-
- repos = []
- response = True
- page = 1
-
- try:
- while response:
- response = get(url + f"?page={page}", timeout=10).json()
- repos.extend(response)
- page += 1
-
- return repos
- except Exception as exception:
- log.error(f"Failed to retrieve {url}, saw {str(exception)}")
- exit(1)
+from abralib import REPOS_TO_SKIP, get_repos_json
def main():