From ce55310c65225cdb29c37ed6477bca344c255ac1 Mon Sep 17 00:00:00 2001 From: Max Fowler Date: Wed, 11 Nov 2020 11:21:52 +0100 Subject: [PATCH] Pythong script for initializing and updating peachcloud debian repo of microservices. --- conf/templates/debian_repo/distributions | 8 ++ conf/templates/debian_repo/nginx_debian.conf | 21 ++++ conf/templates/debian_repo/options | 3 + conf/templates/debian_repo/override.buster | 4 + scripts/setup_debian_repo.py | 105 +++++++++++++++++++ scripts/setup_vps.py | 3 +- scripts/utils.py | 11 +- 7 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 conf/templates/debian_repo/distributions create mode 100644 conf/templates/debian_repo/nginx_debian.conf create mode 100644 conf/templates/debian_repo/options create mode 100644 conf/templates/debian_repo/override.buster create mode 100644 scripts/setup_debian_repo.py diff --git a/conf/templates/debian_repo/distributions b/conf/templates/debian_repo/distributions new file mode 100644 index 0000000..f6fed0f --- /dev/null +++ b/conf/templates/debian_repo/distributions @@ -0,0 +1,8 @@ +Origin: PeachCloud +Label: PeachCloud +Codename: buster +Architectures: amd64 +Components: main +Description: Apt repository for PeachCloud debian packages +SignWith: {{gpg_key_id}} +DebOverride: override.buster diff --git a/conf/templates/debian_repo/nginx_debian.conf b/conf/templates/debian_repo/nginx_debian.conf new file mode 100644 index 0000000..73b671e --- /dev/null +++ b/conf/templates/debian_repo/nginx_debian.conf @@ -0,0 +1,21 @@ +server { + listen 80; + server_name 167.99.136.83; + + access_log /var/log/nginx-debian.log; + error_log /var/log/nginx-debian.error; + + location / { + root {{apt_dir}}; + index index.html; + autoindex on; + } + + location ~ /(.*)/conf { + deny all; + } + + location ~ /(.*)/db { + deny all; + } +} \ No newline at end of file diff --git a/conf/templates/debian_repo/options b/conf/templates/debian_repo/options new file mode 100644 index 0000000..5dd7345 --- /dev/null +++ b/conf/templates/debian_repo/options @@ -0,0 +1,3 @@ +verbose +basedir {{debian_rep_dir}} +ask-passphrase diff --git a/conf/templates/debian_repo/override.buster b/conf/templates/debian_repo/override.buster new file mode 100644 index 0000000..ec9fb2d --- /dev/null +++ b/conf/templates/debian_repo/override.buster @@ -0,0 +1,4 @@ +{% for service in services %} +{{service}} Priority optional +{{service}} Section net +{% endfor %} \ No newline at end of file diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py new file mode 100644 index 0000000..71740b1 --- /dev/null +++ b/scripts/setup_debian_repo.py @@ -0,0 +1,105 @@ +from utils import render_template + +import subprocess +import os + + +INITIALIZE_DEBIAN_REPO = True + +MICROSERVICES_SRC_DIR = "/srv/peachcloud/src" +WEB_DIR = "/var/www/" +APT_DIR = "/var/www/repos/apt" +DEBIAN_REPO_DIR = "/var/www/repos/apt/debian" +DEBIAN_REPO_CONF_DIR = "/var/www/repos/apt/debian/conf" + +# before running this script run `gpg --gen-key` on the server, and put the key id here +GPG_KEY_ID = "E62CD13A85763FCEC3EDBA8EA98440817F1A3CE5", + +SERVICES = [ + {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, + {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"} +] + +if INITIALIZE_DEBIAN_REPO: + + print("[ INSTALLING SYSTEM REQUIREMENTS ]") + subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "reprepro", "gcc-aarch64-linux-gnu", ]) + + print("[ CREATING DIRECTORIES ]") + folders = [MICROSERVICES_SRC_DIR, WEB_DIR, APT_DIR, DEBIAN_REPO_DIR, DEBIAN_REPO_CONF_DIR] + for folder in folders: + if not os.path.exists(folder): + os.makedirs(folder) + + print("[ INSTALLING RUST ]") + if not os.path.exists("/root/.cargo/bin/rustc"): + first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) + output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) + first_command.wait() + + print("[ INSTALLING CARGO-DEB ]") + subprocess.call(['cargo', 'install', 'cargo-deb']) + + print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") + subprocess.call(['rustup', 'toolchain', 'install', 'nightly-aarch64-unknown-linux-gnu']) + + print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") + for service in SERVICES: + name = service["name"] + repo_url = service["repo_url"] + service_path = os.path.join(MICROSERVICES_SRC_DIR, name) + if not os.path.exists(service_path): + subprocess.call(["git", "clone", repo_url, service_path]) + + print("[ COPYING DEBIAN REPO CONFIG ]") + render_template( + src="debian_repo/distributions", + dest="{}/distributions".format(DEBIAN_REPO_CONF_DIR), + template_vars={ + "gpg_key_id": GPG_KEY_ID + } + ) + render_template( + src="debian_repo/options", + dest="{}/options".format(DEBIAN_REPO_CONF_DIR), + template_vars={ + "debian_rep_dir": DEBIAN_REPO_DIR + } + ) + render_template( + src="debian_repo/override.buster", + dest="{}/override.buster".format(DEBIAN_REPO_CONF_DIR), + template_vars={ + "services": [service["name"] for service in SERVICES] + } + ) + + print("[ EXPORTING PUBLIC GPG KEY ]") + output_path = "{}/peach_pub.gpg".format(APT_DIR) + subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID]) + + print("[ COPYING NGINX CONFIG ]") + render_template( + src="debian_repo/nginx_debian.conf", + dest="/etc/nginx/sites-enabled/deb.peachcloud.org", + template_vars = { + "apt_dir": APT_DIR + } + ) + + +# below is code for updating the microservices, building the microservices, +# and adding them to the debian repo +for service in SERVICES: + service_name = service['name'] + service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) + print("[ BUILIDING SERVICE {} ]".format(service_name)) + subprocess.call("cd {} && git pull;".format(service_path)) + subprocess.call("cd {} && cargo deb --target aarch64-unknown-linux-gnu;".format(service_path)) + deb_path = '?' + subprocess.call("cd {debian_dir} && reprepro includedeb buster {deb_path}".format( + debian_dir=DEBIAN_REPO_DIR, + deb_path=deb_path + )) + + diff --git a/scripts/setup_vps.py b/scripts/setup_vps.py index 3f85fe8..24aa5e8 100644 --- a/scripts/setup_vps.py +++ b/scripts/setup_vps.py @@ -42,7 +42,7 @@ for folder in folders: os.makedirs(folder) print("[ INSTALLING RUST ]") -if os.path.exists('/root/.cargo/bin/rustc'): +if not os.path.exists('/root/.cargo/bin/rustc'): first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) first_command.wait() @@ -50,6 +50,7 @@ if os.path.exists('/root/.cargo/bin/rustc'): print("[ INSTALLING CARGO PACKAGES ]") cargo_install("cargo-deb") +print("[ COPYING NGINX CONFIG ]") render_template(src='nginx/nginx.conf', dest='/etc/nginx/nginx.conf') diff --git a/scripts/utils.py b/scripts/utils.py index a9508cd..c6362ab 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -1,5 +1,3 @@ -from vars import VARS - import os import jinja2 import subprocess @@ -20,10 +18,8 @@ def render_template(src, dest, template_vars=None): :return: None """ template = template_env.get_template(src) - if template_vars: - template_vars.update(VARS) - else: - template_vars = VARS + if not template_vars: + template_vars= {} output_text = template.render(template_vars=template_vars) if os.path.exists(dest): os.remove(dest) @@ -31,6 +27,3 @@ def render_template(src, dest, template_vars=None): f.write(output_text) -def cargo_install(package): - subprocess.call(['/root/.cargo/bin/cargo', 'install', package]) -