diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fbe14c3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +deploy.sh +secret* +secret_files* +ssh.sh +notes.txt \ No newline at end of file diff --git a/README.md b/README.md index 7828353..04c99f1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,50 @@ -# peach-vps -# simple-ansible-template +# peach-vps config + +Scripts for configuring the peachcloud vps for various hosting and automation. + +Currently: +- debian repository of microservices + + +# setup debian repo +an idempotent script for initializing the debian repo on the vps +``` +apt update +apt install git python python3-pip rsync +git clone https://github.com/peachcloud/peach-vps.git +cd peach-vps +pip3 install -r requirements.txt +python3 scripts/setup_debian_repo.py -i +``` + + +# update debian repo +without the -i flag, the setup_debian_repo rebuilds all +microservices (cross-compiled to arm64) and re-adds them to the debian repo +``` +cd peach-vps +python3 scripts/setup_debian_repo.py +``` + + +# using the debian repo on the pi +To add the peachcloud debian repo as an apt source, +on the pi, +``` +vi /etc/apt/sources.list.d/peach.list +``` +and add the following line: +``` +deb http://apt.peachcloud.org/debian/ buster main +``` + +Then add the gpg pub key to the apt-key list: +``` +wget -O - http://apt.peachcloud.org/peach_pub.gpg | sudo apt-key add - +``` + +You can then install peach packages with apt-get: +``` +apt-get update +apt-get install peach-oled +``` \ No newline at end of file diff --git a/conf/templates/debian_repo/distributions b/conf/templates/debian_repo/distributions new file mode 100644 index 0000000..45d7c64 --- /dev/null +++ b/conf/templates/debian_repo/distributions @@ -0,0 +1,8 @@ +Origin: PeachCloud +Label: PeachCloud +Codename: buster +Architectures: amd64 arm64 +Components: main +Description: Apt repository for PeachCloud debian packages +SignWith: {{gpg_key_id}} +DebOverride: override.buster diff --git a/conf/templates/debian_repo/nginx_debian.conf b/conf/templates/debian_repo/nginx_debian.conf new file mode 100644 index 0000000..fce5521 --- /dev/null +++ b/conf/templates/debian_repo/nginx_debian.conf @@ -0,0 +1,21 @@ +server { + listen 80; + server_name apt.peachcloud.org; + + access_log /var/log/nginx-debian.log; + error_log /var/log/nginx-debian.error; + + location / { + root {{apt_dir}}; + index index.html; + autoindex on; + } + + location ~ /(.*)/conf { + deny all; + } + + location ~ /(.*)/db { + deny all; + } +} \ No newline at end of file diff --git a/conf/templates/debian_repo/options b/conf/templates/debian_repo/options new file mode 100644 index 0000000..5dd7345 --- /dev/null +++ b/conf/templates/debian_repo/options @@ -0,0 +1,3 @@ +verbose +basedir {{debian_rep_dir}} +ask-passphrase diff --git a/conf/templates/debian_repo/override.buster b/conf/templates/debian_repo/override.buster new file mode 100644 index 0000000..ec9fb2d --- /dev/null +++ b/conf/templates/debian_repo/override.buster @@ -0,0 +1,4 @@ +{% for service in services %} +{{service}} Priority optional +{{service}} Section net +{% endfor %} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..758129b --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Jinja2==2.11.2 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py new file mode 100644 index 0000000..7644887 --- /dev/null +++ b/scripts/setup_debian_repo.py @@ -0,0 +1,123 @@ +from utils import render_template + +import subprocess +import os +import argparse + + +# constants +MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" +WEB_DIR = "/var/www/" +APT_DIR = "/var/www/apt.peachcloud.org" +DEBIAN_REPO_DIR = "/var/www/apt.peachcloud.org/debian" +DEBIAN_REPO_CONF_DIR = "/var/www/apt.peachcloud.org/debian/conf" + +# before running this script run `gpg --gen-key` on the server, and put the key id here +# `gpg --list-keys` +GPG_KEY_ID = "4ACEF251EA3E091167E8F03EBF69A52BE3565476" + +SERVICES = [ + {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, + {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"}, + {"name": "peach-stats", "repo_url": "https://github.com/peachcloud/peach-stats.git"}, + # {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear + {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, + {"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"} +] + +# parse CLI args +parser = argparse.ArgumentParser() +parser.add_argument("-i", "--initialize", help="initialize and update debian repo", action="store_true") +args = parser.parse_args() + +# initializing debian repo from a blank slate +# (but this code is idempotent so it can be re-run if already initialized) +if args.initialize: + + print("[ INSTALLING SYSTEM REQUIREMENTS ]") + subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "reprepro", "gcc-aarch64-linux-gnu", ]) + + print("[ CREATING DIRECTORIES ]") + folders = [MICROSERVICES_SRC_DIR, WEB_DIR, APT_DIR, DEBIAN_REPO_DIR, DEBIAN_REPO_CONF_DIR] + for folder in folders: + if not os.path.exists(folder): + os.makedirs(folder) + + print("[ INSTALLING RUST ]") + if not os.path.exists("/root/.cargo/bin/rustc"): + first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) + output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) + first_command.wait() + + print("[ INSTALLING CARGO-DEB ]") + if not os.path.exists("/root/.cargo/bin/cargo-deb"): + subprocess.call(["/root/.cargo/bin/cargo", "install", "cargo-deb"]) + + print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") + subprocess.call(["/root/.cargo/bin/rustup", "target", "add", "aarch64-unknown-linux-gnu"]) + subprocess.call(["/root/.cargo/bin/rustup", "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) + + print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") + for service in SERVICES: + name = service["name"] + repo_url = service["repo_url"] + service_path = os.path.join(MICROSERVICES_SRC_DIR, name) + if not os.path.exists(service_path): + subprocess.call(["git", "clone", repo_url, service_path]) + + print("[ COPYING DEBIAN REPO CONFIG ]") + render_template( + src="debian_repo/distributions", + dest="{}/distributions".format(DEBIAN_REPO_CONF_DIR), + template_vars={ + "gpg_key_id": GPG_KEY_ID + } + ) + render_template( + src="debian_repo/options", + dest="{}/options".format(DEBIAN_REPO_CONF_DIR), + template_vars={ + "debian_rep_dir": DEBIAN_REPO_DIR + } + ) + render_template( + src="debian_repo/override.buster", + dest="{}/override.buster".format(DEBIAN_REPO_CONF_DIR), + template_vars={ + "services": [service["name"] for service in SERVICES] + } + ) + + print("[ EXPORTING PUBLIC GPG KEY ]") + output_path = "{}/peach_pub.gpg".format(APT_DIR) + if not os.path.exists(output_path): + subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID]) + + print("[ COPYING NGINX CONFIG ]") + render_template( + src="debian_repo/nginx_debian.conf", + dest="/etc/nginx/sites-enabled/apt.peachcloud.org", + template_vars = { + "apt_dir": APT_DIR + } + ) + + +# below is code for git updating the microservices, building the microservices, +# and (re)-adding them to the debian repo +print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") +for service in SERVICES: + service_name = service["name"] + service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) + print("[ BUILIDING SERVICE {} ]".format(service_name)) + subprocess.call(["git", "pull"], cwd=service_path) + debian_package_path = subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip() + # remove debian package from repo + # (in the future we could look at some way of updating with versions instead of removing and adding) + subprocess.call(["reprepro", "remove", "buster", service_name], cwd=DEBIAN_REPO_DIR) + # add the package + subprocess.call(["reprepro", "includedeb", "buster", debian_package_path], cwd=DEBIAN_REPO_DIR) + + +print("[ DEBIAN REPO SETUP COMPLETE ]") + diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 0000000..6d208a1 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,29 @@ +import os +import jinja2 +import subprocess + +PROJECT_PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +print('PROJECT_PATH: {}'.format(PROJECT_PATH)) + +template_path = os.path.join(PROJECT_PATH, 'conf/templates') +template_loader = jinja2.FileSystemLoader(searchpath=template_path) +template_env = jinja2.Environment(loader=template_loader, keep_trailing_newline=True) + + +def render_template(src, dest, template_vars=None): + """ + :param src: relative string path to jinja template file + :param dest: absolute string path of output destination file + :param template_vars: variables to render template with + :return: None + """ + template = template_env.get_template(src) + if not template_vars: + template_vars= {} + output_text = template.render(**template_vars) + if os.path.exists(dest): + os.remove(dest) + with open(dest, 'w') as f: + f.write(output_text) + +