diff --git a/README.md b/README.md index 04c99f1..b70af17 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,102 @@ -# peach-vps config +# peach-vps -Scripts for configuring the peachcloud vps for various hosting and automation. +![Generic badge](https://img.shields.io/badge/version-0.2.1-.svg) + +Scripts for configuring the PeachCloud VPS for various hosting and automation functions. Currently: -- debian repository of microservices + - Debian repository of microservices (using [Freight](https://github.com/freight-team/freight)) + +## Setup Debian Repo + +`scripts/setup_debian_repo.py` + +An idempotent script for initializing the Debian repo on the VPS. + +The script currently performs the following actions: + + - Installs system requirements + - Creates directories for microservices and package archive + - Installs Rust + - Installs `cargo deb` + - Installs Rust aarch64 toolchain for cross-compilation + - Installs Freight for package archive creation and management + - Configures Freight + - Pulls microservices code from GitHub repos + - Exports the public GPG key + - Configures nginx + - Builds and updates microservice packages + - Adds packages to Freight library + - Adds packages to Freight cache + +Prior to executing the script for the first time, run the following commands on the target system: -# setup debian repo -an idempotent script for initializing the debian repo on the vps ``` -apt update -apt install git python python3-pip rsync +sudo apt update +sudo apt install git python python3-pip rsync git clone https://github.com/peachcloud/peach-vps.git cd peach-vps pip3 install -r requirements.txt -python3 scripts/setup_debian_repo.py -i ``` +Open `scripts/setup_debian_repo.py` and set the following constants: + + - USER_PATH + - GPG_KEY_EMAIL + - GPG_KEY_PASS_FILE + +Then execute the script with the `-i` flag to run the full system initialization process (_note: several commands executed by the script require `sudo` permissions. You will be prompted for the user password during the execution of the scipt._): + +``` +python3 -u scripts/setup_debian_repo.py -i +``` + +## Update Debian Repo + +Without the -i flag, the `setup_debian_repo.py` script rebuilds all +microservices (cross-compiled to arm64) and updates the Debian repo -# update debian repo -without the -i flag, the setup_debian_repo rebuilds all -microservices (cross-compiled to arm64) and re-adds them to the debian repo ``` cd peach-vps -python3 scripts/setup_debian_repo.py +python3 -u scripts/setup_debian_repo.py ``` +## Install from Debian Repo + +To add the PeachCloud Debian repo as an apt source, run the following commands from your Pi: -# using the debian repo on the pi -To add the peachcloud debian repo as an apt source, -on the pi, ``` vi /etc/apt/sources.list.d/peach.list ``` -and add the following line: + +Append the following line: + ``` deb http://apt.peachcloud.org/debian/ buster main ``` -Then add the gpg pub key to the apt-key list: +Add the gpg pub key to the apt-key list: + ``` wget -O - http://apt.peachcloud.org/peach_pub.gpg | sudo apt-key add - ``` -You can then install peach packages with apt-get: +You can then install peach packages with apt: + ``` -apt-get update -apt-get install peach-oled -``` \ No newline at end of file +sudo apt update +sudo apt install peach-oled +``` + +By default, the latest version of the package will be downloaded and installed. + +Specific versions of packages can be selected for installation by supplying the semantic versioning number (this is useful for downgrading): + +``` +sudo apt install peach-network=0.2.0 +``` + +## Licensing + +AGPL-3.0 diff --git a/conf/templates/debian_repo/distributions b/conf/templates/debian_repo/distributions deleted file mode 100644 index 45d7c64..0000000 --- a/conf/templates/debian_repo/distributions +++ /dev/null @@ -1,8 +0,0 @@ -Origin: PeachCloud -Label: PeachCloud -Codename: buster -Architectures: amd64 arm64 -Components: main -Description: Apt repository for PeachCloud debian packages -SignWith: {{gpg_key_id}} -DebOverride: override.buster diff --git a/conf/templates/debian_repo/options b/conf/templates/debian_repo/options deleted file mode 100644 index 5dd7345..0000000 --- a/conf/templates/debian_repo/options +++ /dev/null @@ -1,3 +0,0 @@ -verbose -basedir {{debian_rep_dir}} -ask-passphrase diff --git a/conf/templates/debian_repo/override.buster b/conf/templates/debian_repo/override.buster deleted file mode 100644 index ec9fb2d..0000000 --- a/conf/templates/debian_repo/override.buster +++ /dev/null @@ -1,4 +0,0 @@ -{% for service in services %} -{{service}} Priority optional -{{service}} Section net -{% endfor %} \ No newline at end of file diff --git a/conf/templates/freight.conf b/conf/templates/freight.conf new file mode 100644 index 0000000..bb9c926 --- /dev/null +++ b/conf/templates/freight.conf @@ -0,0 +1,26 @@ +# Freight configuration. + +# Directories for the Freight library and Freight cache. Your web +# server's document root should be `$VARCACHE`. +VARLIB="{{freight_lib_path}}" +VARCACHE="{{freight_cache_path}}" + +# Default `Origin` and `Label` fields for `Release` files. +ORIGIN="apt.peachcloud.org" +LABEL="PeachCloud" + +# Architectures supported by the package repository. +ARCHS="arm64" + +# Cache the control files after each run (on), or regenerate them every +# time (off). +CACHE="off" + +# GPG key to use to sign repositories. This is required by the `apt` +# repository provider. Use `gpg --gen-key` (see `gpg`(1) for more +# details) to generate a key and put its email address here. +GPG="{{gpg_key_email}}" + +# Whether to follow symbolic links in `$VARLIB` to produce extra components +# in the cache directory (on) or not (off). +SYMLINKS="off" diff --git a/conf/templates/debian_repo/nginx_debian.conf b/conf/templates/nginx_debian.conf similarity index 100% rename from conf/templates/debian_repo/nginx_debian.conf rename to conf/templates/nginx_debian.conf diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 7644887..11ddd4b 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + from utils import render_template import subprocess @@ -5,57 +7,109 @@ import os import argparse -# constants -MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" -WEB_DIR = "/var/www/" -APT_DIR = "/var/www/apt.peachcloud.org" -DEBIAN_REPO_DIR = "/var/www/apt.peachcloud.org/debian" -DEBIAN_REPO_CONF_DIR = "/var/www/apt.peachcloud.org/debian/conf" +# before running this script run `gpg --gen-key` on the server +# assign the email address of the key id here: +GPG_KEY_EMAIL = "andrew@mycelial.technology" +# save the key passphrase to file and assign the path here: +# (ensure the file is only readable by the user running freight) +GPG_KEY_PASS_FILE = "/home/rust/passphrase.txt" +# if you need to list the existing keys: `gpg --list-keys` + + +# constants +AUTOMATION_DIR = "/srv/peachcloud/automation" +FREIGHT_CONF = "/etc/freight.conf" +FREIGHT_LIB = "/var/lib/freight" +FREIGHT_CACHE = "/var/www/apt.peachcloud.org" +MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" +MICROSERVICES_DEB_DIR = "/srv/peachcloud/debs" +USER_PATH = "/home/rust" -# before running this script run `gpg --gen-key` on the server, and put the key id here -# `gpg --list-keys` -GPG_KEY_ID = "4ACEF251EA3E091167E8F03EBF69A52BE3565476" SERVICES = [ + {"name": "peach-buttons", + "repo_url": "https://github.com/peachcloud/peach-buttons.git"}, + {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, + {"name": "peach-monitor", + "repo_url": "https://github.com/peachcloud/peach-monitor.git"}, + {"name": "peach-network", + "repo_url": "https://github.com/peachcloud/peach-network.git"}, {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, - {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"}, {"name": "peach-stats", "repo_url": "https://github.com/peachcloud/peach-stats.git"}, # {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear - {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, - {"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"} ] # parse CLI args parser = argparse.ArgumentParser() -parser.add_argument("-i", "--initialize", help="initialize and update debian repo", action="store_true") +parser.add_argument( + "-i", + "--initialize", + help="initialize and update debian repo", + action="store_true") args = parser.parse_args() +cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") + # initializing debian repo from a blank slate # (but this code is idempotent so it can be re-run if already initialized) if args.initialize: print("[ INSTALLING SYSTEM REQUIREMENTS ]") - subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "reprepro", "gcc-aarch64-linux-gnu", ]) + subprocess.call(["sudo", + "apt-get", + "install", + "git", + "nginx", + "curl", + "build-essential", + "gcc-aarch64-linux-gnu", + ]) print("[ CREATING DIRECTORIES ]") - folders = [MICROSERVICES_SRC_DIR, WEB_DIR, APT_DIR, DEBIAN_REPO_DIR, DEBIAN_REPO_CONF_DIR] + folders = [MICROSERVICES_SRC_DIR, FREIGHT_CACHE, FREIGHT_LIB] for folder in folders: if not os.path.exists(folder): os.makedirs(folder) print("[ INSTALLING RUST ]") - if not os.path.exists("/root/.cargo/bin/rustc"): - first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) - output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) + rustc_path = os.path.join(USER_PATH, ".cargo/bin/rustc") + if not os.path.exists(rustc_path): + first_command = subprocess.Popen( + ["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) + output = subprocess.check_output( + ["sh", "-s", "--", "-y"], stdin=first_command.stdout) first_command.wait() print("[ INSTALLING CARGO-DEB ]") - if not os.path.exists("/root/.cargo/bin/cargo-deb"): - subprocess.call(["/root/.cargo/bin/cargo", "install", "cargo-deb"]) + cargo_deb_path = os.path.join(USER_PATH, ".cargo/bin/cargo-deb") + if not os.path.exists(cargo_deb_path): + subprocess.call([cargo_path, "install", "cargo-deb"]) print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") - subprocess.call(["/root/.cargo/bin/rustup", "target", "add", "aarch64-unknown-linux-gnu"]) - subprocess.call(["/root/.cargo/bin/rustup", "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) + rustup_path = os.path.join(USER_PATH, ".cargo/bin/rustup") + subprocess.call([rustup_path, "target", "add", + "aarch64-unknown-linux-gnu"]) + subprocess.call([rustup_path, "toolchain", "install", + "nightly-aarch64-unknown-linux-gnu"]) + + print("[ INSTALLING FREIGHT ]") + freight_path = os.path.join(AUTOMATION_DIR, "freight") + if not os.path.exists(freight_path): + subprocess.call( + ["git", "clone", "https://github.com/freight-team/freight.git", freight_path]) + + print("[ CONFIGURING FREIGHT ]") + freight_conf_tmp_path = os.path.join(USER_PATH, "freight.conf") + render_template( + src="freight.conf", + dest=freight_conf_tmp_path, + template_vars={ + "freight_lib_path": FREIGHT_LIB, + "freight_cache_path": FREIGHT_CACHE, + "gpg_key_email": GPG_KEY_EMAIL + } + ) + subprocess.call(["sudo", "cp", freight_conf_tmp_path, FREIGHT_CONF]) print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") for service in SERVICES: @@ -65,59 +119,51 @@ if args.initialize: if not os.path.exists(service_path): subprocess.call(["git", "clone", repo_url, service_path]) - print("[ COPYING DEBIAN REPO CONFIG ]") - render_template( - src="debian_repo/distributions", - dest="{}/distributions".format(DEBIAN_REPO_CONF_DIR), - template_vars={ - "gpg_key_id": GPG_KEY_ID - } - ) - render_template( - src="debian_repo/options", - dest="{}/options".format(DEBIAN_REPO_CONF_DIR), - template_vars={ - "debian_rep_dir": DEBIAN_REPO_DIR - } - ) - render_template( - src="debian_repo/override.buster", - dest="{}/override.buster".format(DEBIAN_REPO_CONF_DIR), - template_vars={ - "services": [service["name"] for service in SERVICES] - } - ) - print("[ EXPORTING PUBLIC GPG KEY ]") - output_path = "{}/peach_pub.gpg".format(APT_DIR) + output_path = "{}/peach_pub.gpg".format(FREIGHT_CACHE) if not os.path.exists(output_path): - subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID]) + subprocess.call(["gpg", "--armor", "--output", + output_path, "--export", GPG_KEY_EMAIL]) print("[ COPYING NGINX CONFIG ]") + nginx_conf_tmp_path = os.path.join(USER_PATH, "apt.peachcloud.org") render_template( - src="debian_repo/nginx_debian.conf", - dest="/etc/nginx/sites-enabled/apt.peachcloud.org", - template_vars = { - "apt_dir": APT_DIR + src="nginx_debian.conf", + dest=nginx_conf_tmp_path, + template_vars={ + "apt_dir": FREIGHT_CACHE } ) + subprocess.call(["sudo", "cp", nginx_conf_tmp_path, + "/etc/nginx/sites-enabled/apt.peachcloud.org"]) - -# below is code for git updating the microservices, building the microservices, -# and (re)-adding them to the debian repo +# update the microservices from git and build the debian packages print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") for service in SERVICES: service_name = service["name"] service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) print("[ BUILIDING SERVICE {} ]".format(service_name)) subprocess.call(["git", "pull"], cwd=service_path) - debian_package_path = subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip() - # remove debian package from repo - # (in the future we could look at some way of updating with versions instead of removing and adding) - subprocess.call(["reprepro", "remove", "buster", service_name], cwd=DEBIAN_REPO_DIR) - # add the package - subprocess.call(["reprepro", "includedeb", "buster", debian_package_path], cwd=DEBIAN_REPO_DIR) + debian_package_path = subprocess.run( + [ + cargo_path, + "deb", + "--target", + "aarch64-unknown-linux-gnu"], + cwd=service_path, + stdout=subprocess.PIPE).stdout.decode("utf-8").strip() + subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) +print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") +for package in os.scandir(MICROSERVICES_DEB_DIR): + if package.name.endswith(".deb"): + print("[ ADDING PACKAGE {} ]".format(package.name)) + subprocess.call(["freight", "add", "-c", FREIGHT_CONF, + package.path, "apt/buster"]) + +print("[ ADDING PACKAGES TO FREIGHT CACHE ]") +# needs to be run as sudo user +subprocess.call(["sudo", "freight", "cache", "-g", + GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) print("[ DEBIAN REPO SETUP COMPLETE ]") - diff --git a/scripts/utils.py b/scripts/utils.py index 6d208a1..69f2bdf 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -25,5 +25,3 @@ def render_template(src, dest, template_vars=None): os.remove(dest) with open(dest, 'w') as f: f.write(output_text) - -