From 4fc6d8ae8efe78d0531b275370abeb7fb591faef Mon Sep 17 00:00:00 2001 From: mycognosist Date: Mon, 30 Nov 2020 13:39:04 +0000 Subject: [PATCH 01/22] Update script to use Freight instead of reprepro --- scripts/setup_debian_repo.py | 112 ++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 7644887..658c423 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -6,23 +6,31 @@ import argparse # constants +AUTOMATION_DIR = "/srv/peachcloud/automation" MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" -WEB_DIR = "/var/www/" -APT_DIR = "/var/www/apt.peachcloud.org" -DEBIAN_REPO_DIR = "/var/www/apt.peachcloud.org/debian" -DEBIAN_REPO_CONF_DIR = "/var/www/apt.peachcloud.org/debian/conf" +MICROSERVICES_DEB_DIR = "/srv/peachcloud/debs" +FREIGHT_CONF = "/etc/freight.conf" +FREIGHT_LIB = "/var/lib/freight" +FREIGHT_CACHE = "/var/www/apt.peachcloud.org" +# define user path before running the script +USER_PATH = "/home/rust" -# before running this script run `gpg --gen-key` on the server, and put the key id here -# `gpg --list-keys` -GPG_KEY_ID = "4ACEF251EA3E091167E8F03EBF69A52BE3565476" +# before running this script run `gpg --gen-key` on the server +# assign the email address of the key id here: +GPG_KEY_EMAIL = "andrew@mycelial.technology" +# save the key passphrase to file and assign the path here: +# (ensure the file is only readable by the user running freight) +GPG_KEY_PASS_FILE = "/home/rust/passphrase.txt" +# if you need to list the existing keys: `gpg --list-keys` SERVICES = [ - {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, + {"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"}, + {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, + {"name": "peach-monitor", "repo_url": "https://github.com/peachcloud/peach-monitor.git"}, {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"}, + {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, {"name": "peach-stats", "repo_url": "https://github.com/peachcloud/peach-stats.git"}, # {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear - {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, - {"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"} ] # parse CLI args @@ -35,27 +43,47 @@ args = parser.parse_args() if args.initialize: print("[ INSTALLING SYSTEM REQUIREMENTS ]") - subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "reprepro", "gcc-aarch64-linux-gnu", ]) + subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "gcc-aarch64-linux-gnu", ]) print("[ CREATING DIRECTORIES ]") - folders = [MICROSERVICES_SRC_DIR, WEB_DIR, APT_DIR, DEBIAN_REPO_DIR, DEBIAN_REPO_CONF_DIR] + folders = [MICROSERVICES_SRC_DIR, FREIGHT_CACHE, FREIGHT_LIB] for folder in folders: if not os.path.exists(folder): os.makedirs(folder) print("[ INSTALLING RUST ]") - if not os.path.exists("/root/.cargo/bin/rustc"): + rustc_path = os.path.join(USER_PATH, ".cargo/bin/rustc") + if not os.path.exists(rustc_path): first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) first_command.wait() print("[ INSTALLING CARGO-DEB ]") - if not os.path.exists("/root/.cargo/bin/cargo-deb"): - subprocess.call(["/root/.cargo/bin/cargo", "install", "cargo-deb"]) + cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") + cargo_deb_path = os.path.join(USER_PATH, ".cargo/bin/cargo-deb") + if not os.path.exists(cargo_deb_path): + subprocess.call([cargo_path, "install", "cargo-deb"]) print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") - subprocess.call(["/root/.cargo/bin/rustup", "target", "add", "aarch64-unknown-linux-gnu"]) - subprocess.call(["/root/.cargo/bin/rustup", "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) + rustup_path = os.path.join(USER_PATH, ".cargo/bin/rustup") + subprocess.call([rustup_path, "target", "add", "aarch64-unknown-linux-gnu"]) + subprocess.call([rustup_path, "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) + + print("[ INSTALLING FREIGHT ]") + freight_path = os.path.join(AUTOMATION_DIR, "freight") + if not os.path.exists(freight_path): + subprocess.call(["git", "clone", "https://github.com/freight-team/freight.git", freight_path]) + + print("[ CONFIGURING FREIGHT ]") + render_template( + src="debian_repo/freight.conf", + dest=FREIGHT_CONF, + template_vars={ + "freight_lib_path": FREIGHT_LIB, + "freight_cache_path": FREIGHT_CACHE, + "gpg_key_email": GPG_KEY_EMAIL + } + ) print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") for service in SERVICES: @@ -65,59 +93,37 @@ if args.initialize: if not os.path.exists(service_path): subprocess.call(["git", "clone", repo_url, service_path]) - print("[ COPYING DEBIAN REPO CONFIG ]") - render_template( - src="debian_repo/distributions", - dest="{}/distributions".format(DEBIAN_REPO_CONF_DIR), - template_vars={ - "gpg_key_id": GPG_KEY_ID - } - ) - render_template( - src="debian_repo/options", - dest="{}/options".format(DEBIAN_REPO_CONF_DIR), - template_vars={ - "debian_rep_dir": DEBIAN_REPO_DIR - } - ) - render_template( - src="debian_repo/override.buster", - dest="{}/override.buster".format(DEBIAN_REPO_CONF_DIR), - template_vars={ - "services": [service["name"] for service in SERVICES] - } - ) - print("[ EXPORTING PUBLIC GPG KEY ]") - output_path = "{}/peach_pub.gpg".format(APT_DIR) + output_path = "{}/peach_pub.gpg".format(FREIGHT_CACHE) if not os.path.exists(output_path): - subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID]) + subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_EMAIL]) print("[ COPYING NGINX CONFIG ]") render_template( src="debian_repo/nginx_debian.conf", dest="/etc/nginx/sites-enabled/apt.peachcloud.org", template_vars = { - "apt_dir": APT_DIR + "apt_dir": FREIGHT_CACHE } ) - -# below is code for git updating the microservices, building the microservices, -# and (re)-adding them to the debian repo +# update the microservices from git and build the debian packages print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") for service in SERVICES: service_name = service["name"] service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) print("[ BUILIDING SERVICE {} ]".format(service_name)) subprocess.call(["git", "pull"], cwd=service_path) - debian_package_path = subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip() - # remove debian package from repo - # (in the future we could look at some way of updating with versions instead of removing and adding) - subprocess.call(["reprepro", "remove", "buster", service_name], cwd=DEBIAN_REPO_DIR) - # add the package - subprocess.call(["reprepro", "includedeb", "buster", debian_package_path], cwd=DEBIAN_REPO_DIR) + debian_package_path = subprocess.check_output([cargo_path, "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip() + # copy package to staging folder + subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) +print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") +for package in MICROSERVICES_DEB_DIR: + subprocess.call(["freight", "add", package, "apt/buster"]) + +print("[ ADDING PACKAGES TO FREIGHT CACHE ]") +# needs to be run as sudo user +subprocess.call(["sudo", "freight", "cache", "-g", GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) print("[ DEBIAN REPO SETUP COMPLETE ]") - From 13edd641afc73dea952eaf06dfbe5a401c349fbf Mon Sep 17 00:00:00 2001 From: mycognosist Date: Mon, 30 Nov 2020 13:39:21 +0000 Subject: [PATCH 02/22] Add Freight configuration template --- conf/templates/debian_repo/freight.conf | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 conf/templates/debian_repo/freight.conf diff --git a/conf/templates/debian_repo/freight.conf b/conf/templates/debian_repo/freight.conf new file mode 100644 index 0000000..40b731f --- /dev/null +++ b/conf/templates/debian_repo/freight.conf @@ -0,0 +1,23 @@ +# Freight configuration. + +# Directories for the Freight library and Freight cache. Your web +# server's document root should be `$VARCACHE`. +VARLIB="{{freight_lib_path}}" +VARCACHE="{{freight_cache_path}}" + +# Default `Origin` and `Label` fields for `Release` files. +ORIGIN="apt.peachcloud.org" +LABEL="PeachCloud" + +# Cache the control files after each run (on), or regenerate them every +# time (off). +CACHE="off" + +# GPG key to use to sign repositories. This is required by the `apt` +# repository provider. Use `gpg --gen-key` (see `gpg`(1) for more +# details) to generate a key and put its email address here. +GPG="{{gpg_key_email}}" + +# Whether to follow symbolic links in `$VARLIB` to produce extra components +# in the cache directory (on) or not (off). +SYMLINKS="off" From 24ff9733a954c96641389d471652fe31c3788c7c Mon Sep 17 00:00:00 2001 From: mycognosist Date: Mon, 30 Nov 2020 13:39:44 +0000 Subject: [PATCH 03/22] Remove unnecessary templates --- conf/templates/debian_repo/distributions | 8 -------- conf/templates/debian_repo/options | 3 --- conf/templates/debian_repo/override.buster | 4 ---- 3 files changed, 15 deletions(-) delete mode 100644 conf/templates/debian_repo/distributions delete mode 100644 conf/templates/debian_repo/options delete mode 100644 conf/templates/debian_repo/override.buster diff --git a/conf/templates/debian_repo/distributions b/conf/templates/debian_repo/distributions deleted file mode 100644 index 45d7c64..0000000 --- a/conf/templates/debian_repo/distributions +++ /dev/null @@ -1,8 +0,0 @@ -Origin: PeachCloud -Label: PeachCloud -Codename: buster -Architectures: amd64 arm64 -Components: main -Description: Apt repository for PeachCloud debian packages -SignWith: {{gpg_key_id}} -DebOverride: override.buster diff --git a/conf/templates/debian_repo/options b/conf/templates/debian_repo/options deleted file mode 100644 index 5dd7345..0000000 --- a/conf/templates/debian_repo/options +++ /dev/null @@ -1,3 +0,0 @@ -verbose -basedir {{debian_rep_dir}} -ask-passphrase diff --git a/conf/templates/debian_repo/override.buster b/conf/templates/debian_repo/override.buster deleted file mode 100644 index ec9fb2d..0000000 --- a/conf/templates/debian_repo/override.buster +++ /dev/null @@ -1,4 +0,0 @@ -{% for service in services %} -{{service}} Priority optional -{{service}} Section net -{% endfor %} \ No newline at end of file From 32bb828b74749f03259baa9eb198354e484a20f7 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Mon, 30 Nov 2020 13:45:27 +0000 Subject: [PATCH 04/22] Update usage instructions and add version number --- README.md | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 04c99f1..f815118 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,60 @@ -# peach-vps config +# peach-vps -Scripts for configuring the peachcloud vps for various hosting and automation. +![Generic badge](https://img.shields.io/badge/version-0.2.0-.svg) + +Scripts for configuring the PeachCloud VPS for various hosting and automation functions. Currently: -- debian repository of microservices +- Debian repository of microservices +# Setup Debian repo + +An idempotent script for initializing the Debian repo on the VPS -# setup debian repo -an idempotent script for initializing the debian repo on the vps ``` apt update apt install git python python3-pip rsync git clone https://github.com/peachcloud/peach-vps.git cd peach-vps pip3 install -r requirements.txt +# open scripts/setup_debian_repo.py and set the following constants: +# USER_PATH, GPG_KEY_EMAIL, GPG_KEY_PASS_FILE python3 scripts/setup_debian_repo.py -i ``` +# Update Debian repo + +Without the -i flag, the `setup_debian_repo` script rebuilds all +microservices (cross-compiled to arm64) and updates the Debian repo -# update debian repo -without the -i flag, the setup_debian_repo rebuilds all -microservices (cross-compiled to arm64) and re-adds them to the debian repo ``` cd peach-vps python3 scripts/setup_debian_repo.py ``` +# Using the Debian repo + +To add the PeachCloud Debian repo as an apt source, run the following commands from your Pi: -# using the debian repo on the pi -To add the peachcloud debian repo as an apt source, -on the pi, ``` vi /etc/apt/sources.list.d/peach.list ``` + and add the following line: + ``` deb http://apt.peachcloud.org/debian/ buster main ``` Then add the gpg pub key to the apt-key list: + ``` wget -O - http://apt.peachcloud.org/peach_pub.gpg | sudo apt-key add - ``` You can then install peach packages with apt-get: + ``` apt-get update apt-get install peach-oled -``` \ No newline at end of file +``` From 257ada4a6f2a7bc07c129232a2750e537a425f34 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 2 Dec 2020 15:50:36 +0000 Subject: [PATCH 05/22] Update install instructions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f815118..72aa85d 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ Currently: An idempotent script for initializing the Debian repo on the VPS ``` -apt update -apt install git python python3-pip rsync +sudo apt update +sudo apt install git python python3-pip rsync git clone https://github.com/peachcloud/peach-vps.git cd peach-vps pip3 install -r requirements.txt From 2e2d2fc5b24e3099573d17b6b90fa277d0aabb0b Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 4 Dec 2020 08:13:44 +0000 Subject: [PATCH 06/22] Move assignment of cargo_path outside of loop --- scripts/setup_debian_repo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 658c423..046c1af 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -38,6 +38,8 @@ parser = argparse.ArgumentParser() parser.add_argument("-i", "--initialize", help="initialize and update debian repo", action="store_true") args = parser.parse_args() +cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") + # initializing debian repo from a blank slate # (but this code is idempotent so it can be re-run if already initialized) if args.initialize: @@ -59,7 +61,6 @@ if args.initialize: first_command.wait() print("[ INSTALLING CARGO-DEB ]") - cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") cargo_deb_path = os.path.join(USER_PATH, ".cargo/bin/cargo-deb") if not os.path.exists(cargo_deb_path): subprocess.call([cargo_path, "install", "cargo-deb"]) From ce4a25395c5676c0303b6c15763c299e3dcb65c4 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 4 Dec 2020 08:39:12 +0000 Subject: [PATCH 07/22] Add sudo for apt update --- scripts/setup_debian_repo.py | 2 +- scripts/utils.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 046c1af..625384f 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -45,7 +45,7 @@ cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") if args.initialize: print("[ INSTALLING SYSTEM REQUIREMENTS ]") - subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "gcc-aarch64-linux-gnu", ]) + subprocess.call(["sudo", "apt-get", "install", "git", "nginx", "curl", "build-essential", "gcc-aarch64-linux-gnu", ]) print("[ CREATING DIRECTORIES ]") folders = [MICROSERVICES_SRC_DIR, FREIGHT_CACHE, FREIGHT_LIB] diff --git a/scripts/utils.py b/scripts/utils.py index 6d208a1..69f2bdf 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -25,5 +25,3 @@ def render_template(src, dest, template_vars=None): os.remove(dest) with open(dest, 'w') as f: f.write(output_text) - - From 0abda0b46b9382ba28de46fb15d3e974624734ab Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 4 Dec 2020 09:09:18 +0000 Subject: [PATCH 08/22] Replace deprecated check_output with run --- scripts/setup_debian_repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 625384f..05c96e1 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -115,7 +115,7 @@ for service in SERVICES: service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) print("[ BUILIDING SERVICE {} ]".format(service_name)) subprocess.call(["git", "pull"], cwd=service_path) - debian_package_path = subprocess.check_output([cargo_path, "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip() + debian_package_path = subprocess.run([cargo_path, "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path, stdout=subprocess.PIPE).stdout.decode("utf-8").strip() # copy package to staging folder subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) From f438e6b8ed140d96c4af7d89189447ecb8cac85a Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 4 Dec 2020 09:20:41 +0000 Subject: [PATCH 09/22] Add copy step for Freight conf to avoid permissions error --- scripts/setup_debian_repo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 05c96e1..dc9ea39 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -78,13 +78,14 @@ if args.initialize: print("[ CONFIGURING FREIGHT ]") render_template( src="debian_repo/freight.conf", - dest=FREIGHT_CONF, + dest="/tmp", template_vars={ "freight_lib_path": FREIGHT_LIB, "freight_cache_path": FREIGHT_CACHE, "gpg_key_email": GPG_KEY_EMAIL } ) + subprocess.call(["sudo", "cp", "/tmp/freight.conf", FREIGHT_CONF]) print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") for service in SERVICES: From a99effa4f59a68421b359b998ef3242f7ea44a72 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 4 Dec 2020 09:29:04 +0000 Subject: [PATCH 10/22] Add copy commands for freight and nginx conf --- scripts/setup_debian_repo.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index dc9ea39..7273f7c 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -76,16 +76,17 @@ if args.initialize: subprocess.call(["git", "clone", "https://github.com/freight-team/freight.git", freight_path]) print("[ CONFIGURING FREIGHT ]") + freight_conf_tmp_path = os.path.join(USER_PATH, "freight.conf") render_template( src="debian_repo/freight.conf", - dest="/tmp", + dest=freight_conf_tmp_path, template_vars={ "freight_lib_path": FREIGHT_LIB, "freight_cache_path": FREIGHT_CACHE, "gpg_key_email": GPG_KEY_EMAIL } ) - subprocess.call(["sudo", "cp", "/tmp/freight.conf", FREIGHT_CONF]) + subprocess.call(["sudo", "cp", freight_conf_tmp_path, FREIGHT_CONF]) print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") for service in SERVICES: @@ -101,6 +102,8 @@ if args.initialize: subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_EMAIL]) print("[ COPYING NGINX CONFIG ]") + nginx_conf_tmp_path = os.path.join(USER_PATH, "apt.peachcloud.org") + render_template( render_template( src="debian_repo/nginx_debian.conf", dest="/etc/nginx/sites-enabled/apt.peachcloud.org", @@ -108,6 +111,7 @@ if args.initialize: "apt_dir": FREIGHT_CACHE } ) + subprocess.call(["sudo", "cp", nginx_conf_tmp_path, "/etc/nginx/sites-enabled/apt.peachcloud.org"]) # update the microservices from git and build the debian packages print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") From 77a22868282f4ce41b14383b1f488667be210fb3 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 4 Dec 2020 09:30:08 +0000 Subject: [PATCH 11/22] Fix syntax error --- scripts/setup_debian_repo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 7273f7c..7dc7cb6 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -103,7 +103,6 @@ if args.initialize: print("[ COPYING NGINX CONFIG ]") nginx_conf_tmp_path = os.path.join(USER_PATH, "apt.peachcloud.org") - render_template( render_template( src="debian_repo/nginx_debian.conf", dest="/etc/nginx/sites-enabled/apt.peachcloud.org", From 8206b4d8d8f1893e94fda00fd0a403b8f7d9e1a6 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 4 Dec 2020 09:31:27 +0000 Subject: [PATCH 12/22] Fix tmp nginx conf path --- scripts/setup_debian_repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 7dc7cb6..1ba4af7 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -105,7 +105,7 @@ if args.initialize: nginx_conf_tmp_path = os.path.join(USER_PATH, "apt.peachcloud.org") render_template( src="debian_repo/nginx_debian.conf", - dest="/etc/nginx/sites-enabled/apt.peachcloud.org", + dest=nginx_conf_tmp_path, template_vars = { "apt_dir": FREIGHT_CACHE } From dd8a340afff660e72b2480d75f9ef3fac076f35f Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 09:26:11 +0000 Subject: [PATCH 13/22] Add freight conf file location to add command --- scripts/setup_debian_repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 1ba4af7..381b172 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -125,7 +125,7 @@ for service in SERVICES: print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") for package in MICROSERVICES_DEB_DIR: - subprocess.call(["freight", "add", package, "apt/buster"]) + subprocess.call(["freight", "add", "-c", FREIGHT_CONF, package, "apt/buster"]) print("[ ADDING PACKAGES TO FREIGHT CACHE ]") # needs to be run as sudo user From 2cdfaa1116fb0d34a7bc5168ae55446165d3b667 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 09:38:50 +0000 Subject: [PATCH 14/22] Add full deb path for each package when adding to freight --- scripts/setup_debian_repo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 381b172..1261378 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -125,7 +125,9 @@ for service in SERVICES: print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") for package in MICROSERVICES_DEB_DIR: - subprocess.call(["freight", "add", "-c", FREIGHT_CONF, package, "apt/buster"]) + package_path = os.path.join(MICROSERVICES_DEB_DIR, package) + print("[ ADDING PACKAGE {} ]".format(package)) + subprocess.call(["freight", "add", "-c", FREIGHT_CONF, package_path, "apt/buster"]) print("[ ADDING PACKAGES TO FREIGHT CACHE ]") # needs to be run as sudo user From 3c04a5857069e44e79bbb8ea88c87f6879620c19 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 10:05:02 +0000 Subject: [PATCH 15/22] Fix path for adding debs to freight library --- scripts/setup_debian_repo.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 1261378..a7aee20 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -124,10 +124,12 @@ for service in SERVICES: subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") -for package in MICROSERVICES_DEB_DIR: - package_path = os.path.join(MICROSERVICES_DEB_DIR, package) - print("[ ADDING PACKAGE {} ]".format(package)) - subprocess.call(["freight", "add", "-c", FREIGHT_CONF, package_path, "apt/buster"]) +# loop through all files in the microservices deb directory +for filename in os.scandir(MICROSERVICES_DEB_DIR): + # avoid any files which are not debian packages + if filename.endswith(".deb"): + print("[ ADDING PACKAGE {} ]".format(filename.name)) + subprocess.call(["freight", "add", "-c", FREIGHT_CONF, filename.path, "apt/buster"]) print("[ ADDING PACKAGES TO FREIGHT CACHE ]") # needs to be run as sudo user From b972f1274a09c9332d28b0b232cc80850d21c5a4 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 10:12:08 +0000 Subject: [PATCH 16/22] Fix package endswith error --- scripts/setup_debian_repo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index a7aee20..ffef7c3 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -125,11 +125,11 @@ for service in SERVICES: print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") # loop through all files in the microservices deb directory -for filename in os.scandir(MICROSERVICES_DEB_DIR): +for package in os.scandir(MICROSERVICES_DEB_DIR): # avoid any files which are not debian packages - if filename.endswith(".deb"): - print("[ ADDING PACKAGE {} ]".format(filename.name)) - subprocess.call(["freight", "add", "-c", FREIGHT_CONF, filename.path, "apt/buster"]) + if package.name.endswith(".deb"): + print("[ ADDING PACKAGE {} ]".format(package.name)) + subprocess.call(["freight", "add", "-c", FREIGHT_CONF, package.path, "apt/buster"]) print("[ ADDING PACKAGES TO FREIGHT CACHE ]") # needs to be run as sudo user From db26c7137fb1ee3075221366ac03d3e272d6ec1e Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 10:37:22 +0000 Subject: [PATCH 17/22] Define arm64 architecture for freight --- conf/templates/debian_repo/freight.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/templates/debian_repo/freight.conf b/conf/templates/debian_repo/freight.conf index 40b731f..bb9c926 100644 --- a/conf/templates/debian_repo/freight.conf +++ b/conf/templates/debian_repo/freight.conf @@ -9,6 +9,9 @@ VARCACHE="{{freight_cache_path}}" ORIGIN="apt.peachcloud.org" LABEL="PeachCloud" +# Architectures supported by the package repository. +ARCHS="arm64" + # Cache the control files after each run (on), or regenerate them every # time (off). CACHE="off" From 973e663eb096abca1ddb931a5723f815b0e8196b Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 12:43:05 +0000 Subject: [PATCH 18/22] Move conf files to template dir root --- conf/templates/{debian_repo => }/freight.conf | 0 conf/templates/{debian_repo => }/nginx_debian.conf | 0 scripts/setup_debian_repo.py | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename conf/templates/{debian_repo => }/freight.conf (100%) rename conf/templates/{debian_repo => }/nginx_debian.conf (100%) diff --git a/conf/templates/debian_repo/freight.conf b/conf/templates/freight.conf similarity index 100% rename from conf/templates/debian_repo/freight.conf rename to conf/templates/freight.conf diff --git a/conf/templates/debian_repo/nginx_debian.conf b/conf/templates/nginx_debian.conf similarity index 100% rename from conf/templates/debian_repo/nginx_debian.conf rename to conf/templates/nginx_debian.conf diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index ffef7c3..c5a94ab 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -78,7 +78,7 @@ if args.initialize: print("[ CONFIGURING FREIGHT ]") freight_conf_tmp_path = os.path.join(USER_PATH, "freight.conf") render_template( - src="debian_repo/freight.conf", + src="freight.conf", dest=freight_conf_tmp_path, template_vars={ "freight_lib_path": FREIGHT_LIB, @@ -104,7 +104,7 @@ if args.initialize: print("[ COPYING NGINX CONFIG ]") nginx_conf_tmp_path = os.path.join(USER_PATH, "apt.peachcloud.org") render_template( - src="debian_repo/nginx_debian.conf", + src="nginx_debian.conf", dest=nginx_conf_tmp_path, template_vars = { "apt_dir": FREIGHT_CACHE From 85c0bf323ae3f96c9d91819d7427ce2a598d3ebd Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 12:57:01 +0000 Subject: [PATCH 19/22] Fix formatting and remove unnecessary comments --- scripts/setup_debian_repo.py | 87 ++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index c5a94ab..2350122 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -5,16 +5,6 @@ import os import argparse -# constants -AUTOMATION_DIR = "/srv/peachcloud/automation" -MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" -MICROSERVICES_DEB_DIR = "/srv/peachcloud/debs" -FREIGHT_CONF = "/etc/freight.conf" -FREIGHT_LIB = "/var/lib/freight" -FREIGHT_CACHE = "/var/www/apt.peachcloud.org" -# define user path before running the script -USER_PATH = "/home/rust" - # before running this script run `gpg --gen-key` on the server # assign the email address of the key id here: GPG_KEY_EMAIL = "andrew@mycelial.technology" @@ -23,11 +13,25 @@ GPG_KEY_EMAIL = "andrew@mycelial.technology" GPG_KEY_PASS_FILE = "/home/rust/passphrase.txt" # if you need to list the existing keys: `gpg --list-keys` + +# constants +AUTOMATION_DIR = "/srv/peachcloud/automation" +FREIGHT_CONF = "/etc/freight.conf" +FREIGHT_LIB = "/var/lib/freight" +FREIGHT_CACHE = "/var/www/apt.peachcloud.org" +MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" +MICROSERVICES_DEB_DIR = "/srv/peachcloud/debs" +USER_PATH = "/home/rust" + + SERVICES = [ - {"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"}, + {"name": "peach-buttons", + "repo_url": "https://github.com/peachcloud/peach-buttons.git"}, {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, - {"name": "peach-monitor", "repo_url": "https://github.com/peachcloud/peach-monitor.git"}, - {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"}, + {"name": "peach-monitor", + "repo_url": "https://github.com/peachcloud/peach-monitor.git"}, + {"name": "peach-network", + "repo_url": "https://github.com/peachcloud/peach-network.git"}, {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, {"name": "peach-stats", "repo_url": "https://github.com/peachcloud/peach-stats.git"}, # {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear @@ -35,7 +39,11 @@ SERVICES = [ # parse CLI args parser = argparse.ArgumentParser() -parser.add_argument("-i", "--initialize", help="initialize and update debian repo", action="store_true") +parser.add_argument( + "-i", + "--initialize", + help="initialize and update debian repo", + action="store_true") args = parser.parse_args() cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") @@ -45,7 +53,15 @@ cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") if args.initialize: print("[ INSTALLING SYSTEM REQUIREMENTS ]") - subprocess.call(["sudo", "apt-get", "install", "git", "nginx", "curl", "build-essential", "gcc-aarch64-linux-gnu", ]) + subprocess.call(["sudo", + "apt-get", + "install", + "git", + "nginx", + "curl", + "build-essential", + "gcc-aarch64-linux-gnu", + ]) print("[ CREATING DIRECTORIES ]") folders = [MICROSERVICES_SRC_DIR, FREIGHT_CACHE, FREIGHT_LIB] @@ -56,8 +72,10 @@ if args.initialize: print("[ INSTALLING RUST ]") rustc_path = os.path.join(USER_PATH, ".cargo/bin/rustc") if not os.path.exists(rustc_path): - first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) - output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) + first_command = subprocess.Popen( + ["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) + output = subprocess.check_output( + ["sh", "-s", "--", "-y"], stdin=first_command.stdout) first_command.wait() print("[ INSTALLING CARGO-DEB ]") @@ -67,13 +85,16 @@ if args.initialize: print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") rustup_path = os.path.join(USER_PATH, ".cargo/bin/rustup") - subprocess.call([rustup_path, "target", "add", "aarch64-unknown-linux-gnu"]) - subprocess.call([rustup_path, "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) + subprocess.call([rustup_path, "target", "add", + "aarch64-unknown-linux-gnu"]) + subprocess.call([rustup_path, "toolchain", "install", + "nightly-aarch64-unknown-linux-gnu"]) print("[ INSTALLING FREIGHT ]") freight_path = os.path.join(AUTOMATION_DIR, "freight") if not os.path.exists(freight_path): - subprocess.call(["git", "clone", "https://github.com/freight-team/freight.git", freight_path]) + subprocess.call( + ["git", "clone", "https://github.com/freight-team/freight.git", freight_path]) print("[ CONFIGURING FREIGHT ]") freight_conf_tmp_path = os.path.join(USER_PATH, "freight.conf") @@ -99,18 +120,20 @@ if args.initialize: print("[ EXPORTING PUBLIC GPG KEY ]") output_path = "{}/peach_pub.gpg".format(FREIGHT_CACHE) if not os.path.exists(output_path): - subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_EMAIL]) + subprocess.call(["gpg", "--armor", "--output", + output_path, "--export", GPG_KEY_EMAIL]) print("[ COPYING NGINX CONFIG ]") nginx_conf_tmp_path = os.path.join(USER_PATH, "apt.peachcloud.org") render_template( src="nginx_debian.conf", dest=nginx_conf_tmp_path, - template_vars = { + template_vars={ "apt_dir": FREIGHT_CACHE } ) - subprocess.call(["sudo", "cp", nginx_conf_tmp_path, "/etc/nginx/sites-enabled/apt.peachcloud.org"]) + subprocess.call(["sudo", "cp", nginx_conf_tmp_path, + "/etc/nginx/sites-enabled/apt.peachcloud.org"]) # update the microservices from git and build the debian packages print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") @@ -119,20 +142,26 @@ for service in SERVICES: service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) print("[ BUILIDING SERVICE {} ]".format(service_name)) subprocess.call(["git", "pull"], cwd=service_path) - debian_package_path = subprocess.run([cargo_path, "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path, stdout=subprocess.PIPE).stdout.decode("utf-8").strip() - # copy package to staging folder + debian_package_path = subprocess.run( + [ + cargo_path, + "deb", + "--target", + "aarch64-unknown-linux-gnu"], + cwd=service_path, + stdout=subprocess.PIPE).stdout.decode("utf-8").strip() subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") -# loop through all files in the microservices deb directory for package in os.scandir(MICROSERVICES_DEB_DIR): - # avoid any files which are not debian packages if package.name.endswith(".deb"): print("[ ADDING PACKAGE {} ]".format(package.name)) - subprocess.call(["freight", "add", "-c", FREIGHT_CONF, package.path, "apt/buster"]) + subprocess.call(["freight", "add", "-c", FREIGHT_CONF, + package.path, "apt/buster"]) print("[ ADDING PACKAGES TO FREIGHT CACHE ]") # needs to be run as sudo user -subprocess.call(["sudo", "freight", "cache", "-g", GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) +subprocess.call(["sudo", "freight", "cache", "-g", + GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) print("[ DEBIAN REPO SETUP COMPLETE ]") From 44ff03a18d3bfe682969e053469c570059f384c0 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 13:19:56 +0000 Subject: [PATCH 20/22] Update README with usage instructions and license --- README.md | 74 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 72aa85d..7b2a782 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,36 @@ # peach-vps -![Generic badge](https://img.shields.io/badge/version-0.2.0-.svg) +![Generic badge](https://img.shields.io/badge/version-0.2.1-.svg) Scripts for configuring the PeachCloud VPS for various hosting and automation functions. Currently: -- Debian repository of microservices -# Setup Debian repo + - Debian repository of microservices -An idempotent script for initializing the Debian repo on the VPS +## Setup Debian Repo + +`scripts/setup_debian_repo.py` + +An idempotent script for initializing the Debian repo on the VPS. + +The script currently performs the following actions: + + - Installs system requirements + - Creates directories for microservices and package archive + - Installs Rust + - Installs `cargo deb` + - Installs Rust aarch64 toolchain for cross-compilation + - Installs Freight for package archive creation and management + - Configures Freight + - Pulls microservices code from GitHub repos + - Exports the public GPG key + - Configures nginx + - Builds and updates microservice packages + - Adds packages to Freight library + - Adds packages to Freight cache + +Prior to executing the script for the first time, run the following commands on the target system: ``` sudo apt update @@ -17,22 +38,31 @@ sudo apt install git python python3-pip rsync git clone https://github.com/peachcloud/peach-vps.git cd peach-vps pip3 install -r requirements.txt -# open scripts/setup_debian_repo.py and set the following constants: -# USER_PATH, GPG_KEY_EMAIL, GPG_KEY_PASS_FILE -python3 scripts/setup_debian_repo.py -i ``` -# Update Debian repo +Open `scripts/setup_debian_repo.py` and set the following constants: -Without the -i flag, the `setup_debian_repo` script rebuilds all + - USER_PATH + - GPG_KEY_EMAIL + - GPG_KEY_PASS_FILE + +Then execute the script with the `-i` flag to run the full system initialization process (_note: several commands executed by the script require `sudo` permissions. You will be prompted for the user password during the execution of the scipt._): + +``` +python3 -u scripts/setup_debian_repo.py -i +``` + +## Update Debian Repo + +Without the -i flag, the `setup_debian_repo.py` script rebuilds all microservices (cross-compiled to arm64) and updates the Debian repo ``` cd peach-vps -python3 scripts/setup_debian_repo.py +python3 -u scripts/setup_debian_repo.py ``` -# Using the Debian repo +## Install from Debian Repo To add the PeachCloud Debian repo as an apt source, run the following commands from your Pi: @@ -40,21 +70,33 @@ To add the PeachCloud Debian repo as an apt source, run the following commands f vi /etc/apt/sources.list.d/peach.list ``` -and add the following line: +Append the following line: ``` deb http://apt.peachcloud.org/debian/ buster main ``` -Then add the gpg pub key to the apt-key list: +Add the gpg pub key to the apt-key list: ``` wget -O - http://apt.peachcloud.org/peach_pub.gpg | sudo apt-key add - ``` -You can then install peach packages with apt-get: +You can then install peach packages with apt: ``` -apt-get update -apt-get install peach-oled +sudo apt update +sudo apt install peach-oled ``` + +By default, the latest version of the package will be downloaded and installed. + +Specific versions of packages can be selected for installation by supplying the semantic versioning number (this is useful for downgrading): + +``` +sudo apt install peach-network=0.2.0 +``` + +## Licensing + +AGPL-3.0 From 22653a2ea28e15db9f49a330e8fe5e7c1bc7704d Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 13:20:37 +0000 Subject: [PATCH 21/22] Add shebang line for python3 --- scripts/setup_debian_repo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 2350122..11ddd4b 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + from utils import render_template import subprocess From 9799b0f1c02101820585a6055792c4b74a9ee038 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Wed, 16 Dec 2020 13:21:48 +0000 Subject: [PATCH 22/22] Add link to correct Freight repo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b2a782..b70af17 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Scripts for configuring the PeachCloud VPS for various hosting and automation fu Currently: - - Debian repository of microservices + - Debian repository of microservices (using [Freight](https://github.com/freight-team/freight)) ## Setup Debian Repo