From 4f5fdcf9adab6f6a7503598b3ddc9b9a48be51f5 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 18 Dec 2020 09:05:39 +0000 Subject: [PATCH 1/5] Split scripts for env setup and package building --- scripts/build_packages.py | 62 ++++++++++++++ ...etup_debian_repo.py => setup_build_env.py} | 85 +++++++------------ 2 files changed, 92 insertions(+), 55 deletions(-) create mode 100644 scripts/build_packages.py rename scripts/{setup_debian_repo.py => setup_build_env.py} (65%) diff --git a/scripts/build_packages.py b/scripts/build_packages.py new file mode 100644 index 0000000..107637a --- /dev/null +++ b/scripts/build_packages.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import subprocess +import os + + +GPG_KEY_EMAIL = "andrew@mycelial.technology" +# save the key passphrase to file and assign the path here: +# (ensure the file is only readable by the user running freight) +GPG_KEY_PASS_FILE = "/home/rust/passphrase.txt" + + +FREIGHT_CONF = "/etc/freight.conf" +MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" +MICROSERVICES_DEB_DIR = "/srv/peachcloud/debs" +USER_PATH = "/home/rust" + + +SERVICES = [ + {"name": "peach-buttons", + "repo_url": "https://github.com/peachcloud/peach-buttons.git"}, + {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, + {"name": "peach-monitor", + "repo_url": "https://github.com/peachcloud/peach-monitor.git"}, + {"name": "peach-network", + "repo_url": "https://github.com/peachcloud/peach-network.git"}, + {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, + {"name": "peach-stats", "repo_url": "https://github.com/peachcloud/peach-stats.git"}, + # {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear +] + +cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") + +print("\n[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]\n") +for service in SERVICES: + service_name = service["name"] + service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) + print("\n[ BUILIDING SERVICE {} ]\n".format(service_name)) + subprocess.call(["git", "pull"], cwd=service_path) + debian_package_path = subprocess.run( + [ + cargo_path, + "deb", + "--target", + "aarch64-unknown-linux-gnu"], + cwd=service_path, + stdout=subprocess.PIPE).stdout.decode("utf-8").strip() + subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) + +print("\n[ ADDING PACKAGES TO FREIGHT LIBRARY ]\n") +for package in os.scandir(MICROSERVICES_DEB_DIR): + if package.name.endswith(".deb"): + print("\n[ ADDING PACKAGE {} ]\n".format(package.name)) + subprocess.call(["freight", "add", "-c", FREIGHT_CONF, + package.path, "apt/buster"]) + +print("\n[ ADDING PACKAGES TO FREIGHT CACHE ]\n") +# needs to be run as sudo user +subprocess.call(["sudo", "freight", "cache", "-g", + GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) + +print("\n[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]") diff --git a/scripts/setup_debian_repo.py b/scripts/setup_build_env.py similarity index 65% rename from scripts/setup_debian_repo.py rename to scripts/setup_build_env.py index 11ddd4b..6fcd39c 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_build_env.py @@ -6,14 +6,9 @@ import subprocess import os import argparse - # before running this script run `gpg --gen-key` on the server # assign the email address of the key id here: GPG_KEY_EMAIL = "andrew@mycelial.technology" -# save the key passphrase to file and assign the path here: -# (ensure the file is only readable by the user running freight) -GPG_KEY_PASS_FILE = "/home/rust/passphrase.txt" -# if you need to list the existing keys: `gpg --list-keys` # constants @@ -22,7 +17,6 @@ FREIGHT_CONF = "/etc/freight.conf" FREIGHT_LIB = "/var/lib/freight" FREIGHT_CACHE = "/var/www/apt.peachcloud.org" MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" -MICROSERVICES_DEB_DIR = "/srv/peachcloud/debs" USER_PATH = "/home/rust" @@ -39,22 +33,32 @@ SERVICES = [ # {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear ] +cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") + # parse CLI args parser = argparse.ArgumentParser() parser.add_argument( - "-i", - "--initialize", - help="initialize and update debian repo", - action="store_true") + "-u", + "--update", + help="Update Rust installation", + action="store_true" +) args = parser.parse_args() -cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") -# initializing debian repo from a blank slate -# (but this code is idempotent so it can be re-run if already initialized) -if args.initialize: - - print("[ INSTALLING SYSTEM REQUIREMENTS ]") +# update rust installation +if args.update: + print("\n[ UPDATING RUST ]\n") + rustup_path = os.path.join(USER_PATH, ".cargo/bin/rustup") + if not os.path.exists(rustup_path): + print("rustup installation not found") + print("rerun this script without the '-u' flag to install rust") + else: + subprocess.call([rustup_path, "update"]) +else: + # initialize debian package build environment from a blank slate + # (but this code is idempotent so it can be re-run if already initialized) + print("\n[ INSTALLING SYSTEM REQUIREMENTS ]\n") subprocess.call(["sudo", "apt-get", "install", @@ -65,13 +69,13 @@ if args.initialize: "gcc-aarch64-linux-gnu", ]) - print("[ CREATING DIRECTORIES ]") + print("\n[ CREATING DIRECTORIES ]\n") folders = [MICROSERVICES_SRC_DIR, FREIGHT_CACHE, FREIGHT_LIB] for folder in folders: if not os.path.exists(folder): os.makedirs(folder) - print("[ INSTALLING RUST ]") + print("\n[ INSTALLING RUST ]\n") rustc_path = os.path.join(USER_PATH, ".cargo/bin/rustc") if not os.path.exists(rustc_path): first_command = subprocess.Popen( @@ -80,25 +84,25 @@ if args.initialize: ["sh", "-s", "--", "-y"], stdin=first_command.stdout) first_command.wait() - print("[ INSTALLING CARGO-DEB ]") + print("\n[ INSTALLING CARGO-DEB ]\n") cargo_deb_path = os.path.join(USER_PATH, ".cargo/bin/cargo-deb") if not os.path.exists(cargo_deb_path): subprocess.call([cargo_path, "install", "cargo-deb"]) - print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") + print("\n[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]\n") rustup_path = os.path.join(USER_PATH, ".cargo/bin/rustup") subprocess.call([rustup_path, "target", "add", "aarch64-unknown-linux-gnu"]) subprocess.call([rustup_path, "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) - print("[ INSTALLING FREIGHT ]") + print("\n[ INSTALLING FREIGHT ]\n") freight_path = os.path.join(AUTOMATION_DIR, "freight") if not os.path.exists(freight_path): subprocess.call( ["git", "clone", "https://github.com/freight-team/freight.git", freight_path]) - print("[ CONFIGURING FREIGHT ]") + print("\n[ CONFIGURING FREIGHT ]\n") freight_conf_tmp_path = os.path.join(USER_PATH, "freight.conf") render_template( src="freight.conf", @@ -111,7 +115,7 @@ if args.initialize: ) subprocess.call(["sudo", "cp", freight_conf_tmp_path, FREIGHT_CONF]) - print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") + print("\n[ PULLING MICROSERVICES CODE FROM GITHUB ]\n") for service in SERVICES: name = service["name"] repo_url = service["repo_url"] @@ -119,13 +123,13 @@ if args.initialize: if not os.path.exists(service_path): subprocess.call(["git", "clone", repo_url, service_path]) - print("[ EXPORTING PUBLIC GPG KEY ]") + print("\n[ EXPORTING PUBLIC GPG KEY ]\n") output_path = "{}/peach_pub.gpg".format(FREIGHT_CACHE) if not os.path.exists(output_path): subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_EMAIL]) - print("[ COPYING NGINX CONFIG ]") + print("\n[ COPYING NGINX CONFIG ]\n") nginx_conf_tmp_path = os.path.join(USER_PATH, "apt.peachcloud.org") render_template( src="nginx_debian.conf", @@ -137,33 +141,4 @@ if args.initialize: subprocess.call(["sudo", "cp", nginx_conf_tmp_path, "/etc/nginx/sites-enabled/apt.peachcloud.org"]) -# update the microservices from git and build the debian packages -print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") -for service in SERVICES: - service_name = service["name"] - service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) - print("[ BUILIDING SERVICE {} ]".format(service_name)) - subprocess.call(["git", "pull"], cwd=service_path) - debian_package_path = subprocess.run( - [ - cargo_path, - "deb", - "--target", - "aarch64-unknown-linux-gnu"], - cwd=service_path, - stdout=subprocess.PIPE).stdout.decode("utf-8").strip() - subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) - -print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") -for package in os.scandir(MICROSERVICES_DEB_DIR): - if package.name.endswith(".deb"): - print("[ ADDING PACKAGE {} ]".format(package.name)) - subprocess.call(["freight", "add", "-c", FREIGHT_CONF, - package.path, "apt/buster"]) - -print("[ ADDING PACKAGES TO FREIGHT CACHE ]") -# needs to be run as sudo user -subprocess.call(["sudo", "freight", "cache", "-g", - GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) - -print("[ DEBIAN REPO SETUP COMPLETE ]") + print("\n[ DEBIAN PACKAGE BUILD ENVIRONMENT SETUP COMPLETE ]") From 3ad8d90076e9ade4c15f9078208e9d96c046b33f Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 18 Dec 2020 09:18:59 +0000 Subject: [PATCH 2/5] Update README and bump version --- README.md | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 9780588..dc660e3 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,14 @@ # peach-vps -![Generic badge](https://img.shields.io/badge/version-0.2.1-.svg) +![Generic badge](https://img.shields.io/badge/version-0.3.0-.svg) Scripts for configuring the PeachCloud VPS for various hosting and automation functions. -Currently: +## Setup Build Environment - - Debian repository of microservices (using [Freight](https://github.com/freight-team/freight)) +`scripts/setup_build_env.py` -## Setup Debian Repo - -`scripts/setup_debian_repo.py` - -An idempotent script for initializing the Debian repo on the VPS. +An idempotent script for initializing a build and deployment environment for PeachCloud packages. The script currently performs the following actions: @@ -21,16 +17,15 @@ The script currently performs the following actions: - Installs Rust - Installs `cargo deb` - Installs Rust aarch64 toolchain for cross-compilation - - Installs Freight for package archive creation and management + - Installs [Freight](https://github.com/freight-team/freight) for package archive creation and management - Configures Freight - Pulls microservices code from GitHub repos - Exports the public GPG key - Configures nginx - - Builds and updates microservice packages - - Adds packages to Freight library - - Adds packages to Freight cache -Prior to executing the script for the first time, run the following commands on the target system: +The script can also be run with the optional `-u` flag (`--update`) to update the Rust compiler and installed toolchains. + +**NB:** Prior to executing the script for the first time, run the following commands on the target system: ``` sudo apt update @@ -40,33 +35,39 @@ cd peach-vps pip3 install -r requirements.txt ``` -Open `scripts/setup_debian_repo.py` and set the following constants: +Open `scripts/setup_build_env.py` and set the following constants: - USER_PATH - GPG_KEY_EMAIL - GPG_KEY_PASS_FILE -Then execute the script with the `-i` flag to run the full system initialization process (_note: several commands executed by the script require `sudo` permissions. You will be prompted for the user password during the execution of the scipt._): +Then execute the script to run the full system initialization process (_note: several commands executed by the script require `sudo` permissions. You will be prompted for the user password during the execution of the scipt._): ``` -python3 -u scripts/setup_debian_repo.py -i +python3 -u scripts/setup_build_env.py ``` -## Update Debian Repo +## Build and Serve Debian Packages -Without the -i flag, the `setup_debian_repo.py` script rebuilds all -microservices (cross-compiled to arm64) and updates the Debian repo: +`scripts/build_packages.py` + +An idempotent script for building the latest versions of all PeachCloud packages and adding them to the Debian package archive. + +The script currently performs the following actions: + + - Builds and updates microservice packages + - Adds packages to Freight library + - Adds packages to Freight cache ``` -cd peach-vps -python3 -u scripts/setup_debian_repo.py +python3 -u scripts/build_packages.py ``` Freight supports the ability to have multiple versions of a package in a single Debian package archive. If a particular version of a package already exists in the Freight library, it will not be readded or overwritten. -## Install from Debian Repo +## Install Packages from Debian Package Archive -To add the PeachCloud Debian repo as an apt source, run the following commands from your Pi: +To add the PeachCloud Debian package archive as an apt source, run the following commands from your Pi: ``` vi /etc/apt/sources.list.d/peach.list From 70fddbb6c420ef4c8a20bcdec4080b46a2e4eb10 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Fri, 18 Dec 2020 09:26:15 +0000 Subject: [PATCH 3/5] Remove newline characters --- scripts/build_packages.py | 12 ++++++------ scripts/setup_build_env.py | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/build_packages.py b/scripts/build_packages.py index 107637a..201cb07 100644 --- a/scripts/build_packages.py +++ b/scripts/build_packages.py @@ -31,11 +31,11 @@ SERVICES = [ cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") -print("\n[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]\n") +print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") for service in SERVICES: service_name = service["name"] service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) - print("\n[ BUILIDING SERVICE {} ]\n".format(service_name)) + print("[ BUILIDING SERVICE {} ]".format(service_name)) subprocess.call(["git", "pull"], cwd=service_path) debian_package_path = subprocess.run( [ @@ -47,16 +47,16 @@ for service in SERVICES: stdout=subprocess.PIPE).stdout.decode("utf-8").strip() subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) -print("\n[ ADDING PACKAGES TO FREIGHT LIBRARY ]\n") +print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") for package in os.scandir(MICROSERVICES_DEB_DIR): if package.name.endswith(".deb"): - print("\n[ ADDING PACKAGE {} ]\n".format(package.name)) + print("[ ADDING PACKAGE {} ]".format(package.name)) subprocess.call(["freight", "add", "-c", FREIGHT_CONF, package.path, "apt/buster"]) -print("\n[ ADDING PACKAGES TO FREIGHT CACHE ]\n") +print("[ ADDING PACKAGES TO FREIGHT CACHE ]") # needs to be run as sudo user subprocess.call(["sudo", "freight", "cache", "-g", GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) -print("\n[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]") +print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]") diff --git a/scripts/setup_build_env.py b/scripts/setup_build_env.py index 6fcd39c..0a70f89 100644 --- a/scripts/setup_build_env.py +++ b/scripts/setup_build_env.py @@ -48,7 +48,7 @@ args = parser.parse_args() # update rust installation if args.update: - print("\n[ UPDATING RUST ]\n") + print("[ UPDATING RUST ]") rustup_path = os.path.join(USER_PATH, ".cargo/bin/rustup") if not os.path.exists(rustup_path): print("rustup installation not found") @@ -58,7 +58,7 @@ if args.update: else: # initialize debian package build environment from a blank slate # (but this code is idempotent so it can be re-run if already initialized) - print("\n[ INSTALLING SYSTEM REQUIREMENTS ]\n") + print("[ INSTALLING SYSTEM REQUIREMENTS ]") subprocess.call(["sudo", "apt-get", "install", @@ -69,13 +69,13 @@ else: "gcc-aarch64-linux-gnu", ]) - print("\n[ CREATING DIRECTORIES ]\n") + print("[ CREATING DIRECTORIES ]") folders = [MICROSERVICES_SRC_DIR, FREIGHT_CACHE, FREIGHT_LIB] for folder in folders: if not os.path.exists(folder): os.makedirs(folder) - print("\n[ INSTALLING RUST ]\n") + print("[ INSTALLING RUST ]") rustc_path = os.path.join(USER_PATH, ".cargo/bin/rustc") if not os.path.exists(rustc_path): first_command = subprocess.Popen( @@ -84,25 +84,25 @@ else: ["sh", "-s", "--", "-y"], stdin=first_command.stdout) first_command.wait() - print("\n[ INSTALLING CARGO-DEB ]\n") + print("[ INSTALLING CARGO-DEB ]") cargo_deb_path = os.path.join(USER_PATH, ".cargo/bin/cargo-deb") if not os.path.exists(cargo_deb_path): subprocess.call([cargo_path, "install", "cargo-deb"]) - print("\n[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]\n") + print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") rustup_path = os.path.join(USER_PATH, ".cargo/bin/rustup") subprocess.call([rustup_path, "target", "add", "aarch64-unknown-linux-gnu"]) subprocess.call([rustup_path, "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) - print("\n[ INSTALLING FREIGHT ]\n") + print("[ INSTALLING FREIGHT ]") freight_path = os.path.join(AUTOMATION_DIR, "freight") if not os.path.exists(freight_path): subprocess.call( ["git", "clone", "https://github.com/freight-team/freight.git", freight_path]) - print("\n[ CONFIGURING FREIGHT ]\n") + print("[ CONFIGURING FREIGHT ]") freight_conf_tmp_path = os.path.join(USER_PATH, "freight.conf") render_template( src="freight.conf", @@ -115,7 +115,7 @@ else: ) subprocess.call(["sudo", "cp", freight_conf_tmp_path, FREIGHT_CONF]) - print("\n[ PULLING MICROSERVICES CODE FROM GITHUB ]\n") + print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") for service in SERVICES: name = service["name"] repo_url = service["repo_url"] @@ -123,13 +123,13 @@ else: if not os.path.exists(service_path): subprocess.call(["git", "clone", repo_url, service_path]) - print("\n[ EXPORTING PUBLIC GPG KEY ]\n") + print("[ EXPORTING PUBLIC GPG KEY ]") output_path = "{}/peach_pub.gpg".format(FREIGHT_CACHE) if not os.path.exists(output_path): subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_EMAIL]) - print("\n[ COPYING NGINX CONFIG ]\n") + print("[ COPYING NGINX CONFIG ]") nginx_conf_tmp_path = os.path.join(USER_PATH, "apt.peachcloud.org") render_template( src="nginx_debian.conf", @@ -141,4 +141,4 @@ else: subprocess.call(["sudo", "cp", nginx_conf_tmp_path, "/etc/nginx/sites-enabled/apt.peachcloud.org"]) - print("\n[ DEBIAN PACKAGE BUILD ENVIRONMENT SETUP COMPLETE ]") + print("[ DEBIAN PACKAGE BUILD ENVIRONMENT SETUP COMPLETE ]") From a0e0c5a3223af3f3e6d658f75a06bbe9f89c6963 Mon Sep 17 00:00:00 2001 From: notplants Date: Mon, 21 Dec 2020 18:34:11 +0100 Subject: [PATCH 4/5] Update readme to reflect new url structure for apt repoa --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dc660e3..a0e74e4 100644 --- a/README.md +++ b/README.md @@ -76,13 +76,13 @@ vi /etc/apt/sources.list.d/peach.list Append the following line: ``` -deb http://apt.peachcloud.org/debian/ buster main +deb http://apt.peachcloud.org/ buster main ``` Add the gpg pub key to the apt-key list: ``` -wget -O - http://apt.peachcloud.org/peach_pub.gpg | sudo apt-key add - +wget -O - http://apt.peachcloud.org/pubkey.gpg | sudo apt-key add - ``` You can then install peach packages with apt: From 84b78fa9e71656196baa5ef829294bbdb2f69ca7 Mon Sep 17 00:00:00 2001 From: mycognosist Date: Mon, 28 Dec 2020 09:07:35 +0000 Subject: [PATCH 5/5] Update GPG key name for APT repo --- scripts/setup_build_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup_build_env.py b/scripts/setup_build_env.py index 0a70f89..4c47ed2 100644 --- a/scripts/setup_build_env.py +++ b/scripts/setup_build_env.py @@ -124,7 +124,7 @@ else: subprocess.call(["git", "clone", repo_url, service_path]) print("[ EXPORTING PUBLIC GPG KEY ]") - output_path = "{}/peach_pub.gpg".format(FREIGHT_CACHE) + output_path = "{}/pubkey.gpg".format(FREIGHT_CACHE) if not os.path.exists(output_path): subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_EMAIL])