Merge pull request #3 from peachcloud/freight_refactor

Freight refactor for Debian archiving

README.md

@@ -1,50 +1,102 @@
-# peach-vps config
+# peach-vps
 
-Scripts for configuring the peachcloud vps for various hosting and automation.
+![Generic badge](https://img.shields.io/badge/version-0.2.1-<COLOR>.svg)
+
+Scripts for configuring the PeachCloud VPS for various hosting and automation functions.
 
 Currently:
-- debian repository of microservices
 
+ - Debian repository of microservices (using [Freight](https://github.com/freight-team/freight))
+
+## Setup Debian Repo
+
+`scripts/setup_debian_repo.py`
+
+An idempotent script for initializing the Debian repo on the VPS.
+
+The script currently performs the following actions:
+
+ - Installs system requirements
+ - Creates directories for microservices and package archive
+ - Installs Rust
+ - Installs `cargo deb`
+ - Installs Rust aarch64 toolchain for cross-compilation
+ - Installs Freight for package archive creation and management
+ - Configures Freight
+ - Pulls microservices code from GitHub repos
+ - Exports the public GPG key
+ - Configures nginx
+ - Builds and updates microservice packages
+ - Adds packages to Freight library
+ - Adds packages to Freight cache
+
+Prior to executing the script for the first time, run the following commands on the target system:
 
-# setup debian repo
-an idempotent script for initializing the debian repo on the vps
 ```
-apt update
-apt install git python python3-pip rsync
+sudo apt update
+sudo apt install git python python3-pip rsync
 git clone https://github.com/peachcloud/peach-vps.git
 cd peach-vps
 pip3 install -r requirements.txt
-python3 scripts/setup_debian_repo.py -i
 ```
 
+Open `scripts/setup_debian_repo.py` and set the following constants:
+
+ - USER_PATH
+ - GPG_KEY_EMAIL
+ - GPG_KEY_PASS_FILE
+
+Then execute the script with the `-i` flag to run the full system initialization process (_note: several commands executed by the script require `sudo` permissions. You will be prompted for the user password during the execution of the scipt._):
+
+```
+python3 -u scripts/setup_debian_repo.py -i
+```
+
+## Update Debian Repo
+
+Without the -i flag, the `setup_debian_repo.py` script rebuilds all
+microservices (cross-compiled to arm64) and updates the Debian repo
 
-# update debian repo
-without the -i flag, the setup_debian_repo rebuilds all
-microservices (cross-compiled to arm64) and re-adds them to the debian repo
 ```
 cd peach-vps
-python3 scripts/setup_debian_repo.py
+python3 -u scripts/setup_debian_repo.py
 ```
 
+## Install from Debian Repo
+
+To add the PeachCloud Debian repo as an apt source, run the following commands from your Pi:
 
-# using the debian repo on the pi
-To add the peachcloud debian repo as an apt source,
-on the pi,
 ```
 vi /etc/apt/sources.list.d/peach.list
 ```
-and add the following line:
+
+Append the following line:
+
 ```
 deb http://apt.peachcloud.org/debian/ buster main
 ```
 
-Then add the gpg pub key to the apt-key list:
+Add the gpg pub key to the apt-key list:
+
 ```
 wget -O - http://apt.peachcloud.org/peach_pub.gpg | sudo apt-key add -
 ```
 
-You can then install peach packages with apt-get:
+You can then install peach packages with apt:
+
 ```
-apt-get update
-apt-get install peach-oled
-```
+sudo apt update
+sudo apt install peach-oled
+```
+
+By default, the latest version of the package will be downloaded and installed.
+
+Specific versions of packages can be selected for installation by supplying the semantic versioning number (this is useful for downgrading):
+
+```
+sudo apt install peach-network=0.2.0
+```
+
+## Licensing
+
+AGPL-3.0

conf/templates/debian_repo/distributions

@@ -1,8 +0,0 @@
-Origin: PeachCloud
-Label: PeachCloud
-Codename: buster
-Architectures: amd64 arm64
-Components: main
-Description: Apt repository for PeachCloud debian packages
-SignWith: {{gpg_key_id}}
-DebOverride: override.buster

conf/templates/debian_repo/options

@@ -1,3 +0,0 @@
-verbose
-basedir {{debian_rep_dir}}
-ask-passphrase

conf/templates/debian_repo/override.buster

@@ -1,4 +0,0 @@
-{% for service in services %}
-{{service}} Priority        optional
-{{service}} Section         net
-{% endfor %}

conf/templates/freight.conf

@@ -0,0 +1,26 @@
+# Freight configuration.
+
+# Directories for the Freight library and Freight cache.  Your web
+# server's document root should be `$VARCACHE`.
+VARLIB="{{freight_lib_path}}"
+VARCACHE="{{freight_cache_path}}"
+
+# Default `Origin` and `Label` fields for `Release` files.
+ORIGIN="apt.peachcloud.org"
+LABEL="PeachCloud"
+
+# Architectures supported by the package repository.
+ARCHS="arm64"
+
+# Cache the control files after each run (on), or regenerate them every
+# time (off).
+CACHE="off"
+
+# GPG key to use to sign repositories.  This is required by the `apt`
+# repository provider.  Use `gpg --gen-key` (see `gpg`(1) for more
+# details) to generate a key and put its email address here.
+GPG="{{gpg_key_email}}"
+
+# Whether to follow symbolic links in `$VARLIB` to produce extra components
+# in the cache directory (on) or not (off).
+SYMLINKS="off"

scripts/setup_debian_repo.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 from utils import render_template
 
 import subprocess
@@ -5,57 +7,109 @@ import os
 import argparse
 
 
-# constants
-MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices"
-WEB_DIR = "/var/www/"
-APT_DIR = "/var/www/apt.peachcloud.org"
-DEBIAN_REPO_DIR = "/var/www/apt.peachcloud.org/debian"
-DEBIAN_REPO_CONF_DIR = "/var/www/apt.peachcloud.org/debian/conf"
+# before running this script run `gpg --gen-key` on the server
+# assign the email address of the key id here:
+GPG_KEY_EMAIL = "andrew@mycelial.technology"
+# save the key passphrase to file and assign the path here:
+# (ensure the file is only readable by the user running freight)
+GPG_KEY_PASS_FILE = "/home/rust/passphrase.txt"
+# if you need to list the existing keys: `gpg --list-keys`
+
+
+# constants
+AUTOMATION_DIR = "/srv/peachcloud/automation"
+FREIGHT_CONF = "/etc/freight.conf"
+FREIGHT_LIB = "/var/lib/freight"
+FREIGHT_CACHE = "/var/www/apt.peachcloud.org"
+MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices"
+MICROSERVICES_DEB_DIR = "/srv/peachcloud/debs"
+USER_PATH = "/home/rust"
 
-# before running this script run `gpg --gen-key` on the server, and put the key id here
-# `gpg --list-keys`
-GPG_KEY_ID = "4ACEF251EA3E091167E8F03EBF69A52BE3565476"
 
 SERVICES = [
+    {"name": "peach-buttons",
+     "repo_url": "https://github.com/peachcloud/peach-buttons.git"},
+    {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"},
+    {"name": "peach-monitor",
+     "repo_url": "https://github.com/peachcloud/peach-monitor.git"},
+    {"name": "peach-network",
+     "repo_url": "https://github.com/peachcloud/peach-network.git"},
     {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"},
-    {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"},
     {"name": "peach-stats", "repo_url": "https://github.com/peachcloud/peach-stats.git"},
     # {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear
-    {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"},
-    {"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"}
 ]
 
 # parse CLI args
 parser = argparse.ArgumentParser()
-parser.add_argument("-i", "--initialize", help="initialize and update debian repo", action="store_true")
+parser.add_argument(
+    "-i",
+    "--initialize",
+    help="initialize and update debian repo",
+    action="store_true")
 args = parser.parse_args()
 
+cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo")
+
 # initializing debian repo from a blank slate
 # (but this code is idempotent so it can be re-run if already initialized)
 if args.initialize:
 
     print("[ INSTALLING SYSTEM REQUIREMENTS ]")
-    subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "reprepro", "gcc-aarch64-linux-gnu", ])
+    subprocess.call(["sudo",
+                     "apt-get",
+                     "install",
+                     "git",
+                     "nginx",
+                     "curl",
+                     "build-essential",
+                     "gcc-aarch64-linux-gnu",
+                     ])
 
     print("[ CREATING DIRECTORIES ]")
-    folders = [MICROSERVICES_SRC_DIR, WEB_DIR, APT_DIR, DEBIAN_REPO_DIR, DEBIAN_REPO_CONF_DIR]
+    folders = [MICROSERVICES_SRC_DIR, FREIGHT_CACHE, FREIGHT_LIB]
     for folder in folders:
         if not os.path.exists(folder):
             os.makedirs(folder)
 
     print("[ INSTALLING RUST ]")
-    if not os.path.exists("/root/.cargo/bin/rustc"):
-        first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE)
-        output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout)
+    rustc_path = os.path.join(USER_PATH, ".cargo/bin/rustc")
+    if not os.path.exists(rustc_path):
+        first_command = subprocess.Popen(
+            ["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE)
+        output = subprocess.check_output(
+            ["sh", "-s", "--", "-y"], stdin=first_command.stdout)
         first_command.wait()
 
     print("[ INSTALLING CARGO-DEB ]")
-    if not os.path.exists("/root/.cargo/bin/cargo-deb"):
-        subprocess.call(["/root/.cargo/bin/cargo", "install", "cargo-deb"])
+    cargo_deb_path = os.path.join(USER_PATH, ".cargo/bin/cargo-deb")
+    if not os.path.exists(cargo_deb_path):
+        subprocess.call([cargo_path, "install", "cargo-deb"])
 
     print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]")
-    subprocess.call(["/root/.cargo/bin/rustup", "target", "add", "aarch64-unknown-linux-gnu"])
-    subprocess.call(["/root/.cargo/bin/rustup", "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"])
+    rustup_path = os.path.join(USER_PATH, ".cargo/bin/rustup")
+    subprocess.call([rustup_path, "target", "add",
+                     "aarch64-unknown-linux-gnu"])
+    subprocess.call([rustup_path, "toolchain", "install",
+                     "nightly-aarch64-unknown-linux-gnu"])
+
+    print("[ INSTALLING FREIGHT ]")
+    freight_path = os.path.join(AUTOMATION_DIR, "freight")
+    if not os.path.exists(freight_path):
+        subprocess.call(
+            ["git", "clone", "https://github.com/freight-team/freight.git", freight_path])
+
+    print("[ CONFIGURING FREIGHT ]")
+    freight_conf_tmp_path = os.path.join(USER_PATH, "freight.conf")
+    render_template(
+        src="freight.conf",
+        dest=freight_conf_tmp_path,
+        template_vars={
+            "freight_lib_path": FREIGHT_LIB,
+            "freight_cache_path": FREIGHT_CACHE,
+            "gpg_key_email": GPG_KEY_EMAIL
+        }
+    )
+    subprocess.call(["sudo", "cp", freight_conf_tmp_path, FREIGHT_CONF])
 
     print("[ PULLING MICROSERVICES CODE FROM GITHUB ]")
     for service in SERVICES:
@@ -65,59 +119,51 @@ if args.initialize:
         if not os.path.exists(service_path):
             subprocess.call(["git", "clone", repo_url, service_path])
 
-    print("[ COPYING DEBIAN REPO CONFIG ]")
-    render_template(
-        src="debian_repo/distributions",
-        dest="{}/distributions".format(DEBIAN_REPO_CONF_DIR),
-        template_vars={
-            "gpg_key_id": GPG_KEY_ID
-        }
-    )
-    render_template(
-        src="debian_repo/options",
-        dest="{}/options".format(DEBIAN_REPO_CONF_DIR),
-        template_vars={
-            "debian_rep_dir": DEBIAN_REPO_DIR
-        }
-    )
-    render_template(
-        src="debian_repo/override.buster",
-        dest="{}/override.buster".format(DEBIAN_REPO_CONF_DIR),
-        template_vars={
-            "services": [service["name"] for service in SERVICES]
-        }
-    )
-
     print("[ EXPORTING PUBLIC GPG KEY ]")
-    output_path = "{}/peach_pub.gpg".format(APT_DIR)
+    output_path = "{}/peach_pub.gpg".format(FREIGHT_CACHE)
     if not os.path.exists(output_path):
-        subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID])
+        subprocess.call(["gpg", "--armor", "--output",
+                         output_path, "--export", GPG_KEY_EMAIL])
 
     print("[ COPYING NGINX CONFIG ]")
+    nginx_conf_tmp_path = os.path.join(USER_PATH, "apt.peachcloud.org")
     render_template(
-        src="debian_repo/nginx_debian.conf",
-        dest="/etc/nginx/sites-enabled/apt.peachcloud.org",
-        template_vars = {
-            "apt_dir": APT_DIR
+        src="nginx_debian.conf",
+        dest=nginx_conf_tmp_path,
+        template_vars={
+            "apt_dir": FREIGHT_CACHE
         }
     )
+    subprocess.call(["sudo", "cp", nginx_conf_tmp_path,
+                     "/etc/nginx/sites-enabled/apt.peachcloud.org"])
 
-
-# below is code for git updating the microservices, building the microservices,
-# and (re)-adding them to the debian repo
+# update the microservices from git and build the debian packages
 print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]")
 for service in SERVICES:
     service_name = service["name"]
     service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name)
     print("[ BUILIDING SERVICE {} ]".format(service_name))
     subprocess.call(["git", "pull"], cwd=service_path)
-    debian_package_path = subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip()
-    # remove debian package from repo
-    # (in the future we could look at some way of updating with versions instead of removing and adding)
-    subprocess.call(["reprepro", "remove", "buster", service_name], cwd=DEBIAN_REPO_DIR)
-    # add the package
-    subprocess.call(["reprepro", "includedeb", "buster", debian_package_path], cwd=DEBIAN_REPO_DIR)
+    debian_package_path = subprocess.run(
+        [
+            cargo_path,
+            "deb",
+            "--target",
+            "aarch64-unknown-linux-gnu"],
+        cwd=service_path,
+        stdout=subprocess.PIPE).stdout.decode("utf-8").strip()
+    subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
 
+print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]")
+for package in os.scandir(MICROSERVICES_DEB_DIR):
+    if package.name.endswith(".deb"):
+        print("[ ADDING PACKAGE {} ]".format(package.name))
+        subprocess.call(["freight", "add", "-c", FREIGHT_CONF,
+                         package.path, "apt/buster"])
+
+print("[ ADDING PACKAGES TO FREIGHT CACHE ]")
+# needs to be run as sudo user
+subprocess.call(["sudo", "freight", "cache", "-g",
+                 GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])
 
 print("[ DEBIAN REPO SETUP COMPLETE ]")
-

scripts/utils.py

@@ -25,5 +25,3 @@ def render_template(src, dest, template_vars=None):
         os.remove(dest)
     with open(dest, 'w') as f:
         f.write(output_text)
-
-