From 4fc6d8ae8efe78d0531b275370abeb7fb591faef Mon Sep 17 00:00:00 2001 From: mycognosist Date: Mon, 30 Nov 2020 13:39:04 +0000 Subject: [PATCH] Update script to use Freight instead of reprepro --- scripts/setup_debian_repo.py | 112 ++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 7644887..658c423 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -6,23 +6,31 @@ import argparse # constants +AUTOMATION_DIR = "/srv/peachcloud/automation" MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" -WEB_DIR = "/var/www/" -APT_DIR = "/var/www/apt.peachcloud.org" -DEBIAN_REPO_DIR = "/var/www/apt.peachcloud.org/debian" -DEBIAN_REPO_CONF_DIR = "/var/www/apt.peachcloud.org/debian/conf" +MICROSERVICES_DEB_DIR = "/srv/peachcloud/debs" +FREIGHT_CONF = "/etc/freight.conf" +FREIGHT_LIB = "/var/lib/freight" +FREIGHT_CACHE = "/var/www/apt.peachcloud.org" +# define user path before running the script +USER_PATH = "/home/rust" -# before running this script run `gpg --gen-key` on the server, and put the key id here -# `gpg --list-keys` -GPG_KEY_ID = "4ACEF251EA3E091167E8F03EBF69A52BE3565476" +# before running this script run `gpg --gen-key` on the server +# assign the email address of the key id here: +GPG_KEY_EMAIL = "andrew@mycelial.technology" +# save the key passphrase to file and assign the path here: +# (ensure the file is only readable by the user running freight) +GPG_KEY_PASS_FILE = "/home/rust/passphrase.txt" +# if you need to list the existing keys: `gpg --list-keys` SERVICES = [ - {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, + {"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"}, + {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, + {"name": "peach-monitor", "repo_url": "https://github.com/peachcloud/peach-monitor.git"}, {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"}, + {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, {"name": "peach-stats", "repo_url": "https://github.com/peachcloud/peach-stats.git"}, # {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear - {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, - {"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"} ] # parse CLI args @@ -35,27 +43,47 @@ args = parser.parse_args() if args.initialize: print("[ INSTALLING SYSTEM REQUIREMENTS ]") - subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "reprepro", "gcc-aarch64-linux-gnu", ]) + subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "gcc-aarch64-linux-gnu", ]) print("[ CREATING DIRECTORIES ]") - folders = [MICROSERVICES_SRC_DIR, WEB_DIR, APT_DIR, DEBIAN_REPO_DIR, DEBIAN_REPO_CONF_DIR] + folders = [MICROSERVICES_SRC_DIR, FREIGHT_CACHE, FREIGHT_LIB] for folder in folders: if not os.path.exists(folder): os.makedirs(folder) print("[ INSTALLING RUST ]") - if not os.path.exists("/root/.cargo/bin/rustc"): + rustc_path = os.path.join(USER_PATH, ".cargo/bin/rustc") + if not os.path.exists(rustc_path): first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) first_command.wait() print("[ INSTALLING CARGO-DEB ]") - if not os.path.exists("/root/.cargo/bin/cargo-deb"): - subprocess.call(["/root/.cargo/bin/cargo", "install", "cargo-deb"]) + cargo_path = os.path.join(USER_PATH, ".cargo/bin/cargo") + cargo_deb_path = os.path.join(USER_PATH, ".cargo/bin/cargo-deb") + if not os.path.exists(cargo_deb_path): + subprocess.call([cargo_path, "install", "cargo-deb"]) print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") - subprocess.call(["/root/.cargo/bin/rustup", "target", "add", "aarch64-unknown-linux-gnu"]) - subprocess.call(["/root/.cargo/bin/rustup", "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) + rustup_path = os.path.join(USER_PATH, ".cargo/bin/rustup") + subprocess.call([rustup_path, "target", "add", "aarch64-unknown-linux-gnu"]) + subprocess.call([rustup_path, "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) + + print("[ INSTALLING FREIGHT ]") + freight_path = os.path.join(AUTOMATION_DIR, "freight") + if not os.path.exists(freight_path): + subprocess.call(["git", "clone", "https://github.com/freight-team/freight.git", freight_path]) + + print("[ CONFIGURING FREIGHT ]") + render_template( + src="debian_repo/freight.conf", + dest=FREIGHT_CONF, + template_vars={ + "freight_lib_path": FREIGHT_LIB, + "freight_cache_path": FREIGHT_CACHE, + "gpg_key_email": GPG_KEY_EMAIL + } + ) print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") for service in SERVICES: @@ -65,59 +93,37 @@ if args.initialize: if not os.path.exists(service_path): subprocess.call(["git", "clone", repo_url, service_path]) - print("[ COPYING DEBIAN REPO CONFIG ]") - render_template( - src="debian_repo/distributions", - dest="{}/distributions".format(DEBIAN_REPO_CONF_DIR), - template_vars={ - "gpg_key_id": GPG_KEY_ID - } - ) - render_template( - src="debian_repo/options", - dest="{}/options".format(DEBIAN_REPO_CONF_DIR), - template_vars={ - "debian_rep_dir": DEBIAN_REPO_DIR - } - ) - render_template( - src="debian_repo/override.buster", - dest="{}/override.buster".format(DEBIAN_REPO_CONF_DIR), - template_vars={ - "services": [service["name"] for service in SERVICES] - } - ) - print("[ EXPORTING PUBLIC GPG KEY ]") - output_path = "{}/peach_pub.gpg".format(APT_DIR) + output_path = "{}/peach_pub.gpg".format(FREIGHT_CACHE) if not os.path.exists(output_path): - subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID]) + subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_EMAIL]) print("[ COPYING NGINX CONFIG ]") render_template( src="debian_repo/nginx_debian.conf", dest="/etc/nginx/sites-enabled/apt.peachcloud.org", template_vars = { - "apt_dir": APT_DIR + "apt_dir": FREIGHT_CACHE } ) - -# below is code for git updating the microservices, building the microservices, -# and (re)-adding them to the debian repo +# update the microservices from git and build the debian packages print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") for service in SERVICES: service_name = service["name"] service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) print("[ BUILIDING SERVICE {} ]".format(service_name)) subprocess.call(["git", "pull"], cwd=service_path) - debian_package_path = subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip() - # remove debian package from repo - # (in the future we could look at some way of updating with versions instead of removing and adding) - subprocess.call(["reprepro", "remove", "buster", service_name], cwd=DEBIAN_REPO_DIR) - # add the package - subprocess.call(["reprepro", "includedeb", "buster", debian_package_path], cwd=DEBIAN_REPO_DIR) + debian_package_path = subprocess.check_output([cargo_path, "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip() + # copy package to staging folder + subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) +print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") +for package in MICROSERVICES_DEB_DIR: + subprocess.call(["freight", "add", package, "apt/buster"]) + +print("[ ADDING PACKAGES TO FREIGHT CACHE ]") +# needs to be run as sudo user +subprocess.call(["sudo", "freight", "cache", "-g", GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) print("[ DEBIAN REPO SETUP COMPLETE ]") -