Merge pull request #2 from peachcloud/setup-debian-repo
Python script for initializing and updating apt debian repo
This commit is contained in:
commit
cc9cfda7f4
|
@ -0,0 +1,5 @@
|
|||
deploy.sh
|
||||
secret*
|
||||
secret_files*
|
||||
ssh.sh
|
||||
notes.txt
|
52
README.md
52
README.md
|
@ -1,2 +1,50 @@
|
|||
# peach-vps
|
||||
# simple-ansible-template
|
||||
# peach-vps config
|
||||
|
||||
Scripts for configuring the peachcloud vps for various hosting and automation.
|
||||
|
||||
Currently:
|
||||
- debian repository of microservices
|
||||
|
||||
|
||||
# setup debian repo
|
||||
an idempotent script for initializing the debian repo on the vps
|
||||
```
|
||||
apt update
|
||||
apt install git python python3-pip rsync
|
||||
git clone https://github.com/peachcloud/peach-vps.git
|
||||
cd peach-vps
|
||||
pip3 install -r requirements.txt
|
||||
python3 scripts/setup_debian_repo.py -i
|
||||
```
|
||||
|
||||
|
||||
# update debian repo
|
||||
without the -i flag, the setup_debian_repo rebuilds all
|
||||
microservices (cross-compiled to arm64) and re-adds them to the debian repo
|
||||
```
|
||||
cd peach-vps
|
||||
python3 scripts/setup_debian_repo.py
|
||||
```
|
||||
|
||||
|
||||
# using the debian repo on the pi
|
||||
To add the peachcloud debian repo as an apt source,
|
||||
on the pi,
|
||||
```
|
||||
vi /etc/apt/sources.list.d/peach.list
|
||||
```
|
||||
and add the following line:
|
||||
```
|
||||
deb http://apt.peachcloud.org/debian/ buster main
|
||||
```
|
||||
|
||||
Then add the gpg pub key to the apt-key list:
|
||||
```
|
||||
wget -O - http://apt.peachcloud.org/peach_pub.gpg | sudo apt-key add -
|
||||
```
|
||||
|
||||
You can then install peach packages with apt-get:
|
||||
```
|
||||
apt-get update
|
||||
apt-get install peach-oled
|
||||
```
|
|
@ -0,0 +1,8 @@
|
|||
Origin: PeachCloud
|
||||
Label: PeachCloud
|
||||
Codename: buster
|
||||
Architectures: amd64 arm64
|
||||
Components: main
|
||||
Description: Apt repository for PeachCloud debian packages
|
||||
SignWith: {{gpg_key_id}}
|
||||
DebOverride: override.buster
|
|
@ -0,0 +1,21 @@
|
|||
server {
|
||||
listen 80;
|
||||
server_name apt.peachcloud.org;
|
||||
|
||||
access_log /var/log/nginx-debian.log;
|
||||
error_log /var/log/nginx-debian.error;
|
||||
|
||||
location / {
|
||||
root {{apt_dir}};
|
||||
index index.html;
|
||||
autoindex on;
|
||||
}
|
||||
|
||||
location ~ /(.*)/conf {
|
||||
deny all;
|
||||
}
|
||||
|
||||
location ~ /(.*)/db {
|
||||
deny all;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,3 @@
|
|||
verbose
|
||||
basedir {{debian_rep_dir}}
|
||||
ask-passphrase
|
|
@ -0,0 +1,4 @@
|
|||
{% for service in services %}
|
||||
{{service}} Priority optional
|
||||
{{service}} Section net
|
||||
{% endfor %}
|
|
@ -0,0 +1 @@
|
|||
Jinja2==2.11.2
|
|
@ -0,0 +1,123 @@
|
|||
from utils import render_template
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
import argparse
|
||||
|
||||
|
||||
# constants
|
||||
MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices"
|
||||
WEB_DIR = "/var/www/"
|
||||
APT_DIR = "/var/www/apt.peachcloud.org"
|
||||
DEBIAN_REPO_DIR = "/var/www/apt.peachcloud.org/debian"
|
||||
DEBIAN_REPO_CONF_DIR = "/var/www/apt.peachcloud.org/debian/conf"
|
||||
|
||||
# before running this script run `gpg --gen-key` on the server, and put the key id here
|
||||
# `gpg --list-keys`
|
||||
GPG_KEY_ID = "4ACEF251EA3E091167E8F03EBF69A52BE3565476"
|
||||
|
||||
SERVICES = [
|
||||
{"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"},
|
||||
{"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"},
|
||||
{"name": "peach-stats", "repo_url": "https://github.com/peachcloud/peach-stats.git"},
|
||||
# {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear
|
||||
{"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"},
|
||||
{"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"}
|
||||
]
|
||||
|
||||
# parse CLI args
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-i", "--initialize", help="initialize and update debian repo", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
# initializing debian repo from a blank slate
|
||||
# (but this code is idempotent so it can be re-run if already initialized)
|
||||
if args.initialize:
|
||||
|
||||
print("[ INSTALLING SYSTEM REQUIREMENTS ]")
|
||||
subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "reprepro", "gcc-aarch64-linux-gnu", ])
|
||||
|
||||
print("[ CREATING DIRECTORIES ]")
|
||||
folders = [MICROSERVICES_SRC_DIR, WEB_DIR, APT_DIR, DEBIAN_REPO_DIR, DEBIAN_REPO_CONF_DIR]
|
||||
for folder in folders:
|
||||
if not os.path.exists(folder):
|
||||
os.makedirs(folder)
|
||||
|
||||
print("[ INSTALLING RUST ]")
|
||||
if not os.path.exists("/root/.cargo/bin/rustc"):
|
||||
first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE)
|
||||
output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout)
|
||||
first_command.wait()
|
||||
|
||||
print("[ INSTALLING CARGO-DEB ]")
|
||||
if not os.path.exists("/root/.cargo/bin/cargo-deb"):
|
||||
subprocess.call(["/root/.cargo/bin/cargo", "install", "cargo-deb"])
|
||||
|
||||
print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]")
|
||||
subprocess.call(["/root/.cargo/bin/rustup", "target", "add", "aarch64-unknown-linux-gnu"])
|
||||
subprocess.call(["/root/.cargo/bin/rustup", "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"])
|
||||
|
||||
print("[ PULLING MICROSERVICES CODE FROM GITHUB ]")
|
||||
for service in SERVICES:
|
||||
name = service["name"]
|
||||
repo_url = service["repo_url"]
|
||||
service_path = os.path.join(MICROSERVICES_SRC_DIR, name)
|
||||
if not os.path.exists(service_path):
|
||||
subprocess.call(["git", "clone", repo_url, service_path])
|
||||
|
||||
print("[ COPYING DEBIAN REPO CONFIG ]")
|
||||
render_template(
|
||||
src="debian_repo/distributions",
|
||||
dest="{}/distributions".format(DEBIAN_REPO_CONF_DIR),
|
||||
template_vars={
|
||||
"gpg_key_id": GPG_KEY_ID
|
||||
}
|
||||
)
|
||||
render_template(
|
||||
src="debian_repo/options",
|
||||
dest="{}/options".format(DEBIAN_REPO_CONF_DIR),
|
||||
template_vars={
|
||||
"debian_rep_dir": DEBIAN_REPO_DIR
|
||||
}
|
||||
)
|
||||
render_template(
|
||||
src="debian_repo/override.buster",
|
||||
dest="{}/override.buster".format(DEBIAN_REPO_CONF_DIR),
|
||||
template_vars={
|
||||
"services": [service["name"] for service in SERVICES]
|
||||
}
|
||||
)
|
||||
|
||||
print("[ EXPORTING PUBLIC GPG KEY ]")
|
||||
output_path = "{}/peach_pub.gpg".format(APT_DIR)
|
||||
if not os.path.exists(output_path):
|
||||
subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID])
|
||||
|
||||
print("[ COPYING NGINX CONFIG ]")
|
||||
render_template(
|
||||
src="debian_repo/nginx_debian.conf",
|
||||
dest="/etc/nginx/sites-enabled/apt.peachcloud.org",
|
||||
template_vars = {
|
||||
"apt_dir": APT_DIR
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# below is code for git updating the microservices, building the microservices,
|
||||
# and (re)-adding them to the debian repo
|
||||
print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]")
|
||||
for service in SERVICES:
|
||||
service_name = service["name"]
|
||||
service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name)
|
||||
print("[ BUILIDING SERVICE {} ]".format(service_name))
|
||||
subprocess.call(["git", "pull"], cwd=service_path)
|
||||
debian_package_path = subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip()
|
||||
# remove debian package from repo
|
||||
# (in the future we could look at some way of updating with versions instead of removing and adding)
|
||||
subprocess.call(["reprepro", "remove", "buster", service_name], cwd=DEBIAN_REPO_DIR)
|
||||
# add the package
|
||||
subprocess.call(["reprepro", "includedeb", "buster", debian_package_path], cwd=DEBIAN_REPO_DIR)
|
||||
|
||||
|
||||
print("[ DEBIAN REPO SETUP COMPLETE ]")
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
import os
|
||||
import jinja2
|
||||
import subprocess
|
||||
|
||||
PROJECT_PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
|
||||
print('PROJECT_PATH: {}'.format(PROJECT_PATH))
|
||||
|
||||
template_path = os.path.join(PROJECT_PATH, 'conf/templates')
|
||||
template_loader = jinja2.FileSystemLoader(searchpath=template_path)
|
||||
template_env = jinja2.Environment(loader=template_loader, keep_trailing_newline=True)
|
||||
|
||||
|
||||
def render_template(src, dest, template_vars=None):
|
||||
"""
|
||||
:param src: relative string path to jinja template file
|
||||
:param dest: absolute string path of output destination file
|
||||
:param template_vars: variables to render template with
|
||||
:return: None
|
||||
"""
|
||||
template = template_env.get_template(src)
|
||||
if not template_vars:
|
||||
template_vars= {}
|
||||
output_text = template.render(**template_vars)
|
||||
if os.path.exists(dest):
|
||||
os.remove(dest)
|
||||
with open(dest, 'w') as f:
|
||||
f.write(output_text)
|
||||
|
||||
|
Loading…
Reference in New Issue