From 9ae0c66f7336803c0bdfc9b35bcc4e68b3b1f013 Mon Sep 17 00:00:00 2001 From: Max Fowler Date: Tue, 10 Nov 2020 16:03:45 +0100 Subject: [PATCH 1/8] Python script version --- conf/templates/debian/distributions | 8 +++++ conf/templates/debian/options | 3 ++ conf/templates/debian/override.buster | 4 +++ conf/templates/devdocs/git-post-receive | 13 +++++++++ conf/templates/nginx/nginx.conf | 31 ++++++++++++++++++++ conf/templates/nginx/nginx_devdocs.conf | 10 +++++++ peach_vps_scripts/__init__.py | 0 peach_vps_scripts/setup_vps.py | 39 +++++++++++++++++++++++++ peach_vps_scripts/utils.py | 32 ++++++++++++++++++++ peach_vps_scripts/vars.py | 11 +++++++ requirements.txt | 1 + 11 files changed, 152 insertions(+) create mode 100644 conf/templates/debian/distributions create mode 100644 conf/templates/debian/options create mode 100644 conf/templates/debian/override.buster create mode 100644 conf/templates/devdocs/git-post-receive create mode 100644 conf/templates/nginx/nginx.conf create mode 100644 conf/templates/nginx/nginx_devdocs.conf create mode 100644 peach_vps_scripts/__init__.py create mode 100644 peach_vps_scripts/setup_vps.py create mode 100644 peach_vps_scripts/utils.py create mode 100644 peach_vps_scripts/vars.py create mode 100644 requirements.txt diff --git a/conf/templates/debian/distributions b/conf/templates/debian/distributions new file mode 100644 index 0000000..f6fed0f --- /dev/null +++ b/conf/templates/debian/distributions @@ -0,0 +1,8 @@ +Origin: PeachCloud +Label: PeachCloud +Codename: buster +Architectures: amd64 +Components: main +Description: Apt repository for PeachCloud debian packages +SignWith: {{gpg_key_id}} +DebOverride: override.buster diff --git a/conf/templates/debian/options b/conf/templates/debian/options new file mode 100644 index 0000000..5dd7345 --- /dev/null +++ b/conf/templates/debian/options @@ -0,0 +1,3 @@ +verbose +basedir {{debian_rep_dir}} +ask-passphrase diff --git a/conf/templates/debian/override.buster b/conf/templates/debian/override.buster new file mode 100644 index 0000000..ec9fb2d --- /dev/null +++ b/conf/templates/debian/override.buster @@ -0,0 +1,4 @@ +{% for service in services %} +{{service}} Priority optional +{{service}} Section net +{% endfor %} \ No newline at end of file diff --git a/conf/templates/devdocs/git-post-receive b/conf/templates/devdocs/git-post-receive new file mode 100644 index 0000000..36b37bd --- /dev/null +++ b/conf/templates/devdocs/git-post-receive @@ -0,0 +1,13 @@ +#!/bin/bash +while read oldrev newrev ref +do + if [[ $ref =~ .*/master$ ]]; + then + echo "Master ref received. Deploying master branch to build directory..." + git --work-tree={{src_dir}}/devdocs_build --git-dir={{src_dir}}/devdocs_bare checkout -f + echo "Building docs and deploying to production..." + /root/.cargo/bin/mdbook build {{src_dir}}/devdocs_build --dest-dir {{web_dir}}/docs:peachcloud:org/html + else + echo "Ref $ref successfully received. Doing nothing: only the master branch may be deployed on this server." + fi +done \ No newline at end of file diff --git a/conf/templates/nginx/nginx.conf b/conf/templates/nginx/nginx.conf new file mode 100644 index 0000000..668b54c --- /dev/null +++ b/conf/templates/nginx/nginx.conf @@ -0,0 +1,31 @@ +user www-data; +worker_processes 1; +worker_rlimit_nofile 8192; + +events { + worker_connections 3000; +} + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} \ No newline at end of file diff --git a/conf/templates/nginx/nginx_devdocs.conf b/conf/templates/nginx/nginx_devdocs.conf new file mode 100644 index 0000000..c97291c --- /dev/null +++ b/conf/templates/nginx/nginx_devdocs.conf @@ -0,0 +1,10 @@ +server { + listen 80; + server_name 159.89.5.141; + + location / { + root {{web_dir}}/docs:peachcloud:org/html; + index index.html; + try_files $uri $uri/ /index.html; + } +} \ No newline at end of file diff --git a/peach_vps_scripts/__init__.py b/peach_vps_scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/peach_vps_scripts/setup_vps.py b/peach_vps_scripts/setup_vps.py new file mode 100644 index 0000000..4001f1d --- /dev/null +++ b/peach_vps_scripts/setup_vps.py @@ -0,0 +1,39 @@ +from peach_vps_scripts.utils import render_template, cargo_install +from peach_vps_scripts.vars import VARS + +import subprocess +import os + + +print("[ UPDATING OPERATING SYSTEM ]") +subprocess.call(["apt-get", "update", "-y"]) +subprocess.call(["apt-get", "upgrade", "-y"]) + +print("[ INSTALLING SYSTEM REQUIREMENTS ]") +subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "mosh"]) + +print("[ CREATING SYSTEM GROUPS ]") +subprocess.call(["/usr/sbin/groupadd", "peach"]) + +print("[ ADDING SYSTEM USER ]") +users = ["notplants", "glyph"] +for user in users: + subprocess.call(["/usr/sbin/adduser", user]) + subprocess.call(["usermod", "-aG", "sudo", user]) + subprocess.call(["/usr/sbin/usermod", "-a", "-G", user, "peach"]) + +print("[ CREATING DIRECTORIES ]") +folders = [VARS["src_dir"], VARS["www_dir"], VARS["debian_repo_dir"]] +for folder in folders: + if not os.path.exists(folder): + os.makedirs(folder) + +print("[ INSTALLING RUST ]") +subprocess.call(["curl", "https://sh.rustup.rs", "-sSf", "|", "sh", "-s", "--", "-y"]) + +print("[ INSTALLING CARGO PACKAGES ]") +cargo_install("cargo-deb") + +render_template(src='nginx/nginx.conf', dest='/etc/nginx/nginx.conf') + + diff --git a/peach_vps_scripts/utils.py b/peach_vps_scripts/utils.py new file mode 100644 index 0000000..8533937 --- /dev/null +++ b/peach_vps_scripts/utils.py @@ -0,0 +1,32 @@ +from peach_vps_scripts.vars import VARS + +import os +import jinja2 +import subprocess + +PROJECT_PATH = os.path.abspath(os.path.dirname(__file__)) + +template_path = os.path.join(PROJECT_PATH, 'templates') +template_loader = jinja2.FileSystemLoader(searchpath=template_path) +template_env = jinja2.Environment(loader=template_loader) + + +def render_template(src, dest, template_vars=None): + """ + :param src: relative string path to jinja template file + :param dest: absolute string path of output destination file + :param template_vars: variables to render template with + :return: None + """ + template = template_env.get_template(src) + template_vars.update(VARS) + output_text = template.render(template_vars=template_vars) + if os.path.exists(dest): + os.remove(dest) + with open(dest, 'w') as f: + f.write(output_text) + + +def cargo_install(package): + subprocess.call(['/root/.cargo/bin/cargo', 'install', package]) + diff --git a/peach_vps_scripts/vars.py b/peach_vps_scripts/vars.py new file mode 100644 index 0000000..8b66352 --- /dev/null +++ b/peach_vps_scripts/vars.py @@ -0,0 +1,11 @@ +VARS = { + 'log_dir': '/srv/log', + 'src_dir': '/srv/src', + 'web_dir': 'srv/www', + 'debian_rep_dir': '/srv/www/repos/apt/debian', + 'gpg_key_id': 'E62CD13A85763FCEC3EDBA8EA98440817F1A3CE5', + 'services': [ + {'name': 'peach-oled', 'repo_url': 'https://github.com/peachcloud/peach-oled.git'}, + {'name': 'peach-oled', 'repo_url': 'https://github.com/peachcloud/peach-oled.git'} + ] +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..758129b --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Jinja2==2.11.2 From f69fea4256ea8067201211ccca819a4632651b6b Mon Sep 17 00:00:00 2001 From: Max Fowler Date: Tue, 10 Nov 2020 17:14:21 +0100 Subject: [PATCH 2/8] Working on python version --- .gitignore | 4 ++ README.md | 26 +++++++++- peach_vps_scripts/setup_vps.py | 39 --------------- {peach_vps_scripts => scripts}/__init__.py | 0 scripts/setup_vps.py | 55 ++++++++++++++++++++++ {peach_vps_scripts => scripts}/utils.py | 12 +++-- {peach_vps_scripts => scripts}/vars.py | 0 7 files changed, 91 insertions(+), 45 deletions(-) create mode 100644 .gitignore delete mode 100644 peach_vps_scripts/setup_vps.py rename {peach_vps_scripts => scripts}/__init__.py (100%) create mode 100644 scripts/setup_vps.py rename {peach_vps_scripts => scripts}/utils.py (71%) rename {peach_vps_scripts => scripts}/vars.py (100%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5dacb78 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +deploy.sh +secret* +secret_files* +ssh.sh \ No newline at end of file diff --git a/README.md b/README.md index 7828353..8e91bd1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,24 @@ -# peach-vps -# simple-ansible-template +# peach-vps config + +Code for configuring the peachcloud vps for various hosting and automation +- debian repository of microservices +- mdbook builder for devdocs + + +# setup +``` +apt update +apt install git python python3-pip rsync +git clone https://github.com/peachcloud/peach-vps.git +cd peach-vps +pip3 install -r requirements.txt +python peach_vps_scripts/setup_vps.py +``` + + +# update +(for more frequent updates that don't involve the whole initial setup) +``` +cd peach-vps +python peach_vps_scripts/update_vps.py +``` \ No newline at end of file diff --git a/peach_vps_scripts/setup_vps.py b/peach_vps_scripts/setup_vps.py deleted file mode 100644 index 4001f1d..0000000 --- a/peach_vps_scripts/setup_vps.py +++ /dev/null @@ -1,39 +0,0 @@ -from peach_vps_scripts.utils import render_template, cargo_install -from peach_vps_scripts.vars import VARS - -import subprocess -import os - - -print("[ UPDATING OPERATING SYSTEM ]") -subprocess.call(["apt-get", "update", "-y"]) -subprocess.call(["apt-get", "upgrade", "-y"]) - -print("[ INSTALLING SYSTEM REQUIREMENTS ]") -subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "mosh"]) - -print("[ CREATING SYSTEM GROUPS ]") -subprocess.call(["/usr/sbin/groupadd", "peach"]) - -print("[ ADDING SYSTEM USER ]") -users = ["notplants", "glyph"] -for user in users: - subprocess.call(["/usr/sbin/adduser", user]) - subprocess.call(["usermod", "-aG", "sudo", user]) - subprocess.call(["/usr/sbin/usermod", "-a", "-G", user, "peach"]) - -print("[ CREATING DIRECTORIES ]") -folders = [VARS["src_dir"], VARS["www_dir"], VARS["debian_repo_dir"]] -for folder in folders: - if not os.path.exists(folder): - os.makedirs(folder) - -print("[ INSTALLING RUST ]") -subprocess.call(["curl", "https://sh.rustup.rs", "-sSf", "|", "sh", "-s", "--", "-y"]) - -print("[ INSTALLING CARGO PACKAGES ]") -cargo_install("cargo-deb") - -render_template(src='nginx/nginx.conf', dest='/etc/nginx/nginx.conf') - - diff --git a/peach_vps_scripts/__init__.py b/scripts/__init__.py similarity index 100% rename from peach_vps_scripts/__init__.py rename to scripts/__init__.py diff --git a/scripts/setup_vps.py b/scripts/setup_vps.py new file mode 100644 index 0000000..3f85fe8 --- /dev/null +++ b/scripts/setup_vps.py @@ -0,0 +1,55 @@ +from utils import render_template, cargo_install +from vars import VARS + +import subprocess +import os +import pwd +import grp + + +print("[ UPDATING OPERATING SYSTEM ]") +subprocess.check_call(["apt-get", "update", "-y"]) +subprocess.check_call(["apt-get", "upgrade", "-y"]) + +print("[ INSTALLING SYSTEM REQUIREMENTS ]") +subprocess.check_call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "mosh"]) + +print("[ CREATING SYSTEM GROUPS ]") +group = 'peach' +try: + grp.getgrnam(group) + # if group exists +except KeyError: + # if group doesn't eixst + subprocess.check_call(["/usr/sbin/groupadd", "peach"]) + +print("[ ADDING SYSTEM USER ]") +users = ["notplants", "glyph"] +for user in users: + try: + # if user exists + pwd.getpwnam(user) + except: + # if user does not exist + subprocess.check_call(["/usr/sbin/adduser", user]) + subprocess.check_call(["usermod", "-aG", "sudo", user]) + subprocess.check_call(["/usr/sbin/usermod", "-a", "-G", user, "peach"]) + +print("[ CREATING DIRECTORIES ]") +folders = [VARS["src_dir"], VARS["web_dir"], VARS["debian_rep_dir"]] +for folder in folders: + if not os.path.exists(folder): + os.makedirs(folder) + +print("[ INSTALLING RUST ]") +if os.path.exists('/root/.cargo/bin/rustc'): + first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) + output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) + first_command.wait() + +print("[ INSTALLING CARGO PACKAGES ]") +cargo_install("cargo-deb") + +render_template(src='nginx/nginx.conf', dest='/etc/nginx/nginx.conf') + + diff --git a/peach_vps_scripts/utils.py b/scripts/utils.py similarity index 71% rename from peach_vps_scripts/utils.py rename to scripts/utils.py index 8533937..a9508cd 100644 --- a/peach_vps_scripts/utils.py +++ b/scripts/utils.py @@ -1,12 +1,13 @@ -from peach_vps_scripts.vars import VARS +from vars import VARS import os import jinja2 import subprocess -PROJECT_PATH = os.path.abspath(os.path.dirname(__file__)) +PROJECT_PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +print('PROJECT_PATH: {}'.format(PROJECT_PATH)) -template_path = os.path.join(PROJECT_PATH, 'templates') +template_path = os.path.join(PROJECT_PATH, 'conf/templates') template_loader = jinja2.FileSystemLoader(searchpath=template_path) template_env = jinja2.Environment(loader=template_loader) @@ -19,7 +20,10 @@ def render_template(src, dest, template_vars=None): :return: None """ template = template_env.get_template(src) - template_vars.update(VARS) + if template_vars: + template_vars.update(VARS) + else: + template_vars = VARS output_text = template.render(template_vars=template_vars) if os.path.exists(dest): os.remove(dest) diff --git a/peach_vps_scripts/vars.py b/scripts/vars.py similarity index 100% rename from peach_vps_scripts/vars.py rename to scripts/vars.py From ce55310c65225cdb29c37ed6477bca344c255ac1 Mon Sep 17 00:00:00 2001 From: Max Fowler Date: Wed, 11 Nov 2020 11:21:52 +0100 Subject: [PATCH 3/8] Pythong script for initializing and updating peachcloud debian repo of microservices. --- conf/templates/debian_repo/distributions | 8 ++ conf/templates/debian_repo/nginx_debian.conf | 21 ++++ conf/templates/debian_repo/options | 3 + conf/templates/debian_repo/override.buster | 4 + scripts/setup_debian_repo.py | 105 +++++++++++++++++++ scripts/setup_vps.py | 3 +- scripts/utils.py | 11 +- 7 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 conf/templates/debian_repo/distributions create mode 100644 conf/templates/debian_repo/nginx_debian.conf create mode 100644 conf/templates/debian_repo/options create mode 100644 conf/templates/debian_repo/override.buster create mode 100644 scripts/setup_debian_repo.py diff --git a/conf/templates/debian_repo/distributions b/conf/templates/debian_repo/distributions new file mode 100644 index 0000000..f6fed0f --- /dev/null +++ b/conf/templates/debian_repo/distributions @@ -0,0 +1,8 @@ +Origin: PeachCloud +Label: PeachCloud +Codename: buster +Architectures: amd64 +Components: main +Description: Apt repository for PeachCloud debian packages +SignWith: {{gpg_key_id}} +DebOverride: override.buster diff --git a/conf/templates/debian_repo/nginx_debian.conf b/conf/templates/debian_repo/nginx_debian.conf new file mode 100644 index 0000000..73b671e --- /dev/null +++ b/conf/templates/debian_repo/nginx_debian.conf @@ -0,0 +1,21 @@ +server { + listen 80; + server_name 167.99.136.83; + + access_log /var/log/nginx-debian.log; + error_log /var/log/nginx-debian.error; + + location / { + root {{apt_dir}}; + index index.html; + autoindex on; + } + + location ~ /(.*)/conf { + deny all; + } + + location ~ /(.*)/db { + deny all; + } +} \ No newline at end of file diff --git a/conf/templates/debian_repo/options b/conf/templates/debian_repo/options new file mode 100644 index 0000000..5dd7345 --- /dev/null +++ b/conf/templates/debian_repo/options @@ -0,0 +1,3 @@ +verbose +basedir {{debian_rep_dir}} +ask-passphrase diff --git a/conf/templates/debian_repo/override.buster b/conf/templates/debian_repo/override.buster new file mode 100644 index 0000000..ec9fb2d --- /dev/null +++ b/conf/templates/debian_repo/override.buster @@ -0,0 +1,4 @@ +{% for service in services %} +{{service}} Priority optional +{{service}} Section net +{% endfor %} \ No newline at end of file diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py new file mode 100644 index 0000000..71740b1 --- /dev/null +++ b/scripts/setup_debian_repo.py @@ -0,0 +1,105 @@ +from utils import render_template + +import subprocess +import os + + +INITIALIZE_DEBIAN_REPO = True + +MICROSERVICES_SRC_DIR = "/srv/peachcloud/src" +WEB_DIR = "/var/www/" +APT_DIR = "/var/www/repos/apt" +DEBIAN_REPO_DIR = "/var/www/repos/apt/debian" +DEBIAN_REPO_CONF_DIR = "/var/www/repos/apt/debian/conf" + +# before running this script run `gpg --gen-key` on the server, and put the key id here +GPG_KEY_ID = "E62CD13A85763FCEC3EDBA8EA98440817F1A3CE5", + +SERVICES = [ + {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, + {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"} +] + +if INITIALIZE_DEBIAN_REPO: + + print("[ INSTALLING SYSTEM REQUIREMENTS ]") + subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "reprepro", "gcc-aarch64-linux-gnu", ]) + + print("[ CREATING DIRECTORIES ]") + folders = [MICROSERVICES_SRC_DIR, WEB_DIR, APT_DIR, DEBIAN_REPO_DIR, DEBIAN_REPO_CONF_DIR] + for folder in folders: + if not os.path.exists(folder): + os.makedirs(folder) + + print("[ INSTALLING RUST ]") + if not os.path.exists("/root/.cargo/bin/rustc"): + first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) + output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) + first_command.wait() + + print("[ INSTALLING CARGO-DEB ]") + subprocess.call(['cargo', 'install', 'cargo-deb']) + + print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") + subprocess.call(['rustup', 'toolchain', 'install', 'nightly-aarch64-unknown-linux-gnu']) + + print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") + for service in SERVICES: + name = service["name"] + repo_url = service["repo_url"] + service_path = os.path.join(MICROSERVICES_SRC_DIR, name) + if not os.path.exists(service_path): + subprocess.call(["git", "clone", repo_url, service_path]) + + print("[ COPYING DEBIAN REPO CONFIG ]") + render_template( + src="debian_repo/distributions", + dest="{}/distributions".format(DEBIAN_REPO_CONF_DIR), + template_vars={ + "gpg_key_id": GPG_KEY_ID + } + ) + render_template( + src="debian_repo/options", + dest="{}/options".format(DEBIAN_REPO_CONF_DIR), + template_vars={ + "debian_rep_dir": DEBIAN_REPO_DIR + } + ) + render_template( + src="debian_repo/override.buster", + dest="{}/override.buster".format(DEBIAN_REPO_CONF_DIR), + template_vars={ + "services": [service["name"] for service in SERVICES] + } + ) + + print("[ EXPORTING PUBLIC GPG KEY ]") + output_path = "{}/peach_pub.gpg".format(APT_DIR) + subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID]) + + print("[ COPYING NGINX CONFIG ]") + render_template( + src="debian_repo/nginx_debian.conf", + dest="/etc/nginx/sites-enabled/deb.peachcloud.org", + template_vars = { + "apt_dir": APT_DIR + } + ) + + +# below is code for updating the microservices, building the microservices, +# and adding them to the debian repo +for service in SERVICES: + service_name = service['name'] + service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) + print("[ BUILIDING SERVICE {} ]".format(service_name)) + subprocess.call("cd {} && git pull;".format(service_path)) + subprocess.call("cd {} && cargo deb --target aarch64-unknown-linux-gnu;".format(service_path)) + deb_path = '?' + subprocess.call("cd {debian_dir} && reprepro includedeb buster {deb_path}".format( + debian_dir=DEBIAN_REPO_DIR, + deb_path=deb_path + )) + + diff --git a/scripts/setup_vps.py b/scripts/setup_vps.py index 3f85fe8..24aa5e8 100644 --- a/scripts/setup_vps.py +++ b/scripts/setup_vps.py @@ -42,7 +42,7 @@ for folder in folders: os.makedirs(folder) print("[ INSTALLING RUST ]") -if os.path.exists('/root/.cargo/bin/rustc'): +if not os.path.exists('/root/.cargo/bin/rustc'): first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) first_command.wait() @@ -50,6 +50,7 @@ if os.path.exists('/root/.cargo/bin/rustc'): print("[ INSTALLING CARGO PACKAGES ]") cargo_install("cargo-deb") +print("[ COPYING NGINX CONFIG ]") render_template(src='nginx/nginx.conf', dest='/etc/nginx/nginx.conf') diff --git a/scripts/utils.py b/scripts/utils.py index a9508cd..c6362ab 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -1,5 +1,3 @@ -from vars import VARS - import os import jinja2 import subprocess @@ -20,10 +18,8 @@ def render_template(src, dest, template_vars=None): :return: None """ template = template_env.get_template(src) - if template_vars: - template_vars.update(VARS) - else: - template_vars = VARS + if not template_vars: + template_vars= {} output_text = template.render(template_vars=template_vars) if os.path.exists(dest): os.remove(dest) @@ -31,6 +27,3 @@ def render_template(src, dest, template_vars=None): f.write(output_text) -def cargo_install(package): - subprocess.call(['/root/.cargo/bin/cargo', 'install', package]) - From e9feefa03529728db84c974b50e970f35209f421 Mon Sep 17 00:00:00 2001 From: Max Fowler Date: Thu, 12 Nov 2020 16:46:28 +0100 Subject: [PATCH 4/8] Basic working script --- .gitignore | 3 +- scripts/setup_debian_repo.py | 29 ++++++++++--------- scripts/setup_vps.py | 56 ------------------------------------ scripts/vars.py | 11 ------- 4 files changed, 18 insertions(+), 81 deletions(-) delete mode 100644 scripts/setup_vps.py delete mode 100644 scripts/vars.py diff --git a/.gitignore b/.gitignore index 5dacb78..fbe14c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ deploy.sh secret* secret_files* -ssh.sh \ No newline at end of file +ssh.sh +notes.txt \ No newline at end of file diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 71740b1..a29b341 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -3,17 +3,17 @@ from utils import render_template import subprocess import os - INITIALIZE_DEBIAN_REPO = True -MICROSERVICES_SRC_DIR = "/srv/peachcloud/src" +MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" WEB_DIR = "/var/www/" APT_DIR = "/var/www/repos/apt" DEBIAN_REPO_DIR = "/var/www/repos/apt/debian" DEBIAN_REPO_CONF_DIR = "/var/www/repos/apt/debian/conf" # before running this script run `gpg --gen-key` on the server, and put the key id here -GPG_KEY_ID = "E62CD13A85763FCEC3EDBA8EA98440817F1A3CE5", +# `gpg --list-keys` +GPG_KEY_ID = "4ACEF251EA3E091167E8F03EBF69A52BE3565476" SERVICES = [ {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, @@ -38,10 +38,12 @@ if INITIALIZE_DEBIAN_REPO: first_command.wait() print("[ INSTALLING CARGO-DEB ]") - subprocess.call(['cargo', 'install', 'cargo-deb']) + if not os.path.exists("/root/.cargo/bin/cargo-deb"): + subprocess.call(["/root/.cargo/bin/cargo", "install", "cargo-deb"]) print("[ INSTALL TOOLCHAIN FOR CROSS-COMPILATION ]") - subprocess.call(['rustup', 'toolchain', 'install', 'nightly-aarch64-unknown-linux-gnu']) + subprocess.call(["/root/.cargo/bin/rustup", "target", "add", "aarch64-unknown-linux-gnu"]) + subprocess.call(["/root/.cargo/bin/rustup", "toolchain", "install", "nightly-aarch64-unknown-linux-gnu"]) print("[ PULLING MICROSERVICES CODE FROM GITHUB ]") for service in SERVICES: @@ -76,7 +78,8 @@ if INITIALIZE_DEBIAN_REPO: print("[ EXPORTING PUBLIC GPG KEY ]") output_path = "{}/peach_pub.gpg".format(APT_DIR) - subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID]) + if not os.path.exists(output_path): + subprocess.call(["gpg", "--armor", "--output", output_path, "--export", GPG_KEY_ID]) print("[ COPYING NGINX CONFIG ]") render_template( @@ -91,15 +94,15 @@ if INITIALIZE_DEBIAN_REPO: # below is code for updating the microservices, building the microservices, # and adding them to the debian repo for service in SERVICES: - service_name = service['name'] + service_name = service["name"] service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) print("[ BUILIDING SERVICE {} ]".format(service_name)) - subprocess.call("cd {} && git pull;".format(service_path)) - subprocess.call("cd {} && cargo deb --target aarch64-unknown-linux-gnu;".format(service_path)) - deb_path = '?' - subprocess.call("cd {debian_dir} && reprepro includedeb buster {deb_path}".format( + subprocess.call(["git", "pull"], cwd=service_path) + debian_package_path = str(subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path)) + print('OUTPUT: {}'.format(debian_package_path)) + subprocess.call("reprepro includedeb buster {deb_path}".format( debian_dir=DEBIAN_REPO_DIR, - deb_path=deb_path - )) + deb_path=debian_package_path + ), cwd=DEBIAN_REPO_DIR) diff --git a/scripts/setup_vps.py b/scripts/setup_vps.py deleted file mode 100644 index 24aa5e8..0000000 --- a/scripts/setup_vps.py +++ /dev/null @@ -1,56 +0,0 @@ -from utils import render_template, cargo_install -from vars import VARS - -import subprocess -import os -import pwd -import grp - - -print("[ UPDATING OPERATING SYSTEM ]") -subprocess.check_call(["apt-get", "update", "-y"]) -subprocess.check_call(["apt-get", "upgrade", "-y"]) - -print("[ INSTALLING SYSTEM REQUIREMENTS ]") -subprocess.check_call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "mosh"]) - -print("[ CREATING SYSTEM GROUPS ]") -group = 'peach' -try: - grp.getgrnam(group) - # if group exists -except KeyError: - # if group doesn't eixst - subprocess.check_call(["/usr/sbin/groupadd", "peach"]) - -print("[ ADDING SYSTEM USER ]") -users = ["notplants", "glyph"] -for user in users: - try: - # if user exists - pwd.getpwnam(user) - except: - # if user does not exist - subprocess.check_call(["/usr/sbin/adduser", user]) - subprocess.check_call(["usermod", "-aG", "sudo", user]) - subprocess.check_call(["/usr/sbin/usermod", "-a", "-G", user, "peach"]) - -print("[ CREATING DIRECTORIES ]") -folders = [VARS["src_dir"], VARS["web_dir"], VARS["debian_rep_dir"]] -for folder in folders: - if not os.path.exists(folder): - os.makedirs(folder) - -print("[ INSTALLING RUST ]") -if not os.path.exists('/root/.cargo/bin/rustc'): - first_command = subprocess.Popen(["curl", "https://sh.rustup.rs", "-sSf"], stdout=subprocess.PIPE) - output = subprocess.check_output(["sh", "-s", "--", "-y"], stdin=first_command.stdout) - first_command.wait() - -print("[ INSTALLING CARGO PACKAGES ]") -cargo_install("cargo-deb") - -print("[ COPYING NGINX CONFIG ]") -render_template(src='nginx/nginx.conf', dest='/etc/nginx/nginx.conf') - - diff --git a/scripts/vars.py b/scripts/vars.py deleted file mode 100644 index 8b66352..0000000 --- a/scripts/vars.py +++ /dev/null @@ -1,11 +0,0 @@ -VARS = { - 'log_dir': '/srv/log', - 'src_dir': '/srv/src', - 'web_dir': 'srv/www', - 'debian_rep_dir': '/srv/www/repos/apt/debian', - 'gpg_key_id': 'E62CD13A85763FCEC3EDBA8EA98440817F1A3CE5', - 'services': [ - {'name': 'peach-oled', 'repo_url': 'https://github.com/peachcloud/peach-oled.git'}, - {'name': 'peach-oled', 'repo_url': 'https://github.com/peachcloud/peach-oled.git'} - ] -} \ No newline at end of file From fed137f82d7635f995c208864dd3e3f265c91061 Mon Sep 17 00:00:00 2001 From: Max Fowler Date: Thu, 12 Nov 2020 17:21:48 +0100 Subject: [PATCH 5/8] Reprepro working --- conf/templates/debian/distributions | 8 ----- conf/templates/debian/options | 3 -- conf/templates/debian/override.buster | 4 --- conf/templates/debian_repo/distributions | 2 +- conf/templates/debian_repo/nginx_debian.conf | 2 +- conf/templates/devdocs/git-post-receive | 13 -------- conf/templates/nginx/nginx.conf | 31 -------------------- conf/templates/nginx/nginx_devdocs.conf | 10 ------- scripts/setup_debian_repo.py | 7 ++--- scripts/utils.py | 4 +-- 10 files changed, 6 insertions(+), 78 deletions(-) delete mode 100644 conf/templates/debian/distributions delete mode 100644 conf/templates/debian/options delete mode 100644 conf/templates/debian/override.buster delete mode 100644 conf/templates/devdocs/git-post-receive delete mode 100644 conf/templates/nginx/nginx.conf delete mode 100644 conf/templates/nginx/nginx_devdocs.conf diff --git a/conf/templates/debian/distributions b/conf/templates/debian/distributions deleted file mode 100644 index f6fed0f..0000000 --- a/conf/templates/debian/distributions +++ /dev/null @@ -1,8 +0,0 @@ -Origin: PeachCloud -Label: PeachCloud -Codename: buster -Architectures: amd64 -Components: main -Description: Apt repository for PeachCloud debian packages -SignWith: {{gpg_key_id}} -DebOverride: override.buster diff --git a/conf/templates/debian/options b/conf/templates/debian/options deleted file mode 100644 index 5dd7345..0000000 --- a/conf/templates/debian/options +++ /dev/null @@ -1,3 +0,0 @@ -verbose -basedir {{debian_rep_dir}} -ask-passphrase diff --git a/conf/templates/debian/override.buster b/conf/templates/debian/override.buster deleted file mode 100644 index ec9fb2d..0000000 --- a/conf/templates/debian/override.buster +++ /dev/null @@ -1,4 +0,0 @@ -{% for service in services %} -{{service}} Priority optional -{{service}} Section net -{% endfor %} \ No newline at end of file diff --git a/conf/templates/debian_repo/distributions b/conf/templates/debian_repo/distributions index f6fed0f..45d7c64 100644 --- a/conf/templates/debian_repo/distributions +++ b/conf/templates/debian_repo/distributions @@ -1,7 +1,7 @@ Origin: PeachCloud Label: PeachCloud Codename: buster -Architectures: amd64 +Architectures: amd64 arm64 Components: main Description: Apt repository for PeachCloud debian packages SignWith: {{gpg_key_id}} diff --git a/conf/templates/debian_repo/nginx_debian.conf b/conf/templates/debian_repo/nginx_debian.conf index 73b671e..53ea5bc 100644 --- a/conf/templates/debian_repo/nginx_debian.conf +++ b/conf/templates/debian_repo/nginx_debian.conf @@ -1,6 +1,6 @@ server { listen 80; - server_name 167.99.136.83; + server_name deb.peachcloud.org; access_log /var/log/nginx-debian.log; error_log /var/log/nginx-debian.error; diff --git a/conf/templates/devdocs/git-post-receive b/conf/templates/devdocs/git-post-receive deleted file mode 100644 index 36b37bd..0000000 --- a/conf/templates/devdocs/git-post-receive +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -while read oldrev newrev ref -do - if [[ $ref =~ .*/master$ ]]; - then - echo "Master ref received. Deploying master branch to build directory..." - git --work-tree={{src_dir}}/devdocs_build --git-dir={{src_dir}}/devdocs_bare checkout -f - echo "Building docs and deploying to production..." - /root/.cargo/bin/mdbook build {{src_dir}}/devdocs_build --dest-dir {{web_dir}}/docs:peachcloud:org/html - else - echo "Ref $ref successfully received. Doing nothing: only the master branch may be deployed on this server." - fi -done \ No newline at end of file diff --git a/conf/templates/nginx/nginx.conf b/conf/templates/nginx/nginx.conf deleted file mode 100644 index 668b54c..0000000 --- a/conf/templates/nginx/nginx.conf +++ /dev/null @@ -1,31 +0,0 @@ -user www-data; -worker_processes 1; -worker_rlimit_nofile 8192; - -events { - worker_connections 3000; -} - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - #gzip on; - - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} \ No newline at end of file diff --git a/conf/templates/nginx/nginx_devdocs.conf b/conf/templates/nginx/nginx_devdocs.conf deleted file mode 100644 index c97291c..0000000 --- a/conf/templates/nginx/nginx_devdocs.conf +++ /dev/null @@ -1,10 +0,0 @@ -server { - listen 80; - server_name 159.89.5.141; - - location / { - root {{web_dir}}/docs:peachcloud:org/html; - index index.html; - try_files $uri $uri/ /index.html; - } -} \ No newline at end of file diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index a29b341..fb8fa9c 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -98,11 +98,8 @@ for service in SERVICES: service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) print("[ BUILIDING SERVICE {} ]".format(service_name)) subprocess.call(["git", "pull"], cwd=service_path) - debian_package_path = str(subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path)) + debian_package_path = subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip() print('OUTPUT: {}'.format(debian_package_path)) - subprocess.call("reprepro includedeb buster {deb_path}".format( - debian_dir=DEBIAN_REPO_DIR, - deb_path=debian_package_path - ), cwd=DEBIAN_REPO_DIR) + subprocess.call(["reprepro", "includedeb", "buster", debian_package_path], cwd=DEBIAN_REPO_DIR) diff --git a/scripts/utils.py b/scripts/utils.py index c6362ab..6d208a1 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -7,7 +7,7 @@ print('PROJECT_PATH: {}'.format(PROJECT_PATH)) template_path = os.path.join(PROJECT_PATH, 'conf/templates') template_loader = jinja2.FileSystemLoader(searchpath=template_path) -template_env = jinja2.Environment(loader=template_loader) +template_env = jinja2.Environment(loader=template_loader, keep_trailing_newline=True) def render_template(src, dest, template_vars=None): @@ -20,7 +20,7 @@ def render_template(src, dest, template_vars=None): template = template_env.get_template(src) if not template_vars: template_vars= {} - output_text = template.render(template_vars=template_vars) + output_text = template.render(**template_vars) if os.path.exists(dest): os.remove(dest) with open(dest, 'w') as f: From fba90e6e2c340198e0f6e4f1adb51b6883b2efd1 Mon Sep 17 00:00:00 2001 From: Max Fowler Date: Thu, 12 Nov 2020 17:28:01 +0100 Subject: [PATCH 6/8] Change dns to apt.peachcloud.org --- conf/templates/debian_repo/nginx_debian.conf | 2 +- scripts/setup_debian_repo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/templates/debian_repo/nginx_debian.conf b/conf/templates/debian_repo/nginx_debian.conf index 53ea5bc..fce5521 100644 --- a/conf/templates/debian_repo/nginx_debian.conf +++ b/conf/templates/debian_repo/nginx_debian.conf @@ -1,6 +1,6 @@ server { listen 80; - server_name deb.peachcloud.org; + server_name apt.peachcloud.org; access_log /var/log/nginx-debian.log; error_log /var/log/nginx-debian.error; diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index fb8fa9c..dc13985 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -84,7 +84,7 @@ if INITIALIZE_DEBIAN_REPO: print("[ COPYING NGINX CONFIG ]") render_template( src="debian_repo/nginx_debian.conf", - dest="/etc/nginx/sites-enabled/deb.peachcloud.org", + dest="/etc/nginx/sites-enabled/apt.peachcloud.org", template_vars = { "apt_dir": APT_DIR } From 671df65156a0adc3691886e324fb71c7c9ece65e Mon Sep 17 00:00:00 2001 From: Max Fowler Date: Fri, 13 Nov 2020 12:45:01 +0100 Subject: [PATCH 7/8] Separate setup and update --- README.md | 40 +++++++++++++++++++++++++++++------- scripts/setup_debian_repo.py | 30 +++++++++++++++++++++------ 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8e91bd1..04c99f1 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,50 @@ # peach-vps config -Code for configuring the peachcloud vps for various hosting and automation +Scripts for configuring the peachcloud vps for various hosting and automation. + +Currently: - debian repository of microservices -- mdbook builder for devdocs -# setup +# setup debian repo +an idempotent script for initializing the debian repo on the vps ``` apt update apt install git python python3-pip rsync git clone https://github.com/peachcloud/peach-vps.git cd peach-vps pip3 install -r requirements.txt -python peach_vps_scripts/setup_vps.py +python3 scripts/setup_debian_repo.py -i ``` -# update -(for more frequent updates that don't involve the whole initial setup) +# update debian repo +without the -i flag, the setup_debian_repo rebuilds all +microservices (cross-compiled to arm64) and re-adds them to the debian repo ``` cd peach-vps -python peach_vps_scripts/update_vps.py +python3 scripts/setup_debian_repo.py +``` + + +# using the debian repo on the pi +To add the peachcloud debian repo as an apt source, +on the pi, +``` +vi /etc/apt/sources.list.d/peach.list +``` +and add the following line: +``` +deb http://apt.peachcloud.org/debian/ buster main +``` + +Then add the gpg pub key to the apt-key list: +``` +wget -O - http://apt.peachcloud.org/peach_pub.gpg | sudo apt-key add - +``` + +You can then install peach packages with apt-get: +``` +apt-get update +apt-get install peach-oled ``` \ No newline at end of file diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index dc13985..119d58b 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -2,9 +2,10 @@ from utils import render_template import subprocess import os +import argparse -INITIALIZE_DEBIAN_REPO = True +# constants MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" WEB_DIR = "/var/www/" APT_DIR = "/var/www/repos/apt" @@ -17,10 +18,21 @@ GPG_KEY_ID = "4ACEF251EA3E091167E8F03EBF69A52BE3565476" SERVICES = [ {"name": "peach-oled", "repo_url": "https://github.com/peachcloud/peach-oled.git"}, - {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"} + {"name": "peach-network", "repo_url": "https://github.com/peachcloud/peach-network.git"}, + {"name": "peach-stats", "repo_url": "https://github.com/peachcloud/peach-stats.git"}, + # {"name": "peach-web", "repo_url": "https://github.com/peachcloud/peach-web.git"}, # currently build fails because it needs rust nightly for pear + {"name": "peach-menu", "repo_url": "https://github.com/peachcloud/peach-menu.git"}, + {"name": "peach-buttons", "repo_url": "https://github.com/peachcloud/peach-buttons.git"} ] -if INITIALIZE_DEBIAN_REPO: +# parse CLI args +parser = argparse.ArgumentParser() +parser.add_argument("-i", "--initialize", help="initialize and update debian repo", action="store_true") +args = parser.parse_args() + +# initializing debian repo from a blank slate +# (but this code is idempotent so it can be re-run if already initialized) +if args.initialize: print("[ INSTALLING SYSTEM REQUIREMENTS ]") subprocess.call(["apt-get", "install", "git", "nginx", "curl", "build-essential", "reprepro", "gcc-aarch64-linux-gnu", ]) @@ -91,15 +103,21 @@ if INITIALIZE_DEBIAN_REPO: ) -# below is code for updating the microservices, building the microservices, -# and adding them to the debian repo +# below is code for git updating the microservices, building the microservices, +# and (re)-adding them to the debian repo +print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") for service in SERVICES: service_name = service["name"] service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) print("[ BUILIDING SERVICE {} ]".format(service_name)) subprocess.call(["git", "pull"], cwd=service_path) debian_package_path = subprocess.check_output(["/root/.cargo/bin/cargo", "deb", "--target", "aarch64-unknown-linux-gnu"], cwd=service_path).decode("utf-8").strip() - print('OUTPUT: {}'.format(debian_package_path)) + # remove debian package from repo + # (in the future we could look at some way of updating with versions instead of removing and adding) + subprocess.call(["reprepro", "remove", "buster", service_name], cwd=DEBIAN_REPO_DIR) + # add the package subprocess.call(["reprepro", "includedeb", "buster", debian_package_path], cwd=DEBIAN_REPO_DIR) +print("[ DEBIAN REPO SETUP COMPLETE ]") + From 6240bad2160daa6aeae2926b9077943eec991f2f Mon Sep 17 00:00:00 2001 From: Max Fowler Date: Fri, 13 Nov 2020 13:07:30 +0100 Subject: [PATCH 8/8] Changed /var/www/apt.peachcloud.org to match convention of other subdomains --- scripts/setup_debian_repo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/setup_debian_repo.py b/scripts/setup_debian_repo.py index 119d58b..7644887 100644 --- a/scripts/setup_debian_repo.py +++ b/scripts/setup_debian_repo.py @@ -8,9 +8,9 @@ import argparse # constants MICROSERVICES_SRC_DIR = "/srv/peachcloud/automation/microservices" WEB_DIR = "/var/www/" -APT_DIR = "/var/www/repos/apt" -DEBIAN_REPO_DIR = "/var/www/repos/apt/debian" -DEBIAN_REPO_CONF_DIR = "/var/www/repos/apt/debian/conf" +APT_DIR = "/var/www/apt.peachcloud.org" +DEBIAN_REPO_DIR = "/var/www/apt.peachcloud.org/debian" +DEBIAN_REPO_CONF_DIR = "/var/www/apt.peachcloud.org/debian/conf" # before running this script run `gpg --gen-key` on the server, and put the key id here # `gpg --list-keys`