diff --git a/README.md b/README.md index e4e1474..73ce129 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -# peach-vps +# peach-package-builder -![Generic badge](https://img.shields.io/badge/version-0.3.1-.svg) +![Generic badge](https://img.shields.io/badge/version-0.3.2-.svg) -Scripts for configuring the PeachCloud VPS for various hosting and automation functions. +Scripts for building debian packages for PeachCloud microservices. ## Setup Build Environment -`scripts/setup_build_env.py` +`python3 peach_package_builder/setup_build_env.py` An idempotent script for initializing a build and deployment environment for PeachCloud packages. @@ -35,7 +35,7 @@ cd peach-vps pip3 install -r requirements.txt ``` -Open `scripts/setup_build_env.py` and set the following constants: +Open `peach_package_builder/setup_build_env.py` and set the following constants: - USER_PATH - GPG_KEY_EMAIL @@ -44,34 +44,51 @@ Open `scripts/setup_build_env.py` and set the following constants: Then execute the script to run the full system initialization process (_note: several commands executed by the script require `sudo` permissions. You will be prompted for the user password during the execution of the scipt._): ``` -python3 -u scripts/setup_build_env.py +python3 -u peach_package_builder/setup_build_env.py ``` -## Build and Serve Debian Packages +## Build packages -`scripts/build_packages.py` +`peach_package_builder/build_packages.py` An idempotent script for building the latest versions of all PeachCloud packages and adding them to the Debian package archive. The script currently performs the following actions: - - Builds and updates microservice packages + - Builds and updates Rust microservice packages + - Builds and updates peach-config python package - Adds packages to Freight library - Adds packages to Freight cache ``` -python3 -u scripts/build_packages.py +python3 -d peach_package_builder/build_packages.py ``` +The -d flag ensures that all packages are built from the latest version of the default branch currently on GitHub. +Without the -d flag, whatever version of the code is locally stored will be used (which can be useful for testing). + Freight supports the ability to have multiple versions of a package in a single Debian package archive. If a particular version of a package already exists in the Freight library, it will not be readded or overwritten. -## Build peach-go-sbot Debian package +You can also just build the rust packages by running: +```python3 -d peach_package_builder/build_rust_packages.py``` -`sudo python3 scripts/build_peach_go_sbot.py -v ` +Or just build peach-config by running: +```python3 -d peach_package_builder/build_peach_config.py``` + + +## Build peach-go-sbot package + +First, open peach_package_builder/build_peach_go_sbot.py and manually edit PEACH_GO_SBOT_VERSION. + +We manually increment the version number when we want to build a new version of peach-go-sbot. + +Then run, +`python3 peach_package_builder/build_peach_go_sbot.py` This builds the peach-go-sbot package using the latest code from go-ssb, along with a systemd unit file, and adds the Debian package to the Freight library. + ## Install Packages from Debian Package Archive To add the PeachCloud Debian package archive as an apt source, run the following commands from your Pi: diff --git a/scripts/__init__.py b/peach_package_builder/__init__.py similarity index 100% rename from scripts/__init__.py rename to peach_package_builder/__init__.py diff --git a/peach_package_builder/build_packages.py b/peach_package_builder/build_packages.py new file mode 100644 index 0000000..108f1c1 --- /dev/null +++ b/peach_package_builder/build_packages.py @@ -0,0 +1,25 @@ +import argparse + +from peach_package_builder.build_rust_packages import build_rust_packages +from peach_package_builder.build_peach_config import build_peach_config + + +def build_packages(default_branch=False): + """ + builds all PeachCloud microservices as .deb files and adds them to the freight repo + """ + build_rust_packages(default_branch=default_branch) + build_peach_config(default_branch=default_branch) + print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", + "--default", + help="Ensure default branch for all repos for build", + action="store_true" + ) + args = parser.parse_args() + build_packages(default_branch=args.default) \ No newline at end of file diff --git a/peach_package_builder/build_peach_config.py b/peach_package_builder/build_peach_config.py new file mode 100644 index 0000000..9afb0f9 --- /dev/null +++ b/peach_package_builder/build_peach_config.py @@ -0,0 +1,57 @@ +""" +script to build the peach-config debian module and add it to the freight repository +""" +import argparse +import subprocess +import sys +import os + +from peach_package_builder.constants import * +from peach_package_builder.utils import add_deb_to_freight, update_freight_cache + + +def build_peach_config(default_branch=True): + service_path = os.path.join(MICROSERVICES_SRC_DIR, "peach-config") + if default_branch: + # because some repo have main as default and some as master, we get the default + default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'], + cwd=service_path).decode(sys.stdout.encoding) + branch = default_branch.replace('origin/', '').strip() + subprocess.check_call(["git", "checkout", branch], cwd=service_path) + subprocess.check_call(["git", "reset", "HEAD", "--hard"], cwd=service_path) + subprocess.check_call(["git", "pull"], cwd=service_path) + # remove old build dir + subprocess.check_call([ + "rm", + "-rf", + os.path.join(service_path, 'deb_dist') + ]) + # build .deb + subprocess.check_call([ + "python3", + "setup.py", + "--command-packages=stdeb.command", + "bdist_deb" + ], + cwd=service_path) + version = "0.2.7" # TODO: get this version number from the repo + deb_name = "python3-peach-config_{version}-1_all.deb".format(version=version) + debian_package_path = os.path.join( + service_path, + "deb_dist/{}".format(deb_name) + ) + subprocess.check_call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) + add_deb_to_freight(package_name=debian_package_path, package_path=debian_package_path) + update_freight_cache() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", + "--default", + help="Ensure default branch for all repos for build", + action="store_true" + ) + args = parser.parse_args() + build_peach_config(default_branch=args.default) \ No newline at end of file diff --git a/scripts/build_peach_go_sbot.py b/peach_package_builder/build_peach_go_sbot.py similarity index 77% rename from scripts/build_peach_go_sbot.py rename to peach_package_builder/build_peach_go_sbot.py index 9660553..3cb3fed 100644 --- a/scripts/build_peach_go_sbot.py +++ b/peach_package_builder/build_peach_go_sbot.py @@ -1,20 +1,22 @@ -#!/usr/bin/env python3 """ script to create debian packages for cross-compiled go binaries for go-sbot based off of this post https://unix.stackexchange.com/questions/627689/how-to-create-a-debian-package-from-a-bash-script-and-a-systemd-service """ - - -from constants import * -from utils import render_template - import subprocess import argparse -import sys -import os +import re import shutil +import sys +from packaging import version as pversion +from peach_package_builder.constants import * +from peach_package_builder.utils import render_template, add_deb_to_freight, update_freight_cache + +# manually update this version when we want to build a new peach-go-sbot package +PEACH_GO_SBOT_VERSION = '0.1.4' + +# constants DEB_CONF_DIR = os.path.join(PROJECT_PATH, 'conf/templates/peach_go_sbot') DEB_BUILD_DIR = "/tmp/peach_go_sbot" GO_SSB_DIR = "/srv/peachcloud/automation/go-ssb" @@ -22,14 +24,13 @@ GO_SSB_DIR = "/srv/peachcloud/automation/go-ssb" def crosscompile_peach_go_sbot(): subprocess.check_call(["git", "pull"], cwd=GO_SSB_DIR) - # TODO: confirm that version number in the repo matches the version number we set for the package we are building print("[CROSS-COMPILING sbotcli]") subprocess.check_call(["env", "GOOS=linux", "GOARCH=arm64", "go", "build", "./cmd/sbotcli"], cwd=GO_SSB_DIR) print("[CROSS-COMPILING go-sbot]") subprocess.check_call(["env", "GOOS=linux", "GOARCH=arm64", "go", "build", "./cmd/go-sbot"], cwd=GO_SSB_DIR) -def package_peach_go_sbot(version): +def package_peach_go_sbot(version): print("[ PACKAGING peach-go-sbot ]") # copy debian conf files into correct locations in package build directory DEBIAN_SRC_DIR = os.path.join(DEB_CONF_DIR, 'DEBIAN') @@ -71,35 +72,31 @@ def package_peach_go_sbot(version): print("[ CREATING {}]".format(deb_file_name)) subprocess.check_call(["dpkg-deb", "-b", ".", deb_file_name], cwd=DEB_BUILD_DIR) - # add deb package to freight - print("[ ADDING PACKAGE TO FREIGHT ]") + # copy deb package to MICROSERVICES_DEB_DIR deb_path = os.path.join(DEB_BUILD_DIR, deb_file_name) - subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, deb_path, "apt/buster"]) - print("[ ADDING PACKAGE TO FREIGHT CACHE ]") - subprocess.call(["sudo", "freight", "cache", "-g", - GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) + subprocess.check_call(["cp", deb_path, MICROSERVICES_DEB_DIR]) + + # add deb package to freight + add_deb_to_freight(package_name=deb_file_name, package_path=deb_path) + + # update freight cache + update_freight_cache() def build_peach_go_sbot(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-v", - "--version", - help="Set version number for go-sbot", - ) - args = parser.parse_args() - - print("[ BUILDING PEACH-GO-SBOT VERSION {}]".format(args.version)) + # gets the most recently built peach_go_sbot version, and increments the micro-number by 1 + version = PEACH_GO_SBOT_VERSION + print("[ BUILDING PEACH-GO-SBOT VERSION {}]".format(version)) # delete build directory if it already exists or create it subprocess.check_call(["rm", "-rf", DEB_BUILD_DIR]) if not os.path.exists(DEB_BUILD_DIR): os.makedirs(DEB_BUILD_DIR) + # cross-compile and package peach-go-sbot with new version number crosscompile_peach_go_sbot() - package_peach_go_sbot(version=args.version) - + package_peach_go_sbot(version=version) if __name__ == '__main__': diff --git a/peach_package_builder/build_rust_packages.py b/peach_package_builder/build_rust_packages.py new file mode 100644 index 0000000..68a821c --- /dev/null +++ b/peach_package_builder/build_rust_packages.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import subprocess +import argparse +import sys +import os + +from peach_package_builder.build_peach_config import build_peach_config +from peach_package_builder.constants import * +from peach_package_builder.utils import add_deb_to_freight, update_freight_cache + + +def add_debs_dir_to_freight(): + """ + adds all packages in MICROSERVICES_DEB_DIR to freight cache + """ + print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") + for package in os.scandir(MICROSERVICES_DEB_DIR): + if package.name.endswith(".deb"): + add_deb_to_freight(package_name=package.name, package_path=package.path) + update_freight_cache() + + +def build_rust_packages(default_branch=False): + """ + builds all PeachCloud microservices written in rust and copies them to MICROSERVICES_DEB_DIR + """ + print("[ BUILDING AND UPDATING RUST MICROSERVICE PACKAGES ]") + for service in SERVICES: + service_name = service["name"] + service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) + print("[ BUILIDING SERVICE {} ]".format(service_name)) + # this arg ensures we build the default branch, otherwise we build what ever is found locally + if default_branch: + # because some repo have main as default and some as master, we get the default + default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'], + cwd=service_path).decode(sys.stdout.encoding) + branch = default_branch.replace('origin/', '').strip() + subprocess.check_call(["git", "checkout", branch], cwd=service_path) + subprocess.check_call(["git", "reset", "HEAD", "--hard"], cwd=service_path) + subprocess.check_call(["git", "pull"], cwd=service_path) + debian_package_path = subprocess.check_output( + [ + CARGO_PATH, + "deb", + "--target", + "aarch64-unknown-linux-gnu"], + cwd=service_path).decode("utf-8").strip() + subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) + + # this function adds all .deb files in MICROSERVICES_DEB_DIR to freight + add_debs_dir_to_freight() + + +def build_packages(default_branch=False): + """ + builds all PeachCloud microservices as .deb files and adds them to the freight repo + """ + build_rust_packages(default_branch=default_branch) + build_peach_config(default_branch=default_branch) + print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", + "--default", + help="Ensure default branch for all repos for build", + action="store_true" + ) + args = parser.parse_args() + build_packages(default_branch=args.default) \ No newline at end of file diff --git a/scripts/constants.py b/peach_package_builder/constants.py similarity index 100% rename from scripts/constants.py rename to peach_package_builder/constants.py diff --git a/scripts/setup_build_env.py b/peach_package_builder/setup_build_env.py similarity index 97% rename from scripts/setup_build_env.py rename to peach_package_builder/setup_build_env.py index ac39439..a0f962f 100644 --- a/scripts/setup_build_env.py +++ b/peach_package_builder/setup_build_env.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -from utils import render_template -from constants import * +from peach_package_builder.utils import render_template +from peach_package_builder.constants import * import subprocess import argparse diff --git a/scripts/utils.py b/peach_package_builder/utils.py similarity index 62% rename from scripts/utils.py rename to peach_package_builder/utils.py index 69f2bdf..49c9259 100644 --- a/scripts/utils.py +++ b/peach_package_builder/utils.py @@ -2,8 +2,8 @@ import os import jinja2 import subprocess -PROJECT_PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -print('PROJECT_PATH: {}'.format(PROJECT_PATH)) +from peach_package_builder.constants import * + template_path = os.path.join(PROJECT_PATH, 'conf/templates') template_loader = jinja2.FileSystemLoader(searchpath=template_path) @@ -25,3 +25,15 @@ def render_template(src, dest, template_vars=None): os.remove(dest) with open(dest, 'w') as f: f.write(output_text) + + +def add_deb_to_freight(package_name, package_path): + print("[ ADDING PACKAGE {} ]".format(package_name)) + subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, package_path, "apt/buster"]) + + +def update_freight_cache(): + print("[ ADDING PACKAGES TO FREIGHT CACHE ]") + # needs to be run as sudo user + subprocess.call(["sudo", "freight", "cache", "-g", + GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 758129b..a4e73ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ Jinja2==2.11.2 +packaging==20.9 \ No newline at end of file diff --git a/scripts/build_packages.py b/scripts/build_packages.py deleted file mode 100644 index 75969c4..0000000 --- a/scripts/build_packages.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -from constants import * - -import subprocess -import argparse -import sys -import os - - -parser = argparse.ArgumentParser() -parser.add_argument( - "-d", - "--default", - help="Ensure default branch for all repos for build", - action="store_true" -) -args = parser.parse_args() - - -print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") -for service in SERVICES: - service_name = service["name"] - service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) - print("[ BUILIDING SERVICE {} ]".format(service_name)) - # this arg ensures we build the default branch, otherwise we build what ever is found locally - if args.default: - # because some repo have main as default and some as master, we get the default - default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'], - cwd=service_path).decode(sys.stdout.encoding) - branch = default_branch.replace('origin/', '').strip() - subprocess.run(["git", "checkout", branch], cwd=service_path) - subprocess.run(["git", "reset", "HEAD", "--hard"], cwd=service_path) - subprocess.run(["git", "pull"], cwd=service_path) - debian_package_path = subprocess.run( - [ - CARGO_PATH, - "deb", - "--target", - "aarch64-unknown-linux-gnu"], - cwd=service_path, - stdout=subprocess.PIPE).stdout.decode("utf-8").strip() - subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) - -print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") -for package in os.scandir(MICROSERVICES_DEB_DIR): - if package.name.endswith(".deb"): - print("[ ADDING PACKAGE {} ]".format(package.name)) - subprocess.call(["freight", "add", "-c", FREIGHT_CONF, - package.path, "apt/buster"]) - -print("[ ADDING PACKAGES TO FREIGHT CACHE ]") -# needs to be run as sudo user -subprocess.call(["sudo", "freight", "cache", "-g", - GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) - -print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]") diff --git a/vpsdeploy.sh b/vpsdeploy.sh new file mode 100755 index 0000000..a68b0f5 --- /dev/null +++ b/vpsdeploy.sh @@ -0,0 +1,6 @@ +#rsync -avzh --delete -e "ssh -i /Users/maxfowler/.ssh/peach_rsa" . notplants@167.99.136.8:/srv/peachcloud/automation/peach-vps +KEY_FILE=/Users/notplants/.ssh/peach_rsa +rsync -avzh --exclude target --exclude .idea --exclude .git --delete -e "ssh -i $KEY_FILE" . notplants@167.99.136.83:/srv/peachcloud/automation/peach-package-builder/ +#ssh -i ./secret_files/do_rsa root@159.89.5.141 +#ssh -i /home/notplants/.ssh/peach_rsa rust@167.99.136.83 'cd /srv/peachcloud/automation/peach-vps/; python3 scripts/build_packages.py' +#echo "cd /srv/src/peach-vps; python3 scripts/setup_vps.py" \ No newline at end of file