diff --git a/scripts/__init__.py b/peach_package_builder/__init__.py similarity index 100% rename from scripts/__init__.py rename to peach_package_builder/__init__.py diff --git a/peach_package_builder/build_packages.py b/peach_package_builder/build_packages.py new file mode 100644 index 0000000..4de9723 --- /dev/null +++ b/peach_package_builder/build_packages.py @@ -0,0 +1,25 @@ +import argparse + +from peach_package_builder.build_rust_packages import build_rust_packages +from peach_package_builder.build_peach_config import build_peach_config + + +def build_packages(default=False): + """ + builds all PeachCloud microservices as .deb files and adds them to the freight repo + """ + build_rust_packages(default_branch=default) + build_peach_config(default_branch=default) + print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", + "--default", + help="Ensure default branch for all repos for build", + action="store_true" + ) + args = parser.parse_args() + build_packages(default=args.default) \ No newline at end of file diff --git a/peach_package_builder/build_peach_config.py b/peach_package_builder/build_peach_config.py new file mode 100644 index 0000000..7a3c18b --- /dev/null +++ b/peach_package_builder/build_peach_config.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +""" +script to build the peach-config debian module and add it to the freight repository +""" + +import argparse +import subprocess +import sys +import os + +from peach_package_builder.constants import * +from peach_package_builder.utils import add_deb_to_freight, update_freight_cache + + +def build_peach_config(default_branch=True): + service_path = os.path.join(MICROSERVICES_SRC_DIR, "peach-config") + if default_branch: + # because some repo have main as default and some as master, we get the default + default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'], + cwd=service_path).decode(sys.stdout.encoding) + branch = default_branch.replace('origin/', '').strip() + subprocess.check_call(["git", "checkout", branch], cwd=service_path) + subprocess.check_call(["git", "reset", "HEAD", "--hard"], cwd=service_path) + subprocess.check_call(["git", "pull"], cwd=service_path) + # build .deb + subprocess.check_call([ + "python3", + "setup.py", + "--command-packages=stdeb.command", + "bdist_deb" + ], + cwd=service_path) + version = "0.2.7" # TODO: get this version number from the repo + deb_name = "python3-peach-config_{version}-1_all.deb".format(version=version) + debian_package_path = os.path.join( + service_path, + "deb_dist/{}".format(deb_name) + ) + subprocess.check_call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) + add_deb_to_freight(package_name=debian_package_path, package_path=debian_package_path) + update_freight_cache() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", + "--default", + help="Ensure default branch for all repos for build", + action="store_true" + ) + args = parser.parse_args() + build_peach_config(default_branch=args.default) \ No newline at end of file diff --git a/scripts/build_peach_go_sbot.py b/peach_package_builder/build_peach_go_sbot.py similarity index 89% rename from scripts/build_peach_go_sbot.py rename to peach_package_builder/build_peach_go_sbot.py index 9660553..cf8350c 100644 --- a/scripts/build_peach_go_sbot.py +++ b/peach_package_builder/build_peach_go_sbot.py @@ -4,17 +4,16 @@ script to create debian packages for cross-compiled go binaries for go-sbot based off of this post https://unix.stackexchange.com/questions/627689/how-to-create-a-debian-package-from-a-bash-script-and-a-systemd-service """ - - -from constants import * -from utils import render_template - import subprocess import argparse import sys import os import shutil +from peach_package_builder.constants import * +from peach_package_builder.utils import render_template, add_deb_to_freight, update_freight_cache + + DEB_CONF_DIR = os.path.join(PROJECT_PATH, 'conf/templates/peach_go_sbot') DEB_BUILD_DIR = "/tmp/peach_go_sbot" GO_SSB_DIR = "/srv/peachcloud/automation/go-ssb" @@ -28,6 +27,7 @@ def crosscompile_peach_go_sbot(): print("[CROSS-COMPILING go-sbot]") subprocess.check_call(["env", "GOOS=linux", "GOARCH=arm64", "go", "build", "./cmd/go-sbot"], cwd=GO_SSB_DIR) + def package_peach_go_sbot(version): print("[ PACKAGING peach-go-sbot ]") @@ -71,13 +71,15 @@ def package_peach_go_sbot(version): print("[ CREATING {}]".format(deb_file_name)) subprocess.check_call(["dpkg-deb", "-b", ".", deb_file_name], cwd=DEB_BUILD_DIR) - # add deb package to freight - print("[ ADDING PACKAGE TO FREIGHT ]") + # copy deb package to MICROSERVICES_DEB_DIR deb_path = os.path.join(DEB_BUILD_DIR, deb_file_name) - subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, deb_path, "apt/buster"]) - print("[ ADDING PACKAGE TO FREIGHT CACHE ]") - subprocess.call(["sudo", "freight", "cache", "-g", - GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) + subprocess.check_call(["cp", deb_path, MICROSERVICES_DEB_DIR]) + + # add deb package to freight + add_deb_to_freight(package_name=deb_file_name, package_path=deb_path) + + # update freight cache + update_freight_cache() def build_peach_go_sbot(): diff --git a/peach_package_builder/build_rust_packages.py b/peach_package_builder/build_rust_packages.py new file mode 100644 index 0000000..1a8d1e1 --- /dev/null +++ b/peach_package_builder/build_rust_packages.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import subprocess +import argparse +import sys +import os + +from peach_package_builder.build_peach_config import build_peach_config +from peach_package_builder.constants import * +from peach_package_builder.utils import add_deb_to_freight, update_freight_cache + + +def add_debs_dir_to_freight(): + """ + adds all packages in MICROSERVICES_DEB_DIR to freight cache + """ + print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") + for package in os.scandir(MICROSERVICES_DEB_DIR): + if package.name.endswith(".deb"): + add_deb_to_freight(package_name=package.name, package_path=package.path) + update_freight_cache() + + +def build_rust_packages(default_branch=False): + """ + builds all PeachCloud microservices written in rust and copies them to MICROSERVICES_DEB_DIR + """ + print("[ BUILDING AND UPDATING RUST MICROSERVICE PACKAGES ]") + for service in SERVICES: + service_name = service["name"] + service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) + print("[ BUILIDING SERVICE {} ]".format(service_name)) + # this arg ensures we build the default branch, otherwise we build what ever is found locally + if default_branch: + # because some repo have main as default and some as master, we get the default + default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'], + cwd=service_path).decode(sys.stdout.encoding) + branch = default_branch.replace('origin/', '').strip() + subprocess.run(["git", "checkout", branch], cwd=service_path) + subprocess.run(["git", "reset", "HEAD", "--hard"], cwd=service_path) + subprocess.run(["git", "pull"], cwd=service_path) + debian_package_path = subprocess.run( + [ + CARGO_PATH, + "deb", + "--target", + "aarch64-unknown-linux-gnu"], + cwd=service_path, + stdout=subprocess.PIPE).stdout.decode("utf-8").strip() + subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) + + # this function adds all .deb files in MICROSERVICES_DEB_DIR to freight + add_debs_dir_to_freight() + + +def build_packages(default=False): + """ + builds all PeachCloud microservices as .deb files and adds them to the freight repo + """ + build_rust_packages(default_branch=default) + build_peach_config(default_branch=default) + print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", + "--default", + help="Ensure default branch for all repos for build", + action="store_true" + ) + args = parser.parse_args() + build_packages(default=args.default) \ No newline at end of file diff --git a/scripts/constants.py b/peach_package_builder/constants.py similarity index 100% rename from scripts/constants.py rename to peach_package_builder/constants.py diff --git a/scripts/setup_build_env.py b/peach_package_builder/setup_build_env.py similarity index 97% rename from scripts/setup_build_env.py rename to peach_package_builder/setup_build_env.py index ac39439..a0f962f 100644 --- a/scripts/setup_build_env.py +++ b/peach_package_builder/setup_build_env.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -from utils import render_template -from constants import * +from peach_package_builder.utils import render_template +from peach_package_builder.constants import * import subprocess import argparse diff --git a/scripts/utils.py b/peach_package_builder/utils.py similarity index 62% rename from scripts/utils.py rename to peach_package_builder/utils.py index 69f2bdf..49c9259 100644 --- a/scripts/utils.py +++ b/peach_package_builder/utils.py @@ -2,8 +2,8 @@ import os import jinja2 import subprocess -PROJECT_PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -print('PROJECT_PATH: {}'.format(PROJECT_PATH)) +from peach_package_builder.constants import * + template_path = os.path.join(PROJECT_PATH, 'conf/templates') template_loader = jinja2.FileSystemLoader(searchpath=template_path) @@ -25,3 +25,15 @@ def render_template(src, dest, template_vars=None): os.remove(dest) with open(dest, 'w') as f: f.write(output_text) + + +def add_deb_to_freight(package_name, package_path): + print("[ ADDING PACKAGE {} ]".format(package_name)) + subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, package_path, "apt/buster"]) + + +def update_freight_cache(): + print("[ ADDING PACKAGES TO FREIGHT CACHE ]") + # needs to be run as sudo user + subprocess.call(["sudo", "freight", "cache", "-g", + GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) \ No newline at end of file diff --git a/scripts/build_packages.py b/scripts/build_packages.py deleted file mode 100644 index 75969c4..0000000 --- a/scripts/build_packages.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -from constants import * - -import subprocess -import argparse -import sys -import os - - -parser = argparse.ArgumentParser() -parser.add_argument( - "-d", - "--default", - help="Ensure default branch for all repos for build", - action="store_true" -) -args = parser.parse_args() - - -print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]") -for service in SERVICES: - service_name = service["name"] - service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name) - print("[ BUILIDING SERVICE {} ]".format(service_name)) - # this arg ensures we build the default branch, otherwise we build what ever is found locally - if args.default: - # because some repo have main as default and some as master, we get the default - default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'], - cwd=service_path).decode(sys.stdout.encoding) - branch = default_branch.replace('origin/', '').strip() - subprocess.run(["git", "checkout", branch], cwd=service_path) - subprocess.run(["git", "reset", "HEAD", "--hard"], cwd=service_path) - subprocess.run(["git", "pull"], cwd=service_path) - debian_package_path = subprocess.run( - [ - CARGO_PATH, - "deb", - "--target", - "aarch64-unknown-linux-gnu"], - cwd=service_path, - stdout=subprocess.PIPE).stdout.decode("utf-8").strip() - subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR]) - -print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]") -for package in os.scandir(MICROSERVICES_DEB_DIR): - if package.name.endswith(".deb"): - print("[ ADDING PACKAGE {} ]".format(package.name)) - subprocess.call(["freight", "add", "-c", FREIGHT_CONF, - package.path, "apt/buster"]) - -print("[ ADDING PACKAGES TO FREIGHT CACHE ]") -# needs to be run as sudo user -subprocess.call(["sudo", "freight", "cache", "-g", - GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE]) - -print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]") diff --git a/vpsdeploy.sh b/vpsdeploy.sh new file mode 100755 index 0000000..a68b0f5 --- /dev/null +++ b/vpsdeploy.sh @@ -0,0 +1,6 @@ +#rsync -avzh --delete -e "ssh -i /Users/maxfowler/.ssh/peach_rsa" . notplants@167.99.136.8:/srv/peachcloud/automation/peach-vps +KEY_FILE=/Users/notplants/.ssh/peach_rsa +rsync -avzh --exclude target --exclude .idea --exclude .git --delete -e "ssh -i $KEY_FILE" . notplants@167.99.136.83:/srv/peachcloud/automation/peach-package-builder/ +#ssh -i ./secret_files/do_rsa root@159.89.5.141 +#ssh -i /home/notplants/.ssh/peach_rsa rust@167.99.136.83 'cd /srv/peachcloud/automation/peach-vps/; python3 scripts/build_packages.py' +#echo "cd /srv/src/peach-vps; python3 scripts/setup_vps.py" \ No newline at end of file