Refactor peach-package-builder to build peach-config and peach-go-sbot and rust packages

peach_package_builder/build_packages.py

@@ -0,0 +1,25 @@
+import argparse
+
+from peach_package_builder.build_rust_packages import build_rust_packages
+from peach_package_builder.build_peach_config import build_peach_config
+
+
+def build_packages(default=False):
+    """
+    builds all PeachCloud microservices as .deb files and adds them to the freight repo
+    """
+    build_rust_packages(default_branch=default)
+    build_peach_config(default_branch=default)
+    print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d",
+        "--default",
+        help="Ensure default branch for all repos for build",
+        action="store_true"
+    )
+    args = parser.parse_args()
+    build_packages(default=args.default)

peach_package_builder/build_peach_config.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""
+script to build the peach-config debian module and add it to the freight repository
+"""
+
+import argparse
+import subprocess
+import sys
+import os
+
+from peach_package_builder.constants import *
+from peach_package_builder.utils import add_deb_to_freight, update_freight_cache
+
+
+def build_peach_config(default_branch=True):
+    service_path = os.path.join(MICROSERVICES_SRC_DIR, "peach-config")
+    if default_branch:
+        # because some repo have main as default and some as master, we get the default
+        default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'],
+                                                 cwd=service_path).decode(sys.stdout.encoding)
+        branch = default_branch.replace('origin/', '').strip()
+        subprocess.check_call(["git", "checkout", branch], cwd=service_path)
+        subprocess.check_call(["git", "reset", "HEAD", "--hard"], cwd=service_path)
+        subprocess.check_call(["git", "pull"], cwd=service_path)
+    # build .deb
+    subprocess.check_call([
+            "python3",
+            "setup.py",
+            "--command-packages=stdeb.command",
+            "bdist_deb"
+        ],
+        cwd=service_path)
+    version = "0.2.7"   # TODO: get this version number from the repo
+    deb_name = "python3-peach-config_{version}-1_all.deb".format(version=version)
+    debian_package_path = os.path.join(
+        service_path,
+        "deb_dist/{}".format(deb_name)
+    )
+    subprocess.check_call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
+    add_deb_to_freight(package_name=debian_package_path, package_path=debian_package_path)
+    update_freight_cache()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d",
+        "--default",
+        help="Ensure default branch for all repos for build",
+        action="store_true"
+    )
+    args = parser.parse_args()
+    build_peach_config(default_branch=args.default)

peach_package_builder/build_peach_go_sbot.py

@@ -4,17 +4,16 @@ script to create debian packages for cross-compiled go binaries for go-sbot
 based off of this post
 https://unix.stackexchange.com/questions/627689/how-to-create-a-debian-package-from-a-bash-script-and-a-systemd-service
 """
-
-
-from constants import *
-from utils import render_template
-
 import subprocess
 import argparse
 import sys
 import os
 import shutil
 
+from peach_package_builder.constants import *
+from peach_package_builder.utils import render_template, add_deb_to_freight, update_freight_cache
+
+
 DEB_CONF_DIR = os.path.join(PROJECT_PATH, 'conf/templates/peach_go_sbot')
 DEB_BUILD_DIR = "/tmp/peach_go_sbot"
 GO_SSB_DIR = "/srv/peachcloud/automation/go-ssb"
@@ -28,6 +27,7 @@ def crosscompile_peach_go_sbot():
     print("[CROSS-COMPILING go-sbot]")
     subprocess.check_call(["env", "GOOS=linux", "GOARCH=arm64", "go", "build", "./cmd/go-sbot"], cwd=GO_SSB_DIR)
 
+
 def package_peach_go_sbot(version):
 
     print("[ PACKAGING peach-go-sbot ]")
@@ -71,13 +71,15 @@ def package_peach_go_sbot(version):
     print("[ CREATING {}]".format(deb_file_name))
     subprocess.check_call(["dpkg-deb", "-b", ".", deb_file_name], cwd=DEB_BUILD_DIR)
 
-    # add deb package to freight
-    print("[ ADDING PACKAGE TO FREIGHT ]")
+    # copy deb package to MICROSERVICES_DEB_DIR
     deb_path = os.path.join(DEB_BUILD_DIR, deb_file_name)
-    subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, deb_path, "apt/buster"])
-    print("[ ADDING PACKAGE TO FREIGHT CACHE ]")
-    subprocess.call(["sudo", "freight", "cache", "-g",
-                     GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])
+    subprocess.check_call(["cp", deb_path, MICROSERVICES_DEB_DIR])
+
+    # add deb package to freight
+    add_deb_to_freight(package_name=deb_file_name, package_path=deb_path)
+
+    # update freight cache
+    update_freight_cache()
 
 
 def build_peach_go_sbot():

peach_package_builder/build_rust_packages.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+import subprocess
+import argparse
+import sys
+import os
+
+from peach_package_builder.build_peach_config import build_peach_config
+from peach_package_builder.constants import *
+from peach_package_builder.utils import add_deb_to_freight, update_freight_cache
+
+
+def add_debs_dir_to_freight():
+    """
+    adds all packages in MICROSERVICES_DEB_DIR to freight cache
+    """
+    print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]")
+    for package in os.scandir(MICROSERVICES_DEB_DIR):
+        if package.name.endswith(".deb"):
+            add_deb_to_freight(package_name=package.name, package_path=package.path)
+    update_freight_cache()
+
+
+def build_rust_packages(default_branch=False):
+    """
+    builds all PeachCloud microservices written in rust and copies them to MICROSERVICES_DEB_DIR
+    """
+    print("[ BUILDING AND UPDATING RUST MICROSERVICE PACKAGES ]")
+    for service in SERVICES:
+        service_name = service["name"]
+        service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name)
+        print("[ BUILIDING SERVICE {} ]".format(service_name))
+        # this arg ensures we build the default branch, otherwise we build what ever is found locally
+        if default_branch:
+            # because some repo have main as default and some as master, we get the default
+            default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'],
+                                                     cwd=service_path).decode(sys.stdout.encoding)
+            branch = default_branch.replace('origin/', '').strip()
+            subprocess.run(["git", "checkout", branch], cwd=service_path)
+            subprocess.run(["git", "reset", "HEAD", "--hard"], cwd=service_path)
+            subprocess.run(["git", "pull"], cwd=service_path)
+        debian_package_path = subprocess.run(
+            [
+                CARGO_PATH,
+                "deb",
+                "--target",
+                "aarch64-unknown-linux-gnu"],
+            cwd=service_path,
+            stdout=subprocess.PIPE).stdout.decode("utf-8").strip()
+        subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
+
+        # this function adds all .deb files in MICROSERVICES_DEB_DIR to freight
+        add_debs_dir_to_freight()
+
+
+def build_packages(default=False):
+    """
+    builds all PeachCloud microservices as .deb files and adds them to the freight repo
+    """
+    build_rust_packages(default_branch=default)
+    build_peach_config(default_branch=default)
+    print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d",
+        "--default",
+        help="Ensure default branch for all repos for build",
+        action="store_true"
+    )
+    args = parser.parse_args()
+    build_packages(default=args.default)

peach_package_builder/setup_build_env.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-from utils import render_template
-from constants import *
+from peach_package_builder.utils import render_template
+from peach_package_builder.constants import *
 
 import subprocess
 import argparse

peach_package_builder/utils.py

@@ -2,8 +2,8 @@ import os
 import jinja2
 import subprocess
 
-PROJECT_PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-print('PROJECT_PATH: {}'.format(PROJECT_PATH))
+from peach_package_builder.constants import *
+
 
 template_path = os.path.join(PROJECT_PATH, 'conf/templates')
 template_loader = jinja2.FileSystemLoader(searchpath=template_path)
@@ -25,3 +25,15 @@ def render_template(src, dest, template_vars=None):
         os.remove(dest)
     with open(dest, 'w') as f:
         f.write(output_text)
+
+
+def add_deb_to_freight(package_name, package_path):
+    print("[ ADDING PACKAGE {} ]".format(package_name))
+    subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, package_path, "apt/buster"])
+
+
+def update_freight_cache():
+    print("[ ADDING PACKAGES TO FREIGHT CACHE ]")
+    # needs to be run as sudo user
+    subprocess.call(["sudo", "freight", "cache", "-g",
+                     GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])

scripts/build_packages.py

@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-from constants import *
-
-import subprocess
-import argparse
-import sys
-import os
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "-d",
-    "--default",
-    help="Ensure default branch for all repos for build",
-    action="store_true"
-)
-args = parser.parse_args()
-
-
-print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]")
-for service in SERVICES:
-    service_name = service["name"]
-    service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name)
-    print("[ BUILIDING SERVICE {} ]".format(service_name))
-    # this arg ensures we build the default branch, otherwise we build what ever is found locally
-    if args.default:
-        # because some repo have main as default and some as master, we get the default
-        default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'],
-                                                 cwd=service_path).decode(sys.stdout.encoding)
-        branch = default_branch.replace('origin/', '').strip()
-        subprocess.run(["git", "checkout", branch], cwd=service_path)
-        subprocess.run(["git", "reset", "HEAD", "--hard"], cwd=service_path)
-        subprocess.run(["git", "pull"], cwd=service_path)
-    debian_package_path = subprocess.run(
-        [
-            CARGO_PATH,
-            "deb",
-            "--target",
-            "aarch64-unknown-linux-gnu"],
-        cwd=service_path,
-        stdout=subprocess.PIPE).stdout.decode("utf-8").strip()
-    subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
-
-print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]")
-for package in os.scandir(MICROSERVICES_DEB_DIR):
-    if package.name.endswith(".deb"):
-        print("[ ADDING PACKAGE {} ]".format(package.name))
-        subprocess.call(["freight", "add", "-c", FREIGHT_CONF,
-                         package.path, "apt/buster"])
-
-print("[ ADDING PACKAGES TO FREIGHT CACHE ]")
-# needs to be run as sudo user
-subprocess.call(["sudo", "freight", "cache", "-g",
-                 GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])
-
-print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]")

vpsdeploy.sh

@@ -0,0 +1,6 @@
+#rsync -avzh --delete -e "ssh -i /Users/maxfowler/.ssh/peach_rsa" . notplants@167.99.136.8:/srv/peachcloud/automation/peach-vps
+KEY_FILE=/Users/notplants/.ssh/peach_rsa
+rsync -avzh --exclude target --exclude .idea --exclude .git --delete -e "ssh -i $KEY_FILE" . notplants@167.99.136.83:/srv/peachcloud/automation/peach-package-builder/
+#ssh -i ./secret_files/do_rsa root@159.89.5.141
+#ssh -i /home/notplants/.ssh/peach_rsa rust@167.99.136.83 'cd /srv/peachcloud/automation/peach-vps/; python3 scripts/build_packages.py'
+#echo "cd /srv/src/peach-vps; python3 scripts/setup_vps.py"