Merge pull request #13 from peachcloud/build-peach-config

Build peach-config and refactor build scripts to share code

README.md

@@ -1,12 +1,12 @@
-# peach-vps
+# peach-package-builder
 
-![Generic badge](https://img.shields.io/badge/version-0.3.1-<COLOR>.svg)
+![Generic badge](https://img.shields.io/badge/version-0.3.2-<COLOR>.svg)
 
-Scripts for configuring the PeachCloud VPS for various hosting and automation functions.
+Scripts for building debian packages for PeachCloud microservices.
 
 ## Setup Build Environment
 
-`scripts/setup_build_env.py`
+`python3 peach_package_builder/setup_build_env.py`
 
 An idempotent script for initializing a build and deployment environment for PeachCloud packages.
 
@@ -35,7 +35,7 @@ cd peach-vps
 pip3 install -r requirements.txt
 ```
 
-Open `scripts/setup_build_env.py` and set the following constants:
+Open `peach_package_builder/setup_build_env.py` and set the following constants:
 
  - USER_PATH
  - GPG_KEY_EMAIL
@@ -44,34 +44,51 @@ Open `scripts/setup_build_env.py` and set the following constants:
 Then execute the script to run the full system initialization process (_note: several commands executed by the script require `sudo` permissions. You will be prompted for the user password during the execution of the scipt._):
 
 ```
-python3 -u scripts/setup_build_env.py
+python3 -u peach_package_builder/setup_build_env.py
 ```
 
-## Build and Serve Debian Packages
+## Build packages
 
-`scripts/build_packages.py`
+`peach_package_builder/build_packages.py`
 
 An idempotent script for building the latest versions of all PeachCloud packages and adding them to the Debian package archive.
 
 The script currently performs the following actions:
 
- - Builds and updates microservice packages
+ - Builds and updates Rust microservice packages
+ - Builds and updates peach-config python package 
  - Adds packages to Freight library
  - Adds packages to Freight cache
 
 ```
-python3 -u scripts/build_packages.py
+python3 -d peach_package_builder/build_packages.py
 ```
 
+The -d flag ensures that all packages are built from the latest version of the default branch currently on GitHub. 
+Without the -d flag, whatever version of the code is locally stored will be used (which can be useful for testing). 
+
 Freight supports the ability to have multiple versions of a package in a single Debian package archive. If a particular version of a package already exists in the Freight library, it will not be readded or overwritten.
 
-## Build peach-go-sbot Debian package
+You can also just build the rust packages by running:
+```python3 -d peach_package_builder/build_rust_packages.py```
 
-`sudo python3 scripts/build_peach_go_sbot.py -v <version_number>`
+Or just build peach-config by running:
+```python3 -d peach_package_builder/build_peach_config.py```
+
+
+## Build peach-go-sbot package
+
+First, open peach_package_builder/build_peach_go_sbot.py and manually edit PEACH_GO_SBOT_VERSION. 
+
+We manually increment the version number when we want to build a new version of peach-go-sbot. 
+
+Then run,
+`python3 peach_package_builder/build_peach_go_sbot.py`
 
 This builds the peach-go-sbot package using the latest code from go-ssb, along with a systemd unit file,
 and adds the Debian package to the Freight library.
 
+
 ## Install Packages from Debian Package Archive
 
 To add the PeachCloud Debian package archive as an apt source, run the following commands from your Pi:

peach_package_builder/build_packages.py

@@ -0,0 +1,25 @@
+import argparse
+
+from peach_package_builder.build_rust_packages import build_rust_packages
+from peach_package_builder.build_peach_config import build_peach_config
+
+
+def build_packages(default_branch=False):
+    """
+    builds all PeachCloud microservices as .deb files and adds them to the freight repo
+    """
+    build_rust_packages(default_branch=default_branch)
+    build_peach_config(default_branch=default_branch)
+    print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d",
+        "--default",
+        help="Ensure default branch for all repos for build",
+        action="store_true"
+    )
+    args = parser.parse_args()
+    build_packages(default_branch=args.default)

peach_package_builder/build_peach_config.py

@@ -0,0 +1,57 @@
+"""
+script to build the peach-config debian module and add it to the freight repository
+"""
+import argparse
+import subprocess
+import sys
+import os
+
+from peach_package_builder.constants import *
+from peach_package_builder.utils import add_deb_to_freight, update_freight_cache
+
+
+def build_peach_config(default_branch=True):
+    service_path = os.path.join(MICROSERVICES_SRC_DIR, "peach-config")
+    if default_branch:
+        # because some repo have main as default and some as master, we get the default
+        default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'],
+                                                 cwd=service_path).decode(sys.stdout.encoding)
+        branch = default_branch.replace('origin/', '').strip()
+        subprocess.check_call(["git", "checkout", branch], cwd=service_path)
+        subprocess.check_call(["git", "reset", "HEAD", "--hard"], cwd=service_path)
+        subprocess.check_call(["git", "pull"], cwd=service_path)
+    # remove old build dir
+    subprocess.check_call([
+            "rm",
+            "-rf",
+            os.path.join(service_path, 'deb_dist')
+    ])
+    # build .deb
+    subprocess.check_call([
+            "python3",
+            "setup.py",
+            "--command-packages=stdeb.command",
+            "bdist_deb"
+        ],
+        cwd=service_path)
+    version = "0.2.7"   # TODO: get this version number from the repo
+    deb_name = "python3-peach-config_{version}-1_all.deb".format(version=version)
+    debian_package_path = os.path.join(
+        service_path,
+        "deb_dist/{}".format(deb_name)
+    )
+    subprocess.check_call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
+    add_deb_to_freight(package_name=debian_package_path, package_path=debian_package_path)
+    update_freight_cache()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d",
+        "--default",
+        help="Ensure default branch for all repos for build",
+        action="store_true"
+    )
+    args = parser.parse_args()
+    build_peach_config(default_branch=args.default)

peach_package_builder/build_peach_go_sbot.py

@@ -1,20 +1,22 @@
-#!/usr/bin/env python3
 """
 script to create debian packages for cross-compiled go binaries for go-sbot
 based off of this post
 https://unix.stackexchange.com/questions/627689/how-to-create-a-debian-package-from-a-bash-script-and-a-systemd-service
 """
-
-
-from constants import *
-from utils import render_template
-
 import subprocess
 import argparse
-import sys
+import re
-import os
 import shutil
+import sys
+from packaging import version as pversion
 
+from peach_package_builder.constants import *
+from peach_package_builder.utils import render_template, add_deb_to_freight, update_freight_cache
+
+# manually update this version when we want to build a new peach-go-sbot package
+PEACH_GO_SBOT_VERSION = '0.1.4'
+
+# constants
 DEB_CONF_DIR = os.path.join(PROJECT_PATH, 'conf/templates/peach_go_sbot')
 DEB_BUILD_DIR = "/tmp/peach_go_sbot"
 GO_SSB_DIR = "/srv/peachcloud/automation/go-ssb"
@@ -22,14 +24,13 @@ GO_SSB_DIR = "/srv/peachcloud/automation/go-ssb"
 
 def crosscompile_peach_go_sbot():
     subprocess.check_call(["git", "pull"], cwd=GO_SSB_DIR)
-    # TODO: confirm that version number in the repo matches the version number we set for the package we are building
     print("[CROSS-COMPILING sbotcli]")
     subprocess.check_call(["env", "GOOS=linux", "GOARCH=arm64", "go", "build", "./cmd/sbotcli"], cwd=GO_SSB_DIR)
     print("[CROSS-COMPILING go-sbot]")
     subprocess.check_call(["env", "GOOS=linux", "GOARCH=arm64", "go", "build", "./cmd/go-sbot"], cwd=GO_SSB_DIR)
 
-def package_peach_go_sbot(version):
 
+def package_peach_go_sbot(version):
     print("[ PACKAGING peach-go-sbot ]")
     # copy debian conf files into correct locations in package build directory
     DEBIAN_SRC_DIR = os.path.join(DEB_CONF_DIR, 'DEBIAN')
@@ -71,35 +72,31 @@ def package_peach_go_sbot(version):
     print("[ CREATING {}]".format(deb_file_name))
     subprocess.check_call(["dpkg-deb", "-b", ".", deb_file_name], cwd=DEB_BUILD_DIR)
 
-    # add deb package to freight
+    # copy deb package to MICROSERVICES_DEB_DIR
-    print("[ ADDING PACKAGE TO FREIGHT ]")
     deb_path = os.path.join(DEB_BUILD_DIR, deb_file_name)
-    subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, deb_path, "apt/buster"])
+    subprocess.check_call(["cp", deb_path, MICROSERVICES_DEB_DIR])
-    print("[ ADDING PACKAGE TO FREIGHT CACHE ]")
+
-    subprocess.call(["sudo", "freight", "cache", "-g",
+    # add deb package to freight
-                     GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])
+    add_deb_to_freight(package_name=deb_file_name, package_path=deb_path)
+
+    # update freight cache
+    update_freight_cache()
 
 
 def build_peach_go_sbot():
 
-    parser = argparse.ArgumentParser()
+    # gets the most recently built peach_go_sbot version, and increments the micro-number by 1
-    parser.add_argument(
+    version = PEACH_GO_SBOT_VERSION
-        "-v",
+    print("[ BUILDING PEACH-GO-SBOT VERSION {}]".format(version))
-        "--version",
-        help="Set version number for go-sbot",
-    )
-    args = parser.parse_args()
-
-    print("[ BUILDING PEACH-GO-SBOT VERSION {}]".format(args.version))
 
     # delete build directory if it already exists or create it
     subprocess.check_call(["rm", "-rf", DEB_BUILD_DIR])
     if not os.path.exists(DEB_BUILD_DIR):
         os.makedirs(DEB_BUILD_DIR)
 
+    # cross-compile and package peach-go-sbot with new version number
     crosscompile_peach_go_sbot()
-    package_peach_go_sbot(version=args.version)
+    package_peach_go_sbot(version=version)
-
 
 
 if __name__ == '__main__':

peach_package_builder/build_rust_packages.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import subprocess
+import argparse
+import sys
+import os
+
+from peach_package_builder.build_peach_config import build_peach_config
+from peach_package_builder.constants import *
+from peach_package_builder.utils import add_deb_to_freight, update_freight_cache
+
+
+def add_debs_dir_to_freight():
+    """
+    adds all packages in MICROSERVICES_DEB_DIR to freight cache
+    """
+    print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]")
+    for package in os.scandir(MICROSERVICES_DEB_DIR):
+        if package.name.endswith(".deb"):
+            add_deb_to_freight(package_name=package.name, package_path=package.path)
+    update_freight_cache()
+
+
+def build_rust_packages(default_branch=False):
+    """
+    builds all PeachCloud microservices written in rust and copies them to MICROSERVICES_DEB_DIR
+    """
+    print("[ BUILDING AND UPDATING RUST MICROSERVICE PACKAGES ]")
+    for service in SERVICES:
+        service_name = service["name"]
+        service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name)
+        print("[ BUILIDING SERVICE {} ]".format(service_name))
+        # this arg ensures we build the default branch, otherwise we build what ever is found locally
+        if default_branch:
+            # because some repo have main as default and some as master, we get the default
+            default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'],
+                                                     cwd=service_path).decode(sys.stdout.encoding)
+            branch = default_branch.replace('origin/', '').strip()
+            subprocess.check_call(["git", "checkout", branch], cwd=service_path)
+            subprocess.check_call(["git", "reset", "HEAD", "--hard"], cwd=service_path)
+            subprocess.check_call(["git", "pull"], cwd=service_path)
+        debian_package_path = subprocess.check_output(
+            [
+                CARGO_PATH,
+                "deb",
+                "--target",
+                "aarch64-unknown-linux-gnu"],
+            cwd=service_path).decode("utf-8").strip()
+        subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
+
+    # this function adds all .deb files in MICROSERVICES_DEB_DIR to freight
+    add_debs_dir_to_freight()
+
+
+def build_packages(default_branch=False):
+    """
+    builds all PeachCloud microservices as .deb files and adds them to the freight repo
+    """
+    build_rust_packages(default_branch=default_branch)
+    build_peach_config(default_branch=default_branch)
+    print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d",
+        "--default",
+        help="Ensure default branch for all repos for build",
+        action="store_true"
+    )
+    args = parser.parse_args()
+    build_packages(default_branch=args.default)

peach_package_builder/setup_build_env.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-from utils import render_template
+from peach_package_builder.utils import render_template
-from constants import *
+from peach_package_builder.constants import *
 
 import subprocess
 import argparse

peach_package_builder/utils.py

@@ -2,8 +2,8 @@ import os
 import jinja2
 import subprocess
 
-PROJECT_PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+from peach_package_builder.constants import *
-print('PROJECT_PATH: {}'.format(PROJECT_PATH))
+
 
 template_path = os.path.join(PROJECT_PATH, 'conf/templates')
 template_loader = jinja2.FileSystemLoader(searchpath=template_path)
@@ -25,3 +25,15 @@ def render_template(src, dest, template_vars=None):
         os.remove(dest)
     with open(dest, 'w') as f:
         f.write(output_text)
+
+
+def add_deb_to_freight(package_name, package_path):
+    print("[ ADDING PACKAGE {} ]".format(package_name))
+    subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, package_path, "apt/buster"])
+
+
+def update_freight_cache():
+    print("[ ADDING PACKAGES TO FREIGHT CACHE ]")
+    # needs to be run as sudo user
+    subprocess.call(["sudo", "freight", "cache", "-g",
+                     GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])

requirements.txt

@@ -1 +1,2 @@
 Jinja2==2.11.2
+packaging==20.9

scripts/build_packages.py

@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-from constants import *
-
-import subprocess
-import argparse
-import sys
-import os
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "-d",
-    "--default",
-    help="Ensure default branch for all repos for build",
-    action="store_true"
-)
-args = parser.parse_args()
-
-
-print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]")
-for service in SERVICES:
-    service_name = service["name"]
-    service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name)
-    print("[ BUILIDING SERVICE {} ]".format(service_name))
-    # this arg ensures we build the default branch, otherwise we build what ever is found locally
-    if args.default:
-        # because some repo have main as default and some as master, we get the default
-        default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'],
-                                                 cwd=service_path).decode(sys.stdout.encoding)
-        branch = default_branch.replace('origin/', '').strip()
-        subprocess.run(["git", "checkout", branch], cwd=service_path)
-        subprocess.run(["git", "reset", "HEAD", "--hard"], cwd=service_path)
-        subprocess.run(["git", "pull"], cwd=service_path)
-    debian_package_path = subprocess.run(
-        [
-            CARGO_PATH,
-            "deb",
-            "--target",
-            "aarch64-unknown-linux-gnu"],
-        cwd=service_path,
-        stdout=subprocess.PIPE).stdout.decode("utf-8").strip()
-    subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
-
-print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]")
-for package in os.scandir(MICROSERVICES_DEB_DIR):
-    if package.name.endswith(".deb"):
-        print("[ ADDING PACKAGE {} ]".format(package.name))
-        subprocess.call(["freight", "add", "-c", FREIGHT_CONF,
-                         package.path, "apt/buster"])
-
-print("[ ADDING PACKAGES TO FREIGHT CACHE ]")
-# needs to be run as sudo user
-subprocess.call(["sudo", "freight", "cache", "-g",
-                 GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])
-
-print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]")

vpsdeploy.sh

@@ -0,0 +1,6 @@
+#rsync -avzh --delete -e "ssh -i /Users/maxfowler/.ssh/peach_rsa" . notplants@167.99.136.8:/srv/peachcloud/automation/peach-vps
+KEY_FILE=/Users/notplants/.ssh/peach_rsa
+rsync -avzh --exclude target --exclude .idea --exclude .git --delete -e "ssh -i $KEY_FILE" . notplants@167.99.136.83:/srv/peachcloud/automation/peach-package-builder/
+#ssh -i ./secret_files/do_rsa root@159.89.5.141
+#ssh -i /home/notplants/.ssh/peach_rsa rust@167.99.136.83 'cd /srv/peachcloud/automation/peach-vps/; python3 scripts/build_packages.py'
+#echo "cd /srv/src/peach-vps; python3 scripts/setup_vps.py"