Merge pull request #13 from peachcloud/build-peach-config
Build peach-config and refactor build scripts to share code
This commit is contained in:
Max Fowler
GitHub
commit
38dc151434
41
README.md
41
README.md
|
|
@ -1,12 +1,12 @@
|
|||
# peach-vps
|
||||
# peach-package-builder
|
||||
|
||||
|
||||
|
||||
|
||||
Scripts for configuring the PeachCloud VPS for various hosting and automation functions.
|
||||
Scripts for building debian packages for PeachCloud microservices.
|
||||
|
||||
## Setup Build Environment
|
||||
|
||||
`scripts/setup_build_env.py`
|
||||
`python3 peach_package_builder/setup_build_env.py`
|
||||
|
||||
An idempotent script for initializing a build and deployment environment for PeachCloud packages.
|
||||
|
||||
|
|
@ -35,7 +35,7 @@ cd peach-vps
|
|||
pip3 install -r requirements.txt
|
||||
```
|
||||
|
||||
Open `scripts/setup_build_env.py` and set the following constants:
|
||||
Open `peach_package_builder/setup_build_env.py` and set the following constants:
|
||||
|
||||
- USER_PATH
|
||||
- GPG_KEY_EMAIL
|
||||
|
|
@ -44,34 +44,51 @@ Open `scripts/setup_build_env.py` and set the following constants:
|
|||
Then execute the script to run the full system initialization process (_note: several commands executed by the script require `sudo` permissions. You will be prompted for the user password during the execution of the scipt._):
|
||||
|
||||
```
|
||||
python3 -u scripts/setup_build_env.py
|
||||
python3 -u peach_package_builder/setup_build_env.py
|
||||
```
|
||||
|
||||
## Build and Serve Debian Packages
|
||||
## Build packages
|
||||
|
||||
`scripts/build_packages.py`
|
||||
`peach_package_builder/build_packages.py`
|
||||
|
||||
An idempotent script for building the latest versions of all PeachCloud packages and adding them to the Debian package archive.
|
||||
|
||||
The script currently performs the following actions:
|
||||
|
||||
- Builds and updates microservice packages
|
||||
- Builds and updates Rust microservice packages
|
||||
- Builds and updates peach-config python package
|
||||
- Adds packages to Freight library
|
||||
- Adds packages to Freight cache
|
||||
|
||||
```
|
||||
python3 -u scripts/build_packages.py
|
||||
python3 -d peach_package_builder/build_packages.py
|
||||
```
|
||||
|
||||
The -d flag ensures that all packages are built from the latest version of the default branch currently on GitHub.
|
||||
Without the -d flag, whatever version of the code is locally stored will be used (which can be useful for testing).
|
||||
|
||||
Freight supports the ability to have multiple versions of a package in a single Debian package archive. If a particular version of a package already exists in the Freight library, it will not be readded or overwritten.
|
||||
|
||||
## Build peach-go-sbot Debian package
|
||||
You can also just build the rust packages by running:
|
||||
```python3 -d peach_package_builder/build_rust_packages.py```
|
||||
|
||||
`sudo python3 scripts/build_peach_go_sbot.py -v <version_number>`
|
||||
Or just build peach-config by running:
|
||||
```python3 -d peach_package_builder/build_peach_config.py```
|
||||
|
||||
|
||||
## Build peach-go-sbot package
|
||||
|
||||
First, open peach_package_builder/build_peach_go_sbot.py and manually edit PEACH_GO_SBOT_VERSION.
|
||||
|
||||
We manually increment the version number when we want to build a new version of peach-go-sbot.
|
||||
|
||||
Then run,
|
||||
`python3 peach_package_builder/build_peach_go_sbot.py`
|
||||
|
||||
This builds the peach-go-sbot package using the latest code from go-ssb, along with a systemd unit file,
|
||||
and adds the Debian package to the Freight library.
|
||||
|
||||
|
||||
## Install Packages from Debian Package Archive
|
||||
|
||||
To add the PeachCloud Debian package archive as an apt source, run the following commands from your Pi:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
import argparse
|
||||
|
||||
from peach_package_builder.build_rust_packages import build_rust_packages
|
||||
from peach_package_builder.build_peach_config import build_peach_config
|
||||
|
||||
|
||||
def build_packages(default_branch=False):
|
||||
"""
|
||||
builds all PeachCloud microservices as .deb files and adds them to the freight repo
|
||||
"""
|
||||
build_rust_packages(default_branch=default_branch)
|
||||
build_peach_config(default_branch=default_branch)
|
||||
print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--default",
|
||||
help="Ensure default branch for all repos for build",
|
||||
action="store_true"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
build_packages(default_branch=args.default)
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
"""
|
||||
script to build the peach-config debian module and add it to the freight repository
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
|
||||
from peach_package_builder.constants import *
|
||||
from peach_package_builder.utils import add_deb_to_freight, update_freight_cache
|
||||
|
||||
|
||||
def build_peach_config(default_branch=True):
|
||||
service_path = os.path.join(MICROSERVICES_SRC_DIR, "peach-config")
|
||||
if default_branch:
|
||||
# because some repo have main as default and some as master, we get the default
|
||||
default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'],
|
||||
cwd=service_path).decode(sys.stdout.encoding)
|
||||
branch = default_branch.replace('origin/', '').strip()
|
||||
subprocess.check_call(["git", "checkout", branch], cwd=service_path)
|
||||
subprocess.check_call(["git", "reset", "HEAD", "--hard"], cwd=service_path)
|
||||
subprocess.check_call(["git", "pull"], cwd=service_path)
|
||||
# remove old build dir
|
||||
subprocess.check_call([
|
||||
"rm",
|
||||
"-rf",
|
||||
os.path.join(service_path, 'deb_dist')
|
||||
])
|
||||
# build .deb
|
||||
subprocess.check_call([
|
||||
"python3",
|
||||
"setup.py",
|
||||
"--command-packages=stdeb.command",
|
||||
"bdist_deb"
|
||||
],
|
||||
cwd=service_path)
|
||||
version = "0.2.7" # TODO: get this version number from the repo
|
||||
deb_name = "python3-peach-config_{version}-1_all.deb".format(version=version)
|
||||
debian_package_path = os.path.join(
|
||||
service_path,
|
||||
"deb_dist/{}".format(deb_name)
|
||||
)
|
||||
subprocess.check_call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
|
||||
add_deb_to_freight(package_name=debian_package_path, package_path=debian_package_path)
|
||||
update_freight_cache()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--default",
|
||||
help="Ensure default branch for all repos for build",
|
||||
action="store_true"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
build_peach_config(default_branch=args.default)
|
||||
|
|
@ -1,20 +1,22 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
script to create debian packages for cross-compiled go binaries for go-sbot
|
||||
based off of this post
|
||||
https://unix.stackexchange.com/questions/627689/how-to-create-a-debian-package-from-a-bash-script-and-a-systemd-service
|
||||
"""
|
||||
|
||||
|
||||
from constants import *
|
||||
from utils import render_template
|
||||
|
||||
import subprocess
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from packaging import version as pversion
|
||||
|
||||
from peach_package_builder.constants import *
|
||||
from peach_package_builder.utils import render_template, add_deb_to_freight, update_freight_cache
|
||||
|
||||
# manually update this version when we want to build a new peach-go-sbot package
|
||||
PEACH_GO_SBOT_VERSION = '0.1.4'
|
||||
|
||||
# constants
|
||||
DEB_CONF_DIR = os.path.join(PROJECT_PATH, 'conf/templates/peach_go_sbot')
|
||||
DEB_BUILD_DIR = "/tmp/peach_go_sbot"
|
||||
GO_SSB_DIR = "/srv/peachcloud/automation/go-ssb"
|
||||
|
|
@ -22,14 +24,13 @@ GO_SSB_DIR = "/srv/peachcloud/automation/go-ssb"
|
|||
|
||||
def crosscompile_peach_go_sbot():
|
||||
subprocess.check_call(["git", "pull"], cwd=GO_SSB_DIR)
|
||||
# TODO: confirm that version number in the repo matches the version number we set for the package we are building
|
||||
print("[CROSS-COMPILING sbotcli]")
|
||||
subprocess.check_call(["env", "GOOS=linux", "GOARCH=arm64", "go", "build", "./cmd/sbotcli"], cwd=GO_SSB_DIR)
|
||||
print("[CROSS-COMPILING go-sbot]")
|
||||
subprocess.check_call(["env", "GOOS=linux", "GOARCH=arm64", "go", "build", "./cmd/go-sbot"], cwd=GO_SSB_DIR)
|
||||
|
||||
def package_peach_go_sbot(version):
|
||||
|
||||
def package_peach_go_sbot(version):
|
||||
print("[ PACKAGING peach-go-sbot ]")
|
||||
# copy debian conf files into correct locations in package build directory
|
||||
DEBIAN_SRC_DIR = os.path.join(DEB_CONF_DIR, 'DEBIAN')
|
||||
|
|
@ -71,35 +72,31 @@ def package_peach_go_sbot(version):
|
|||
print("[ CREATING {}]".format(deb_file_name))
|
||||
subprocess.check_call(["dpkg-deb", "-b", ".", deb_file_name], cwd=DEB_BUILD_DIR)
|
||||
|
||||
# add deb package to freight
|
||||
print("[ ADDING PACKAGE TO FREIGHT ]")
|
||||
# copy deb package to MICROSERVICES_DEB_DIR
|
||||
deb_path = os.path.join(DEB_BUILD_DIR, deb_file_name)
|
||||
subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, deb_path, "apt/buster"])
|
||||
print("[ ADDING PACKAGE TO FREIGHT CACHE ]")
|
||||
subprocess.call(["sudo", "freight", "cache", "-g",
|
||||
GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])
|
||||
subprocess.check_call(["cp", deb_path, MICROSERVICES_DEB_DIR])
|
||||
|
||||
# add deb package to freight
|
||||
add_deb_to_freight(package_name=deb_file_name, package_path=deb_path)
|
||||
|
||||
# update freight cache
|
||||
update_freight_cache()
|
||||
|
||||
|
||||
def build_peach_go_sbot():
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--version",
|
||||
help="Set version number for go-sbot",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("[ BUILDING PEACH-GO-SBOT VERSION {}]".format(args.version))
|
||||
# gets the most recently built peach_go_sbot version, and increments the micro-number by 1
|
||||
version = PEACH_GO_SBOT_VERSION
|
||||
print("[ BUILDING PEACH-GO-SBOT VERSION {}]".format(version))
|
||||
|
||||
# delete build directory if it already exists or create it
|
||||
subprocess.check_call(["rm", "-rf", DEB_BUILD_DIR])
|
||||
if not os.path.exists(DEB_BUILD_DIR):
|
||||
os.makedirs(DEB_BUILD_DIR)
|
||||
|
||||
# cross-compile and package peach-go-sbot with new version number
|
||||
crosscompile_peach_go_sbot()
|
||||
package_peach_go_sbot(version=args.version)
|
||||
|
||||
package_peach_go_sbot(version=version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
#!/usr/bin/env python3
|
||||
import subprocess
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
|
||||
from peach_package_builder.build_peach_config import build_peach_config
|
||||
from peach_package_builder.constants import *
|
||||
from peach_package_builder.utils import add_deb_to_freight, update_freight_cache
|
||||
|
||||
|
||||
def add_debs_dir_to_freight():
|
||||
"""
|
||||
adds all packages in MICROSERVICES_DEB_DIR to freight cache
|
||||
"""
|
||||
print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]")
|
||||
for package in os.scandir(MICROSERVICES_DEB_DIR):
|
||||
if package.name.endswith(".deb"):
|
||||
add_deb_to_freight(package_name=package.name, package_path=package.path)
|
||||
update_freight_cache()
|
||||
|
||||
|
||||
def build_rust_packages(default_branch=False):
|
||||
"""
|
||||
builds all PeachCloud microservices written in rust and copies them to MICROSERVICES_DEB_DIR
|
||||
"""
|
||||
print("[ BUILDING AND UPDATING RUST MICROSERVICE PACKAGES ]")
|
||||
for service in SERVICES:
|
||||
service_name = service["name"]
|
||||
service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name)
|
||||
print("[ BUILIDING SERVICE {} ]".format(service_name))
|
||||
# this arg ensures we build the default branch, otherwise we build what ever is found locally
|
||||
if default_branch:
|
||||
# because some repo have main as default and some as master, we get the default
|
||||
default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'],
|
||||
cwd=service_path).decode(sys.stdout.encoding)
|
||||
branch = default_branch.replace('origin/', '').strip()
|
||||
subprocess.check_call(["git", "checkout", branch], cwd=service_path)
|
||||
subprocess.check_call(["git", "reset", "HEAD", "--hard"], cwd=service_path)
|
||||
subprocess.check_call(["git", "pull"], cwd=service_path)
|
||||
debian_package_path = subprocess.check_output(
|
||||
[
|
||||
CARGO_PATH,
|
||||
"deb",
|
||||
"--target",
|
||||
"aarch64-unknown-linux-gnu"],
|
||||
cwd=service_path).decode("utf-8").strip()
|
||||
subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
|
||||
|
||||
# this function adds all .deb files in MICROSERVICES_DEB_DIR to freight
|
||||
add_debs_dir_to_freight()
|
||||
|
||||
|
||||
def build_packages(default_branch=False):
|
||||
"""
|
||||
builds all PeachCloud microservices as .deb files and adds them to the freight repo
|
||||
"""
|
||||
build_rust_packages(default_branch=default_branch)
|
||||
build_peach_config(default_branch=default_branch)
|
||||
print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--default",
|
||||
help="Ensure default branch for all repos for build",
|
||||
action="store_true"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
build_packages(default_branch=args.default)
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from utils import render_template
|
||||
from constants import *
|
||||
from peach_package_builder.utils import render_template
|
||||
from peach_package_builder.constants import *
|
||||
|
||||
import subprocess
|
||||
import argparse
|
||||
|
|
@ -2,8 +2,8 @@ import os
|
|||
import jinja2
|
||||
import subprocess
|
||||
|
||||
PROJECT_PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
|
||||
print('PROJECT_PATH: {}'.format(PROJECT_PATH))
|
||||
from peach_package_builder.constants import *
|
||||
|
||||
|
||||
template_path = os.path.join(PROJECT_PATH, 'conf/templates')
|
||||
template_loader = jinja2.FileSystemLoader(searchpath=template_path)
|
||||
|
|
@ -25,3 +25,15 @@ def render_template(src, dest, template_vars=None):
|
|||
os.remove(dest)
|
||||
with open(dest, 'w') as f:
|
||||
f.write(output_text)
|
||||
|
||||
|
||||
def add_deb_to_freight(package_name, package_path):
|
||||
print("[ ADDING PACKAGE {} ]".format(package_name))
|
||||
subprocess.check_call(["freight", "add", "-c", FREIGHT_CONF, package_path, "apt/buster"])
|
||||
|
||||
|
||||
def update_freight_cache():
|
||||
print("[ ADDING PACKAGES TO FREIGHT CACHE ]")
|
||||
# needs to be run as sudo user
|
||||
subprocess.call(["sudo", "freight", "cache", "-g",
|
||||
GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])
|
||||
|
|
@ -1 +1,2 @@
|
|||
Jinja2==2.11.2
|
||||
packaging==20.9
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
from constants import *
|
||||
|
||||
import subprocess
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--default",
|
||||
help="Ensure default branch for all repos for build",
|
||||
action="store_true"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
print("[ BUILDING AND UPDATING MICROSERVICE PACKAGES ]")
|
||||
for service in SERVICES:
|
||||
service_name = service["name"]
|
||||
service_path = os.path.join(MICROSERVICES_SRC_DIR, service_name)
|
||||
print("[ BUILIDING SERVICE {} ]".format(service_name))
|
||||
# this arg ensures we build the default branch, otherwise we build what ever is found locally
|
||||
if args.default:
|
||||
# because some repo have main as default and some as master, we get the default
|
||||
default_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'origin/HEAD'],
|
||||
cwd=service_path).decode(sys.stdout.encoding)
|
||||
branch = default_branch.replace('origin/', '').strip()
|
||||
subprocess.run(["git", "checkout", branch], cwd=service_path)
|
||||
subprocess.run(["git", "reset", "HEAD", "--hard"], cwd=service_path)
|
||||
subprocess.run(["git", "pull"], cwd=service_path)
|
||||
debian_package_path = subprocess.run(
|
||||
[
|
||||
CARGO_PATH,
|
||||
"deb",
|
||||
"--target",
|
||||
"aarch64-unknown-linux-gnu"],
|
||||
cwd=service_path,
|
||||
stdout=subprocess.PIPE).stdout.decode("utf-8").strip()
|
||||
subprocess.call(["cp", debian_package_path, MICROSERVICES_DEB_DIR])
|
||||
|
||||
print("[ ADDING PACKAGES TO FREIGHT LIBRARY ]")
|
||||
for package in os.scandir(MICROSERVICES_DEB_DIR):
|
||||
if package.name.endswith(".deb"):
|
||||
print("[ ADDING PACKAGE {} ]".format(package.name))
|
||||
subprocess.call(["freight", "add", "-c", FREIGHT_CONF,
|
||||
package.path, "apt/buster"])
|
||||
|
||||
print("[ ADDING PACKAGES TO FREIGHT CACHE ]")
|
||||
# needs to be run as sudo user
|
||||
subprocess.call(["sudo", "freight", "cache", "-g",
|
||||
GPG_KEY_EMAIL, "-p", GPG_KEY_PASS_FILE])
|
||||
|
||||
print("[ MICROSERVICE PACKAGE ARCHIVE UPDATED ]")
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#rsync -avzh --delete -e "ssh -i /Users/maxfowler/.ssh/peach_rsa" . notplants@167.99.136.8:/srv/peachcloud/automation/peach-vps
|
||||
KEY_FILE=/Users/notplants/.ssh/peach_rsa
|
||||
rsync -avzh --exclude target --exclude .idea --exclude .git --delete -e "ssh -i $KEY_FILE" . notplants@167.99.136.83:/srv/peachcloud/automation/peach-package-builder/
|
||||
#ssh -i ./secret_files/do_rsa root@159.89.5.141
|
||||
#ssh -i /home/notplants/.ssh/peach_rsa rust@167.99.136.83 'cd /srv/peachcloud/automation/peach-vps/; python3 scripts/build_packages.py'
|
||||
#echo "cd /srv/src/peach-vps; python3 scripts/setup_vps.py"
|
||||
Loading…
Reference in New Issue