From 15402be01db32d91abbd25ab70de313d3df1cd0d Mon Sep 17 00:00:00 2001 From: notplants Date: Thu, 25 Feb 2021 19:42:47 +0100 Subject: [PATCH 1/6] Add -d flag to setup_dev_env --- raspi_master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raspi_master.yaml b/raspi_master.yaml index 585441e..06f9225 100644 --- a/raspi_master.yaml +++ b/raspi_master.yaml @@ -143,7 +143,7 @@ steps: - chroot: / shell: | git clone -b dev https://github.com/peachcloud/peach-config.git /srv/peach-config - cd /srv/peach-config/ && python3 /srv/peach-config/scripts/setup_dev_env.py -i -n peach + cd /srv/peach-config/ && python3 /srv/peach-config/scripts/setup_dev_env.py -i -n -d peach # END OF PEACH CONFIG From 9ccd148a8ee5731cf7acd46d5644acbbf3887afa Mon Sep 17 00:00:00 2001 From: notplants Date: Mon, 8 Mar 2021 11:26:13 +0100 Subject: [PATCH 2/6] Working on jinja build for releases.peachcloud.org --- build_img.py | 67 ++++++++++++++++++++++++++++++++++++ templates/release_index.html | 49 ++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 build_img.py create mode 100644 templates/release_index.html diff --git a/build_img.py b/build_img.py new file mode 100644 index 0000000..5811b0b --- /dev/null +++ b/build_img.py @@ -0,0 +1,67 @@ +import os +import subprocess +import jinja2 +from datetime import date + + +PROJECT_PATH = os.path.dirname(os.path.realpath(__file__)) + +# load jinja templates +template_path = os.path.join(PROJECT_PATH, 'templates') +template_loader = jinja2.FileSystemLoader(searchpath=template_path) +template_env = jinja2.Environment(loader=template_loader, keep_trailing_newline=True) + + +def render_template(src, dest, template_vars=None): + """ + helper function fo rendering jinja template + :param src: relative string path to jinja template file + :param dest: absolute string path of output destination file + :param template_vars: variables to render template with + :return: None + """ + template = template_env.get_template(src) + if not template_vars: + template_vars= {} + output_text = template.render(**template_vars) + if os.path.exists(dest): + os.remove(dest) + with open(dest, 'w') as f: + f.write(output_text) + + +# remove old files +os.remove(os.path.join(PROJECT_PATH, 'raspi_3.img')) +os.remove(os.path.join(PROJECT_PATH, 'raspi_3.log')) + +# build img +subprocess.check_call(['make', 'raspi_3.img']) + +# copy image and log to releases dir +today = date.today() +today_str = "{}{}{}".format(today.year, today.month, today.day) +release_dir = "/var/www/releases.peachcloud.org/html/peach-imgs/{}".format(today_str) +print("++ successful image build, copying output to {}", release_dir) + +os.makedirs(release_dir) +img_path = os.path.join(PROJECT_PATH, 'raspi_3.img') +log_path = os.path.join(PROJECT_PATH, 'raspi_3.log') +release_img_name = "{}_peach_raspi3.img".format(today_str) +release_log_name = "{}_peach_raspi3.log".format(today_str) +img_release_path = os.path.join(release_dir, release_img_name) +log_release_path = os.path.join(release_dir, release_log_name) +subprocess.check_call(['cp', img_path, img_release_path]) +subprocess.check_call(['cp', log_path, log_release_path]) + + +# rebuild release index.html +release_index_path = "/var/www/releases.peachcloud.org/html/index.html" +release_img_url = img_release_path.replace('/var/www/releases.peachcloud.org/html/', '/') +render_template( + src="release_index.html", + dest=release_index_path, + template_vars={ + "release_img_url": release_img_url, + "release_img_name": release_img_name, + } +) \ No newline at end of file diff --git a/templates/release_index.html b/templates/release_index.html new file mode 100644 index 0000000..2993e50 --- /dev/null +++ b/templates/release_index.html @@ -0,0 +1,49 @@ + + + +PeachCloud Release Builds + + + + +

PeachCloud Image Builds

+

The latest PeachCloud disc image for Raspberry Pi 3+ with all PeachCloud microservices pre-installed.

+ + +
+
+
+ +

PeachCloud Release Builds

+

The latest aarch64 release builds of PeachCloud microservices and other software components.

+ + +
+ +

For online documentation please refer to +docs.peachcloud.org.
+Code repositories can be found at +github.com/peachcloud.
+Support our efforts at +opencollective.com/peachcloud.

+ +

Thank you for your interest in PeachCloud.

+ + \ No newline at end of file From 48393605bbd4af8f81ea774695f1f928cc9f6442 Mon Sep 17 00:00:00 2001 From: notplants Date: Thu, 11 Mar 2021 13:52:27 +0100 Subject: [PATCH 3/6] Working on including manifest in image build --- build.sh | 1 + build_img.py | 9 ++++----- raspi_master.yaml | 22 +++++++++++++++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index 7c819b4..fcd24e2 100755 --- a/build.sh +++ b/build.sh @@ -9,4 +9,5 @@ RELEASE_DIR=/var/www/releases.peachcloud.org/html/peach-imgs/$TODAY echo "++ successful image build, copying output to ${RELEASE_DIR}" mkdir -p $RELEASE_DIR cp raspi_3.img $RELEASE_DIR/${TODAY}_peach_raspi3.img +cp peach-img-manifest.log $RELEASE_DIR/${TODAY}_img_manifest.log cp raspi_3.log $RELEASE_DIR/${TODAY}_peach_raspi3.log \ No newline at end of file diff --git a/build_img.py b/build_img.py index 5811b0b..06618a3 100644 --- a/build_img.py +++ b/build_img.py @@ -29,7 +29,6 @@ def render_template(src, dest, template_vars=None): with open(dest, 'w') as f: f.write(output_text) - # remove old files os.remove(os.path.join(PROJECT_PATH, 'raspi_3.img')) os.remove(os.path.join(PROJECT_PATH, 'raspi_3.log')) @@ -37,13 +36,14 @@ os.remove(os.path.join(PROJECT_PATH, 'raspi_3.log')) # build img subprocess.check_call(['make', 'raspi_3.img']) -# copy image and log to releases dir +# create releases dir today = date.today() today_str = "{}{}{}".format(today.year, today.month, today.day) release_dir = "/var/www/releases.peachcloud.org/html/peach-imgs/{}".format(today_str) -print("++ successful image build, copying output to {}", release_dir) - os.makedirs(release_dir) + +# copy image and log to releases dir +print("++ successful image build, copying output to {}", release_dir) img_path = os.path.join(PROJECT_PATH, 'raspi_3.img') log_path = os.path.join(PROJECT_PATH, 'raspi_3.log') release_img_name = "{}_peach_raspi3.img".format(today_str) @@ -53,7 +53,6 @@ log_release_path = os.path.join(release_dir, release_log_name) subprocess.check_call(['cp', img_path, img_release_path]) subprocess.check_call(['cp', log_path, log_release_path]) - # rebuild release index.html release_index_path = "/var/www/releases.peachcloud.org/html/index.html" release_img_url = img_release_path.replace('/var/www/releases.peachcloud.org/html/', '/') diff --git a/raspi_master.yaml b/raspi_master.yaml index 06f9225..25845eb 100644 --- a/raspi_master.yaml +++ b/raspi_master.yaml @@ -136,14 +136,30 @@ steps: - apt: install packages: - git - - python + - python3 + - python3-pip - wget + - gnupg2 tag: / - chroot: / shell: | - git clone -b dev https://github.com/peachcloud/peach-config.git /srv/peach-config - cd /srv/peach-config/ && python3 /srv/peach-config/scripts/setup_dev_env.py -i -n -d peach + pip3 install setuptools + echo "deb http://apt.peachcloud.org/ buster main" > /etc/apt/sources.list.d/peach.list + wget -O /tmp/pubkey.gpg http://apt.peachcloud.org/pubkey.gpg + apt-key add /tmp/pubkey.gpg + apt-get update + apt-get install -y python3-peach-config + /usr/bin/peach-config -i -n -d peach + + # lastly log which versions of microservices were installed and copy the provenance to the host machine + - chroot: / + shell: | + apt list --installed | grep peach > /srv/peach-img-manifest.log + + - shell: | + cp "${ROOT?}/srv/peach-img-manifest.log" /srv/peachcloud/automation/peach-img-builder/peach-img-manifest.log + root-fs: / # END OF PEACH CONFIG From 5ed956048dfd0a05ab29672e7d4a34d573cff226 Mon Sep 17 00:00:00 2001 From: notplants Date: Tue, 16 Mar 2021 11:37:18 +0100 Subject: [PATCH 4/6] Working on release script --- build_img.py | 107 +++++++++++++++++++++++++---------- raspi_master.yaml | 6 +- templates/release_index.html | 18 +++--- 3 files changed, 89 insertions(+), 42 deletions(-) diff --git a/build_img.py b/build_img.py index 06618a3..8b5ac0a 100644 --- a/build_img.py +++ b/build_img.py @@ -1,6 +1,8 @@ import os import subprocess import jinja2 +import json +import argparse from datetime import date @@ -29,38 +31,81 @@ def render_template(src, dest, template_vars=None): with open(dest, 'w') as f: f.write(output_text) -# remove old files -os.remove(os.path.join(PROJECT_PATH, 'raspi_3.img')) -os.remove(os.path.join(PROJECT_PATH, 'raspi_3.log')) -# build img -subprocess.check_call(['make', 'raspi_3.img']) +def build_img(build=True, publish=True): + """ + if build=True, + builds a new PeachCloud image, + compresses the image, + creates a manifest for what was included in that image + and a log for the build of the image -# create releases dir -today = date.today() -today_str = "{}{}{}".format(today.year, today.month, today.day) -release_dir = "/var/www/releases.peachcloud.org/html/peach-imgs/{}".format(today_str) -os.makedirs(release_dir) + if publish=True + copies the following files to release dir with the current date + - log of build + - compressed img file + - manifest for that image + rebuilds releases.peachcloud.org to point to the release -# copy image and log to releases dir -print("++ successful image build, copying output to {}", release_dir) -img_path = os.path.join(PROJECT_PATH, 'raspi_3.img') -log_path = os.path.join(PROJECT_PATH, 'raspi_3.log') -release_img_name = "{}_peach_raspi3.img".format(today_str) -release_log_name = "{}_peach_raspi3.log".format(today_str) -img_release_path = os.path.join(release_dir, release_img_name) -log_release_path = os.path.join(release_dir, release_log_name) -subprocess.check_call(['cp', img_path, img_release_path]) -subprocess.check_call(['cp', log_path, log_release_path]) + running with both flags as true is standard usage, + but building and publishing separately can be useful for testing + """ -# rebuild release index.html -release_index_path = "/var/www/releases.peachcloud.org/html/index.html" -release_img_url = img_release_path.replace('/var/www/releases.peachcloud.org/html/', '/') -render_template( - src="release_index.html", - dest=release_index_path, - template_vars={ - "release_img_url": release_img_url, - "release_img_name": release_img_name, - } -) \ No newline at end of file + # these are the three files created by the build + img_path = os.path.join(PROJECT_PATH, 'raspi_3.img') + log_path = os.path.join(PROJECT_PATH, 'raspi_3.log') + manifest_path = os.path.join(PROJECT_PATH, 'peach-img-manifest.log') + + # if build=True, then remove old files and re-build + if build: + # remove old files + if os.path.exists(img_path): + os.remove(img_path) + if os.path.exists(log_path): + os.remove(log_path) + if os.path.exists(manifest_path): + os.remove(manifest_path) + + # build img + subprocess.check_call(['make', 'raspi_3.img']) + + if publish: + # create release dir for this release + today = date.today() + today_str = "{}{}{}".format(today.year, today.month, today.day) + release_dir = "/var/www/releases.peachcloud.org/html/peach-imgs/{}".format(today_str) + os.makedirs(release_dir) + + # copy image, log and manifest to release dir + print("++ successful image build, copying output to {}", release_dir) + img_release_name = "{}_peach_raspi3.img".format(today_str) + release_log_name = "{}_peach_raspi3.log".format(today_str) + manifest_name = "{}_peach_manifest.log".format(today_str) + img_release_path = os.path.join(release_dir, img_release_name) + log_release_path = os.path.join(release_dir, release_log_name) + manifest_release_path = os.path.join(release_dir, manifest_name) + subprocess.check_call(['cp', img_path, img_release_path]) + subprocess.check_call(['cp', log_path, log_release_path]) + subprocess.check_call(['cp', manifest_path, manifest_release_path]) + + # rebuild index.html to point to the new release + release_index_path = "/var/www/releases.peachcloud.org/html/index.html" + release_img_url = img_path.replace('/var/www/releases.peachcloud.org/html/', '/') + with open(manifest_path, 'r') as f: + manifest = json.loads(f.read()) + for k, v in manifest.items(): + print("{}: {}".format(k, v)) + packages = manifest['packages'] + render_template( + src="release_index.html", + dest=release_index_path, + template_vars={ + "release_img_url": release_img_url, + "release_img_name": img_release_name, + "packages": packages + } + ) + + +if __name__ == '__main__': + build_img(build=True, publish=True) \ No newline at end of file diff --git a/raspi_master.yaml b/raspi_master.yaml index 25845eb..22be6e7 100644 --- a/raspi_master.yaml +++ b/raspi_master.yaml @@ -150,12 +150,12 @@ steps: apt-key add /tmp/pubkey.gpg apt-get update apt-get install -y python3-peach-config - /usr/bin/peach-config -i -n -d peach + /usr/bin/peach-config setup -i -n -d peach # lastly log which versions of microservices were installed and copy the provenance to the host machine - chroot: / shell: | - apt list --installed | grep peach > /srv/peach-img-manifest.log + /usr/bin/peach-config manifest > /srv/peach-img-manifest.log - shell: | cp "${ROOT?}/srv/peach-img-manifest.log" /srv/peachcloud/automation/peach-img-builder/peach-img-manifest.log @@ -167,4 +167,4 @@ steps: # clears /etc/resolv.conf on its own. - shell: | rm "${ROOT?}/etc/resolv.conf" - root-fs: / \ No newline at end of file + root-fs: / diff --git a/templates/release_index.html b/templates/release_index.html index 2993e50..52f8cb1 100644 --- a/templates/release_index.html +++ b/templates/release_index.html @@ -24,15 +24,17 @@

-

PeachCloud Release Builds

-

The latest aarch64 release builds of PeachCloud microservices and other software components.

+

PeachCloud Microservices

+

The above image contains the following packages:

From 4c0662459504db1dfaa598646889b3f56258ac7d Mon Sep 17 00:00:00 2001 From: notplants Date: Tue, 16 Mar 2021 11:38:24 +0100 Subject: [PATCH 5/6] Working on publish script --- build_img.py => publish_img.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build_img.py => publish_img.py (100%) diff --git a/build_img.py b/publish_img.py similarity index 100% rename from build_img.py rename to publish_img.py From 83dc63602a4070466cde3d54ad850261b21ee361 Mon Sep 17 00:00:00 2001 From: notplants Date: Tue, 16 Mar 2021 13:02:42 +0100 Subject: [PATCH 6/6] Remove print statement --- publish_img.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/publish_img.py b/publish_img.py index 9a1618f..2321ef7 100644 --- a/publish_img.py +++ b/publish_img.py @@ -56,8 +56,6 @@ def publish_img(release_dir): release_img_url = img_path.replace('/var/www/releases.peachcloud.org/html/', '/') with open(manifest_path, 'r') as f: manifest = json.loads(f.read()) - for k, v in manifest.items(): - print("{}: {}".format(k, v)) packages = manifest['packages'] render_template( src="release_index.html",