From c67120883576894c2bdc0a256de977a214eb31f4 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 11 Oct 2023 00:14:44 +0200 Subject: [PATCH] first PoC --- README.md | 3 + migrabrator.py | 222 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 README.md create mode 100755 migrabrator.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..dd234fc --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Migrabrator + +A tool to migrate abra apps from one server and or domain to another. diff --git a/migrabrator.py b/migrabrator.py new file mode 100755 index 0000000..d2beca1 --- /dev/null +++ b/migrabrator.py @@ -0,0 +1,222 @@ +#!/bin/python3 +import click +import logging +import subprocess +import json +import os +import tempfile +from shutil import copyfile, rmtree +from pathlib import Path + +@click.command() +@click.option('-l', '--log', 'loglevel') +@click.option('source_app', '--source_app', '-s') +@click.option('target_app', '--target_app', '-t') +@click.option('backupbot', '--backupbot', '-b') +def main(loglevel, source_app, target_app, backupbot): + if loglevel: + numeric_level = getattr(logging, loglevel.upper(), None) + if not isinstance(numeric_level, int): + raise ValueError('Invalid log level: %s' % loglevel) + logging.basicConfig(level=numeric_level) + # backup(source_app, backupbot) + + +def move_app(source_app, target_server=False, target_domain=False): + backup(source_app) + copy_volumes(source_app, target_server, target_domain) + copy_configs(source_app, target_server, target_domain) + copy_secrets(source_app, target_domain) + +def backup(app): + # TODO: replace with abra app backup command + backupbot = get_backupbot(source_app) + output = abra('app', 'run', backupbot, 'app', + '--', 'backup', '-h', app, 'create') + print(output) + +def copy_files_between_servers(source_server, source_dir, destination_server, destination_dir): + # Generate temporary SSH key pair + ssh_key_pair_dir = tempfile.mkdtemp() + ssh_key_id = ssh_key_pair_dir.split('/')[2] + private_key_path = os.path.join(ssh_key_pair_dir, 'id_ed25519') + public_key_path = os.path.join(ssh_key_pair_dir, 'id_ed25519.pub') + subprocess.run(['ssh-keygen', '-t', 'ed25519', '-N', '', + '-f', private_key_path, '-C', ssh_key_id], check=True) + try: + # Copy the private key to the source server + source_key_dir = run_ssh(source_server, 'mktemp -d', True) + subprocess.run( + ['scp', private_key_path, f'{source_server}:{source_key_dir}'], check=True) + source_key_file = f'{source_key_dir}/id_ed25519' + run_ssh(source_server, f'chmod 600 {source_key_file}') + # Add the public key to the authorized hosts of the destination server + subprocess.run(['ssh-copy-id', '-i', public_key_path, + destination_server], check=True) + # Run rsync over SSH on the source server to copy files to the destination server + source_rsync_cmd = f'rsync -avz -e "ssh -i {source_key_file} -o StrictHostKeyChecking=accept-new" {source_dir} {destination_server}:{destination_dir}' + run_ssh(source_server, source_rsync_cmd) + # Remove the SSH key pair from the source server + run_ssh(source_server, f'rm -r {source_key_dir}') + # Remove the public key from the authorized hosts of the destination server + run_ssh(destination_server, + f'sed -i.bak "/{ssh_key_id}/d" ~/.ssh/authorized_keys') + finally: + # Remove the temporary SSH key pair directory + rmtree(ssh_key_pair_dir) + + +def copy_volumes(source_app, target_server=False, target_domain=False, move=False): + if not any([target_domain, target_server]): + logging.error( + 'At leat one of target_domain or target_app need to be speicified') + exit(1) + source_server = get_server(source_app) + source_domain = get_domain(source_app) + source_service = source_domain.replace(".", "_") + volume_dir = f'/var/lib/docker/volumes/{source_service}_*' + target_dir = f'/var/lib/docker/volumes' + if target_server: + copy_files_between_servers( + source_server, volume_dir, target_server, target_dir) + server = None + if target_server and target_domain: + server = target_server + cmd = 'mv' + elif target_domain: + server = source_server + cmd = 'cp -r' + if move: + cmd = 'mv' + if target_domain: + paths = run_ssh(server, f'echo {volume_dir}', True).split() + target_service = target_domain.replace(".", "_") + for old_path in paths: + container = old_path.split('_')[-1] + new_path = f'{target_dir}/{target_service}_{container}' + print(f'{cmd} {old_path} {new_path}') + run_ssh(server, f'{cmd} {old_path} {new_path}') + + +def copy_configs(source_app, target_server=False, target_domain=False): + source_server = get_server(source_app) + target_path= source_path = Path(f"~/.abra/servers/{source_server}").expanduser() + if target_server: + target_path = Path(f"~/.abra/servers/{target_server}").expanduser() + source_env = source_path.joinpath(f'{source_app}.env') + if target_domain: + target_env = target_path.joinpath(f'{target_domain}.env') + else: + target_env = target_path.joinpath(f'{source_app}.env') + copy_env(source_env, target_env) + if target_domain: + replace_domain(source_app, target_domain, target_env) + + +def copy_env(source_env, target_env): + if not source_env.exists(): + logging.error(f"file {source_env} not found") + exit(1) + copyfile(source_env, target_env) + + +def replace_domain(source_app, target_domain, target_env): + source_domain = get_domain(source_app) + replace_string_in_file(target_env, source_domain, target_domain) + + +def replace_string_in_file(file_path, search_string, replace_string): + try: + with open(file_path, 'r') as file: + content = file.read() + modified_content = content.replace(search_string, replace_string) + with open(file_path, 'w') as file: + file.write(modified_content) + except FileNotFoundError: + print(f"File '{file_path}' not found.") + except Exception as e: + print(f"An error occurred: {str(e)}") + + +def get_server(app_name): + server_list = json.loads(abra('app', 'ls', '-m')) + for server in server_list.values(): + for app in server['apps']: + if app['appName'] == app_name: + return app['server'] + + +def get_domain(app_name): + server_list = json.loads(abra('app', 'ls', '-m')) + for server in server_list.values(): + for app in server['apps']: + if app['appName'] == app_name: + return app['domain'] + + +def copy_secrets(source_app, target_domain): + secrets = get_secrets(source_app) + insert_secrets(target_domain, secrets) + + +def get_secrets(source_app): + # TODO: replace with abra app backup command + backupbot = get_backupbot(source_app) + output = subprocess.run(['abra', 'app', 'run', backupbot, 'app', '--', + 'backup', '-h', source_app, 'download', '--secrets'], capture_output=True, text=True) + secret_path = output.stdout.strip() + output = subprocess.run( + ['abra', 'app', 'cp', backupbot, f"app:{secret_path}", "/tmp"]) + with open(secret_path) as file: + secrets = json.load(file) + return secrets + + +def get_backupbot(app_name): + server = get_server(app_name) + server_list = json.loads(abra('app', 'ls', '-m')) + for app in server_list[server]['apps']: + if app['recipe'] == 'backup-bot-two': + return app['appName'] + + +def insert_secrets(target_domain, secrets): + for sec in secrets: + secret = secrets[sec] + secret_name = "_".join(sec.split('_')[:-1]) + secret_version = sec.split('_')[-1] + cmd = ['abra', 'app', 'secret', 'insert', target_domain, secret_name, secret_version, secret] + print(" ".join(cmd)) + output = subprocess.run(cmd, capture_output=True, text=True) + logging.debug(output.stdout) + if output.returncode: + logging.error(output.stderr) + + +def run_ssh(server, cmd, get_output=False): + output = subprocess.run(["ssh", server] + cmd.split(), + capture_output=get_output, check=True, text=True) + if get_output: + return output.stdout.strip() + + +def abra(*args, machine_output=False, ignore_error=False): + command = ["abra", *args] + if machine_output: + command.append("-m") + logging.debug(f"run command: {' '.join(command)}") + process = subprocess.run(command, capture_output=True, text=True) + if process.stderr: + logging.debug(process.stderr) + if process.stdout: + logging.debug(process.stdout) + if process.returncode and not ignore_error: + raise RuntimeError( + f'{" ".join(command)} \n STDOUT: \n {process.stdout} \n STDERR: {process.stderr}') + if machine_output: + return json.loads(process.stdout) + return process.stdout + + +if __name__ == '__main__': + main()