commit 4a1924587c9d8fa732c2658ec9dd96992e65b200 Author: forest Date: Sat May 9 19:13:20 2020 -0500 postgres automatic schema management roughly working diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..09b24fe --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +notes.txt +.vscode + +*.pyc +__pycache__/ + +instance/ + +.pytest_cache/ +.coverage +htmlcov/ + +dist/ +build/ +*.egg-info/ + +.venv diff --git a/README.md b/README.md new file mode 100644 index 0000000..b3ff279 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# capsulflask + +Python Flask web application for capsul.org + +## postgres database schema management + +capsulflask has a concept of a schema version. When the application starts, it will query the database for a table named +`schemaversion` that has one row and one column (`version`). If the `version` it finds is not equal to the `desiredSchemaVersion` variable set in `db.py`, it will run migration scripts from the `schema_migrations` folder one by one until the `schemaversion` table shows the correct version. + +For example, the script named `02_up_xyz.sql` should contain code that migrates the database in a reverse-able fashion from schema version 1 to schema version 2. Likewise, the script `02_down_xyz.sql` should contain code that migrates from schema version 2 back to schema version 1. + +**IMPORTANT: if you need to make changes to the schema, make a NEW schema version. DO NOT EDIT the existing schema versions.** + +## how to run locally + +Ensure you have the pre-requisites for the psycopg2 Postgres database adapter package + +``` +sudo apt-get install python3-dev libpq-dev +pg_config --version +``` + +Create python virtual environment and install packages + +``` +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +Run an instance of Postgres (I used docker for this, point is its listening on localhost:5432) + +``` +docker run -it -e POSTGRES_PASSWORD=dev -p 5432:5432 postgres +``` + +Run the app + +``` +FLASK_APP=capsulflask flask run +``` \ No newline at end of file diff --git a/capsulflask/__init__.py b/capsulflask/__init__.py new file mode 100644 index 0000000..27f2de9 --- /dev/null +++ b/capsulflask/__init__.py @@ -0,0 +1,24 @@ + +from flask import Flask +import os + +def create_app(): + app = Flask(__name__) + app.config.from_mapping( + SECRET_KEY=os.environ.get("SECRET_KEY", default="dev"), + DATABASE_URL=os.environ.get("DATABASE_URL", default="sql://postgres:dev@localhost:5432/postgres"), + DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"), + ) + + from capsulflask import db + + db.init_app(app) + + # from capsulflask import auth, blog + + # app.register_blueprint(auth.bp) + # app.register_blueprint(blog.bp) + + app.add_url_rule("/", endpoint="index") + + return app \ No newline at end of file diff --git a/capsulflask/db.py b/capsulflask/db.py new file mode 100644 index 0000000..214ce88 --- /dev/null +++ b/capsulflask/db.py @@ -0,0 +1,117 @@ +import psycopg2 +import re +import sys +from urllib.parse import urlparse +from os import listdir +from os.path import isfile, join +from psycopg2 import pool +from flask import current_app +from flask import g + + +def init_app(app): + databaseUrl = urlparse(app.config['DATABASE_URL']) + + app.config['PSYCOPG2_CONNECTION_POOL'] = psycopg2.pool.SimpleConnectionPool( + 1, + 20, + user = databaseUrl.username, + password = databaseUrl.password, + host = databaseUrl.hostname, + port = databaseUrl.port, + database = databaseUrl.path[1:] + ) + + schemaMigrations = {} + schemaMigrationsPath = join(app.root_path, 'schema_migrations') + print("loading schema migration scripts from {}".format(schemaMigrationsPath)) + for filename in listdir(schemaMigrationsPath): + key = re.search(r"^\d+_(up|down)", filename).group() + with open(join(schemaMigrationsPath, filename), 'rb') as file: + schemaMigrations[key] = file.read().decode("utf8") + + db = app.config['PSYCOPG2_CONNECTION_POOL'].getconn() + + hasSchemaVersionTable = False + actionWasTaken = False + schemaVersion = 0 + desiredSchemaVersion = 2 + + cursor = db.cursor() + + cursor.execute(""" + SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema = '{}' + """.format(app.config['DATABASE_SCHEMA'])) + + rows = cursor.fetchall() + for row in rows: + if row[0] == "schemaversion": + hasSchemaVersionTable = True + + if hasSchemaVersionTable == False: + print("no table named schemaversion found in the {} schema. running migration 01_up".format(app.config['DATABASE_SCHEMA'])) + try: + cursor.execute(schemaMigrations["01_up"]) + db.commit() + except: + print("unable to create the schemaversion table because: {}".format(my_exec_info_message(sys.exc_info()))) + exit(1) + actionWasTaken = True + + cursor.execute("SELECT Version FROM schemaversion") + schemaVersion = cursor.fetchall()[0][0] + + # print(schemaVersion) + while schemaVersion < desiredSchemaVersion: + migrationKey = "%02d_up" % (schemaVersion+1) + print("schemaVersion ({}) < desiredSchemaVersion ({}). running migration {}".format( + schemaVersion, desiredSchemaVersion, migrationKey + )) + try: + cursor.execute(schemaMigrations[migrationKey]) + db.commit() + except KeyError: + print("missing schema migration script: {}_xyz.sql".format(migrationKey)) + exit(1) + except: + print("unable to execute the schema migration {} because: {}".format(migrationKey, my_exec_info_message(sys.exc_info()))) + exit(1) + actionWasTaken = True + + schemaVersion += 1 + cursor.execute("SELECT Version FROM schemaversion") + versionFromDatabase = cursor.fetchall()[0][0] + + if schemaVersion != versionFromDatabase: + print("incorrect schema version value \"{}\" after running migration {}, expected \"{}\". exiting.".format( + versionFromDatabase, + migrationKey, + schemaVersion + )) + exit(1) + + cursor.close() + + app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db) + + print("schema migration completed. {}current schemaVersion: \"{}\"".format( + ("" if actionWasTaken else "(no action was taken). "), schemaVersion + )) + + app.teardown_appcontext(close_db) + + +def get_db(): + if 'db' not in g: + g.db = current_app.config['PSYCOPG2_CONNECTION_POOL'].getconn() + return g.db + + +def close_db(e=None): + db = g.pop("db", None) + + if db is not None: + current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db) + +def my_exec_info_message(exec_info): + return "{}: {}".format(".".join([exec_info[0].__module__, exec_info[0].__name__]), exec_info[1]) \ No newline at end of file diff --git a/capsulflask/schema_migrations/01_up_create_SchemaVersion.sql b/capsulflask/schema_migrations/01_up_create_SchemaVersion.sql new file mode 100644 index 0000000..7251a3c --- /dev/null +++ b/capsulflask/schema_migrations/01_up_create_SchemaVersion.sql @@ -0,0 +1,5 @@ +CREATE TABLE schemaversion ( + version INT PRIMARY KEY NOT NULL +); + +INSERT INTO schemaversion(version) VALUES (1); \ No newline at end of file diff --git a/capsulflask/schema_migrations/02_up_test.sql b/capsulflask/schema_migrations/02_up_test.sql new file mode 100644 index 0000000..e303a1a --- /dev/null +++ b/capsulflask/schema_migrations/02_up_test.sql @@ -0,0 +1 @@ +UPDATE schemaversion SET version = 2; \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ee1ecc1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +astroid==2.4.1 +click==7.1.2 +Flask==1.1.2 +isort==4.3.21 +itsdangerous==1.1.0 +Jinja2==2.11.2 +lazy-object-proxy==1.4.3 +MarkupSafe==1.1.1 +mccabe==0.6.1 +pkg-resources==0.0.0 +psycopg2==2.8.5 +pylint==2.5.2 +six==1.14.0 +toml==0.10.0 +typed-ast==1.4.1 +Werkzeug==1.0.1 +wrapt==1.12.1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..1196a7a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,17 @@ +[metadata] +name = capsulflask +version = 0.0.0 +url = https://giit.cyberia.club/~forest/capsul-flask +license = BSD-3-Clause +maintainer = cyberia +maintainer_email = forest.n.johnson@gmail.com +description = Python Flask web application for capsul.org +long_description = file: README.md +long_description_content_type = text/markdown + +[options] +packages = find: +include_package_data = true +install_requires = + Flask + psycopg2 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6068493 --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup()