initial bootstrapping

.gitignore

@@ -0,0 +1,157 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+# .env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# custom folders
+.idea
+lab
+.aws
+.docker
+.deta
+backend/storage/*
+.env
+*.bat
+*.zip
+.scannerwork
+.webcache
+logs
+migrations
+*.sql
+*.json
+celerybeat-schedule*
+*.db

.pre-commit-config.yaml

@@ -0,0 +1,55 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+    - id: check-added-large-files
+    - id: check-ast
+    - id: check-builtin-literals
+    - id: check-case-conflict
+    - id: check-docstring-first
+    - id: check-executables-have-shebangs
+    - id: check-json
+    - id: check-merge-conflict
+    - id: check-shebang-scripts-are-executable
+    - id: check-symlinks
+    - id: check-toml
+    - id: check-vcs-permalinks
+    - id: check-xml
+    - id: check-yaml
+    - id: debug-statements
+      exclude: tests/
+    - id: destroyed-symlinks
+    # - id: detect-aws-credentials
+    - id: detect-private-key
+    - id: end-of-file-fixer
+      exclude: tests/test_changes/
+      files: \.(py|sh|rst|yml|yaml)$
+    - id: fix-byte-order-marker
+    - id: pretty-format-json
+      args: [--autofix]
+    - id: sort-simple-yaml
+    - id: trailing-whitespace
+
+- repo: https://github.com/charliermarsh/ruff-pre-commit
+  # Ruff version.
+  rev: "v0.8.3"
+  hooks:
+    - id: ruff
+
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+    - id: isort
+      name: isort (python)
+
+- repo: https://github.com/psf/black
+  rev: 24.10.0
+  hooks:
+    - id: black
+      exclude: tests/
+
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.19.1
+  hooks:
+    - id: pyupgrade
+      args: [--py311-plus]

Dockerfile

@@ -0,0 +1,37 @@
+FROM s3docker.francissecada.com/fjs_ubuntu:latest AS intermediate
+
+ENV PYTHONUNBUFFERED=1
+ENV LANG=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+ENV TZ="America/New_York"
+ENV LANGUAGE=en_US:en
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG GIT_BRANCH="main"
+
+RUN echo ${GIT_BRANCH}
+
+RUN mkdir -p -m 0700 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
+ENV WORKDIR="/opt/pygentic_ai"
+RUN --mount=type=ssh git clone -b ${GIT_BRANCH} git@github.com:fsecada01/Ranked-Jobs-API-Micro-Service.git ${WORKDIR}
+#COPY . ${WORKDIR}
+WORKDIR ${WORKDIR}
+
+FROM  s3docker.francissecada.com/fjs_ubuntu:latest
+
+RUN apt-get update && \
+    apt-get install -y curl
+ENV WORKDIR="/opt/pygentic_ai"
+COPY --from=intermediate ${WORKDIR} ${WORKDIR}
+WORKDIR ${WORKDIR}
+RUN useradd -ms /bin/bash celery
+RUN mkdir -p /tmp/log/celery && \
+    mkdir -p ${WORKDIR}/src/backend/logs && \
+    chmod 777 ${WORKDIR}/src/backend/logs && \
+    ln -s /run/shm /dev/shm && \
+    mkdir celerybeat-schedule && \
+    chown celery:celery celerybeat-schedule
+RUN find . -name "*.sh" -exec chmod +x {} \;
+RUN echo $WORKDIR
+RUN /bin/bash -c 'source $WORKDIR/docker/pygentic_ai/python_build.sh'
+CMD /bin/bash -c 'source $WORKDIR/docker/pygentic_ai/python_start.sh'

bin/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+python -m pip install -U pip pip-tools setuptools wheel -I
+pip-compile core_requirements.in
+#pip-sync core_requirements.txt
+pip install -r core_requirements.txt

bin/linux_build.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+export DEBIAN_FRONTEND=noninteractive
+export TZ='America/New York'
+
+
+apt-get update && apt-get upgrade -y
+apt install wget \
+  build-essential \
+  software-properties-common \
+  libncursesw5-dev \
+  libssl-dev \
+  libsqlite3-dev \
+  tk-dev \
+  libgdbm-dev \
+  libc6-dev \
+  libbz2-dev \
+  libffi-dev \
+  sox \
+  libcairo2 \
+  libcairo2-dev \
+  zlib1g-dev -y
+add-apt-repository ppa:deadsnakes/ppa
+apt install openssh-client \
+  python3.13-dev \
+  python3.13 full \
+  python3.13-venv \
+  git \
+  postgresql \
+  supervisor \
+  g++ \
+  gcc \
+  locales -y

bin/python_build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+cd /opt/pygentic_ai || exit
+python3.12 -m venv .venv
+source .venv/bin/activate
+python -m pip install -U pip pip-tools setuptools wheel
+for FILE in core_requirements dev_requirements
+do
+	pip-compile --resolver=backtracking --upgrade $FILE.in
+done
+pip-sync core_requirements.txt dev_requirements.txt

bin/start.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+__dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+cd ${__dir}/../../ || exit
+source .venv/bin/activate
+cd src || exit
+# python -m backend.utils
+gunicorn main:app -w ${WORKERS} -k uvicorn.workers.UvicornWorker \
+--timeout "${TIMEOUT}" \
+--forwarded-allow-ips "*" \
+-b 0.0.0.0:"${PORT}"

compose.yaml

@@ -0,0 +1,58 @@
+services:
+  web:
+    image: s3docker.francissecada.com/pygentic_ai:main.2024-11-30
+    deploy:
+      resources:
+        limits:
+          memory: 1024mb
+    ports:
+      - "5051:5051"
+    env_file:
+      - ./stack.env
+    environment:
+      PORT: 5051
+      SERVER_ENV: prod
+    volumes:
+      - ./src/proxy_urls.db:/opt/pygentic_ai/src/proxy_urls.db
+    labels:
+      - traefik.enable=true
+      - traefik.docker.network=proxy
+      - traefik.http.routers.pygentic_ai.entrypoints=websecure
+      - traefik.http.routers.pygentic_ai.rule=Host(`rankedjobsmicroservice.francissecada.com`)
+      - traefik.http.routers.pygentic_ai.tls=true
+    healthcheck:
+      test: curl --fail http://localhost:5051/ || exit 1
+      interval: 40s
+      timeout: 30s
+      retries: 3
+      start_period: 60s
+    networks:
+      - proxy
+  celery_service:
+    image: s3docker.francissecada.com/pygentic_ai:main.2024-11-30
+    deploy:
+      resources:
+        limits:
+          memory: 512mb
+    #    build: .
+    command: ./docker/celery/start.sh
+    env_file:
+      - ./stack.env
+    environment:
+      PORT: ${CELERY_PORT}
+      SERVER_ENV: staging
+      C_FORCE_ROOT: true
+    ports:
+      - "5052:5052"
+    labels:
+      - traefik.enable=true
+      - traefik.docker.network=proxy
+      - traefik.http.routers.celery_ranked_jobs.entrypoints=websecure
+      - traefik.http.routers.celery_ranked_jobs.rule=Host(`celery.rankedjobsmicroservice.francissecada.com`)
+    networks:
+      - proxy
+    depends_on:
+      - web
+networks:
+  proxy:
+    external: true

core_requirements.in

@@ -0,0 +1,31 @@
+aiofiles
+aiomysql
+celery
+fastapi
+fastapi-restful
+fastcrud
+flower
+greenlet
+gunicorn; platform_system != 'Windows'
+httpx
+httpx[socks]
+httpx-html
+html5lib
+hypercorn; platform_system == 'Windows'
+loguru
+lxml
+lxml[html_clean]
+python-dateutil
+python-decouple
+python-slugify
+psycopg
+pytz
+redis
+simplejson
+sqlalchemy_mixins
+sqlmodel
+sqlmodel-crud-utilities @ git+https://github.com/fsecada01/SQLModel-CRUD-Utilities@v0.0.1a1
+typing-inspect
+uvicorn
+xmljson
+xmltodict

core_requirements.txt

@@ -0,0 +1,251 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --strip-extras core_requirements.in -o core_requirements.txt
+aiofiles==24.1.0
+    # via -r core_requirements.in
+aiomysql==0.2.0
+    # via -r core_requirements.in
+amqp==5.3.1
+    # via kombu
+annotated-types==0.7.0
+    # via
+    #   pydantic
+    #   sqlmodel-crud-utilities
+anyio==4.8.0
+    # via
+    #   httpx
+    #   starlette
+appdirs==1.4.4
+    # via pyppeteer
+beautifulsoup4==4.12.3
+    # via httpx-html
+billiard==4.2.1
+    # via celery
+celery==5.4.0
+    # via
+    #   -r core_requirements.in
+    #   flower
+certifi==2024.12.14
+    # via
+    #   httpcore
+    #   httpx
+    #   pyppeteer
+click==8.1.8
+    # via
+    #   celery
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
+    #   uvicorn
+click-didyoumean==0.3.1
+    # via celery
+click-plugins==1.1.1
+    # via celery
+click-repl==0.3.0
+    # via celery
+colorama==0.4.6
+    # via
+    #   click
+    #   loguru
+    #   sqlmodel-crud-utilities
+    #   tqdm
+cssselect==1.2.0
+    # via pyquery
+fake-useragent==2.0.3
+    # via httpx-html
+fastapi==0.115.6
+    # via
+    #   -r core_requirements.in
+    #   fastapi-restful
+    #   fastcrud
+fastapi-restful==0.6.0
+    # via -r core_requirements.in
+fastcrud==0.15.5
+    # via -r core_requirements.in
+flower==2.0.1
+    # via -r core_requirements.in
+greenlet==3.1.1
+    # via
+    #   -r core_requirements.in
+    #   sqlalchemy
+    #   sqlmodel-crud-utilities
+h11==0.14.0
+    # via
+    #   httpcore
+    #   hypercorn
+    #   uvicorn
+    #   wsproto
+h2==4.1.0
+    # via hypercorn
+hpack==4.0.0
+    # via h2
+html5lib==1.1
+    # via -r core_requirements.in
+httpcore==1.0.7
+    # via httpx
+httpx==0.28.1
+    # via
+    #   -r core_requirements.in
+    #   httpx-html
+httpx-html==0.11.0.dev0
+    # via -r core_requirements.in
+humanize==4.11.0
+    # via flower
+hypercorn==0.17.3
+    # via -r core_requirements.in
+hyperframe==6.0.1
+    # via h2
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+importlib-metadata==8.5.0
+    # via pyppeteer
+kombu==5.4.2
+    # via celery
+loguru==0.7.3
+    # via
+    #   -r core_requirements.in
+    #   sqlmodel-crud-utilities
+lxml==5.3.0
+    # via
+    #   -r core_requirements.in
+    #   lxml-html-clean
+    #   pyquery
+lxml-html-clean==0.4.1
+    # via lxml
+mypy-extensions==1.0.0
+    # via typing-inspect
+parse==1.20.2
+    # via httpx-html
+priority==2.0.0
+    # via hypercorn
+prometheus-client==0.21.1
+    # via flower
+prompt-toolkit==3.0.48
+    # via click-repl
+psutil==5.9.8
+    # via fastapi-restful
+psycopg==3.2.4
+    # via -r core_requirements.in
+pydantic==2.10.5
+    # via
+    #   fastapi
+    #   fastapi-restful
+    #   fastcrud
+    #   sqlmodel
+    #   sqlmodel-crud-utilities
+pydantic-core==2.27.2
+    # via
+    #   pydantic
+    #   sqlmodel-crud-utilities
+pyee==11.1.1
+    # via pyppeteer
+pymysql==1.1.1
+    # via aiomysql
+pyppeteer==2.0.0
+    # via httpx-html
+pyquery==2.0.1
+    # via httpx-html
+python-dateutil==2.9.0.post0
+    # via
+    #   -r core_requirements.in
+    #   celery
+    #   sqlmodel-crud-utilities
+python-decouple==3.8
+    # via -r core_requirements.in
+python-dotenv==1.0.1
+    # via sqlmodel-crud-utilities
+python-slugify==8.0.4
+    # via -r core_requirements.in
+pytz==2024.2
+    # via
+    #   -r core_requirements.in
+    #   flower
+redis==5.2.1
+    # via -r core_requirements.in
+simplejson==3.19.3
+    # via -r core_requirements.in
+six==1.17.0
+    # via
+    #   html5lib
+    #   python-dateutil
+    #   sqlalchemy-mixins
+    #   sqlmodel-crud-utilities
+sniffio==1.3.1
+    # via anyio
+socksio==1.0.0
+    # via httpx
+soupsieve==2.6
+    # via beautifulsoup4
+sqlalchemy==2.0.37
+    # via
+    #   fastcrud
+    #   sqlalchemy-mixins
+    #   sqlalchemy-utils
+    #   sqlmodel
+    #   sqlmodel-crud-utilities
+sqlalchemy-mixins==2.0.5
+    # via -r core_requirements.in
+sqlalchemy-utils==0.41.2
+    # via fastcrud
+sqlmodel==0.0.22
+    # via
+    #   -r core_requirements.in
+    #   sqlmodel-crud-utilities
+sqlmodel-crud-utilities @ git+https://github.com/fsecada01/SQLModel-CRUD-Utilities@83e964f6e7b633e339e45ddcaaa49cd8617fa105
+    # via -r core_requirements.in
+starlette==0.41.3
+    # via fastapi
+text-unidecode==1.3
+    # via python-slugify
+tornado==6.4.2
+    # via flower
+tqdm==4.67.1
+    # via pyppeteer
+typing-extensions==4.12.2
+    # via
+    #   anyio
+    #   fastapi
+    #   psycopg
+    #   pydantic
+    #   pydantic-core
+    #   pyee
+    #   sqlalchemy
+    #   sqlmodel-crud-utilities
+    #   typing-inspect
+typing-inspect==0.9.0
+    # via -r core_requirements.in
+tzdata==2024.2
+    # via
+    #   celery
+    #   kombu
+    #   psycopg
+urllib3==1.26.20
+    # via pyppeteer
+uvicorn==0.34.0
+    # via -r core_requirements.in
+vine==5.1.0
+    # via
+    #   amqp
+    #   celery
+    #   kombu
+w3lib==2.2.1
+    # via httpx-html
+wcwidth==0.2.13
+    # via prompt-toolkit
+webencodings==0.5.1
+    # via html5lib
+websockets==10.4
+    # via pyppeteer
+win32-setctime==1.2.0
+    # via
+    #   loguru
+    #   sqlmodel-crud-utilities
+wsproto==1.2.0
+    # via hypercorn
+xmljson==0.2.1
+    # via -r core_requirements.in
+xmltodict==0.14.2
+    # via -r core_requirements.in
+zipp==3.21.0
+    # via importlib-metadata

dev_requirements.in

@@ -0,0 +1,10 @@
+-c core_requirements.txt
+
+alembic
+black
+fastapi-debug-toolbar
+isort
+jupyterlab
+jupyterlab-code-formatter
+pre-commit
+ruff

dev_requirements.txt

@@ -0,0 +1,403 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --strip-extras dev_requirements.in -o dev_requirements.txt
+alembic==1.14.0
+    # via -r dev_requirements.in
+annotated-types==0.7.0
+    # via
+    #   -c core_requirements.txt
+    #   pydantic
+anyio==4.8.0
+    # via
+    #   -c core_requirements.txt
+    #   fastapi-debug-toolbar
+    #   httpx
+    #   jupyter-server
+    #   starlette
+argon2-cffi==23.1.0
+    # via jupyter-server
+argon2-cffi-bindings==21.2.0
+    # via argon2-cffi
+arrow==1.3.0
+    # via isoduration
+asttokens==3.0.0
+    # via stack-data
+async-lru==2.0.4
+    # via jupyterlab
+attrs==24.3.0
+    # via
+    #   jsonschema
+    #   referencing
+babel==2.16.0
+    # via jupyterlab-server
+beautifulsoup4==4.12.3
+    # via
+    #   -c core_requirements.txt
+    #   nbconvert
+black==24.10.0
+    # via -r dev_requirements.in
+bleach==6.2.0
+    # via nbconvert
+certifi==2024.12.14
+    # via
+    #   -c core_requirements.txt
+    #   httpcore
+    #   httpx
+    #   requests
+cffi==1.17.1
+    # via argon2-cffi-bindings
+cfgv==3.4.0
+    # via pre-commit
+charset-normalizer==3.4.1
+    # via requests
+click==8.1.8
+    # via
+    #   -c core_requirements.txt
+    #   black
+colorama==0.4.6
+    # via
+    #   -c core_requirements.txt
+    #   click
+    #   ipython
+comm==0.2.2
+    # via ipykernel
+debugpy==1.8.11
+    # via ipykernel
+decorator==5.1.1
+    # via ipython
+defusedxml==0.7.1
+    # via nbconvert
+distlib==0.3.9
+    # via virtualenv
+executing==2.1.0
+    # via stack-data
+fastapi==0.115.6
+    # via
+    #   -c core_requirements.txt
+    #   fastapi-debug-toolbar
+fastapi-debug-toolbar==0.6.3
+    # via -r dev_requirements.in
+fastjsonschema==2.21.1
+    # via nbformat
+filelock==3.16.1
+    # via virtualenv
+fqdn==1.5.1
+    # via jsonschema
+greenlet==3.1.1
+    # via
+    #   -c core_requirements.txt
+    #   sqlalchemy
+h11==0.14.0
+    # via
+    #   -c core_requirements.txt
+    #   httpcore
+httpcore==1.0.7
+    # via
+    #   -c core_requirements.txt
+    #   httpx
+httpx==0.28.1
+    # via
+    #   -c core_requirements.txt
+    #   jupyterlab
+identify==2.6.5
+    # via pre-commit
+idna==3.10
+    # via
+    #   -c core_requirements.txt
+    #   anyio
+    #   httpx
+    #   jsonschema
+    #   requests
+ipykernel==6.29.5
+    # via jupyterlab
+ipython==8.31.0
+    # via ipykernel
+isoduration==20.11.0
+    # via jsonschema
+isort==5.13.2
+    # via -r dev_requirements.in
+jedi==0.19.2
+    # via ipython
+jinja2==3.1.5
+    # via
+    #   fastapi-debug-toolbar
+    #   jupyter-server
+    #   jupyterlab
+    #   jupyterlab-server
+    #   nbconvert
+json5==0.10.0
+    # via jupyterlab-server
+jsonpointer==3.0.0
+    # via jsonschema
+jsonschema==4.23.0
+    # via
+    #   jupyter-events
+    #   jupyterlab-server
+    #   nbformat
+jsonschema-specifications==2024.10.1
+    # via jsonschema
+jupyter-client==8.6.3
+    # via
+    #   ipykernel
+    #   jupyter-server
+    #   nbclient
+jupyter-core==5.7.2
+    # via
+    #   ipykernel
+    #   jupyter-client
+    #   jupyter-server
+    #   jupyterlab
+    #   nbclient
+    #   nbconvert
+    #   nbformat
+jupyter-events==0.11.0
+    # via jupyter-server
+jupyter-lsp==2.2.5
+    # via jupyterlab
+jupyter-server==2.15.0
+    # via
+    #   jupyter-lsp
+    #   jupyterlab
+    #   jupyterlab-code-formatter
+    #   jupyterlab-server
+    #   notebook-shim
+jupyter-server-terminals==0.5.3
+    # via jupyter-server
+jupyterlab==4.3.4
+    # via -r dev_requirements.in
+jupyterlab-code-formatter==3.0.2
+    # via -r dev_requirements.in
+jupyterlab-pygments==0.3.0
+    # via nbconvert
+jupyterlab-server==2.27.3
+    # via jupyterlab
+mako==1.3.8
+    # via alembic
+markupsafe==3.0.2
+    # via
+    #   jinja2
+    #   mako
+    #   nbconvert
+matplotlib-inline==0.1.7
+    # via
+    #   ipykernel
+    #   ipython
+mistune==3.1.0
+    # via nbconvert
+mypy-extensions==1.0.0
+    # via
+    #   -c core_requirements.txt
+    #   black
+nbclient==0.10.2
+    # via nbconvert
+nbconvert==7.16.5
+    # via jupyter-server
+nbformat==5.10.4
+    # via
+    #   jupyter-server
+    #   nbclient
+    #   nbconvert
+nest-asyncio==1.6.0
+    # via ipykernel
+nodeenv==1.9.1
+    # via pre-commit
+notebook-shim==0.2.4
+    # via jupyterlab
+overrides==7.7.0
+    # via jupyter-server
+packaging==24.2
+    # via
+    #   black
+    #   ipykernel
+    #   jupyter-server
+    #   jupyterlab
+    #   jupyterlab-code-formatter
+    #   jupyterlab-server
+    #   nbconvert
+pandocfilters==1.5.1
+    # via nbconvert
+parso==0.8.4
+    # via jedi
+pathspec==0.12.1
+    # via black
+platformdirs==4.3.6
+    # via
+    #   black
+    #   jupyter-core
+    #   virtualenv
+pre-commit==4.0.1
+    # via -r dev_requirements.in
+prometheus-client==0.21.1
+    # via
+    #   -c core_requirements.txt
+    #   jupyter-server
+prompt-toolkit==3.0.48
+    # via
+    #   -c core_requirements.txt
+    #   ipython
+psutil==5.9.8
+    # via
+    #   -c core_requirements.txt
+    #   ipykernel
+pure-eval==0.2.3
+    # via stack-data
+pycparser==2.22
+    # via cffi
+pydantic==2.10.5
+    # via
+    #   -c core_requirements.txt
+    #   fastapi
+    #   fastapi-debug-toolbar
+    #   pydantic-extra-types
+    #   pydantic-settings
+pydantic-core==2.27.2
+    # via
+    #   -c core_requirements.txt
+    #   pydantic
+pydantic-extra-types==2.10.2
+    # via fastapi-debug-toolbar
+pydantic-settings==2.7.1
+    # via fastapi-debug-toolbar
+pygments==2.19.1
+    # via
+    #   ipython
+    #   nbconvert
+pyinstrument==5.0.0
+    # via fastapi-debug-toolbar
+python-dateutil==2.9.0.post0
+    # via
+    #   -c core_requirements.txt
+    #   arrow
+    #   jupyter-client
+python-dotenv==1.0.1
+    # via
+    #   -c core_requirements.txt
+    #   pydantic-settings
+python-json-logger==3.2.1
+    # via jupyter-events
+pywin32==308
+    # via jupyter-core
+pywinpty==2.0.14
+    # via
+    #   jupyter-server
+    #   jupyter-server-terminals
+    #   terminado
+pyyaml==6.0.2
+    # via
+    #   jupyter-events
+    #   pre-commit
+pyzmq==26.2.0
+    # via
+    #   ipykernel
+    #   jupyter-client
+    #   jupyter-server
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+    #   jupyter-events
+requests==2.32.3
+    # via jupyterlab-server
+rfc3339-validator==0.1.4
+    # via
+    #   jsonschema
+    #   jupyter-events
+rfc3986-validator==0.1.1
+    # via
+    #   jsonschema
+    #   jupyter-events
+rpds-py==0.22.3
+    # via
+    #   jsonschema
+    #   referencing
+ruff==0.9.2
+    # via -r dev_requirements.in
+send2trash==1.8.3
+    # via jupyter-server
+setuptools==75.8.0
+    # via jupyterlab
+six==1.17.0
+    # via
+    #   -c core_requirements.txt
+    #   python-dateutil
+    #   rfc3339-validator
+sniffio==1.3.1
+    # via
+    #   -c core_requirements.txt
+    #   anyio
+soupsieve==2.6
+    # via
+    #   -c core_requirements.txt
+    #   beautifulsoup4
+sqlalchemy==2.0.37
+    # via
+    #   -c core_requirements.txt
+    #   alembic
+sqlparse==0.5.3
+    # via fastapi-debug-toolbar
+stack-data==0.6.3
+    # via ipython
+starlette==0.41.3
+    # via
+    #   -c core_requirements.txt
+    #   fastapi
+terminado==0.18.1
+    # via
+    #   jupyter-server
+    #   jupyter-server-terminals
+tinycss2==1.4.0
+    # via bleach
+tornado==6.4.2
+    # via
+    #   -c core_requirements.txt
+    #   ipykernel
+    #   jupyter-client
+    #   jupyter-server
+    #   jupyterlab
+    #   terminado
+traitlets==5.14.3
+    # via
+    #   comm
+    #   ipykernel
+    #   ipython
+    #   jupyter-client
+    #   jupyter-core
+    #   jupyter-events
+    #   jupyter-server
+    #   jupyterlab
+    #   matplotlib-inline
+    #   nbclient
+    #   nbconvert
+    #   nbformat
+types-python-dateutil==2.9.0.20241206
+    # via arrow
+typing-extensions==4.12.2
+    # via
+    #   -c core_requirements.txt
+    #   alembic
+    #   anyio
+    #   fastapi
+    #   pydantic
+    #   pydantic-core
+    #   pydantic-extra-types
+    #   sqlalchemy
+uri-template==1.3.0
+    # via jsonschema
+urllib3==1.26.20
+    # via
+    #   -c core_requirements.txt
+    #   requests
+virtualenv==20.29.0
+    # via pre-commit
+wcwidth==0.2.13
+    # via
+    #   -c core_requirements.txt
+    #   prompt-toolkit
+webcolors==24.11.1
+    # via jsonschema
+webencodings==0.5.1
+    # via
+    #   -c core_requirements.txt
+    #   bleach
+    #   tinycss2
+websocket-client==1.8.0
+    # via jupyter-server

docker/celery/start.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#set -o errexit
+#set -o nounset
+
+rm -f './celerybeat.pid'
+
+cd /opt/pygentic_ai
+source .venv/bin/activate
+
+#apt-get install supervisor -y
+#
+#echo user=nobody >> /etc/supervisor/supervisord.conf
+#service supervisor stop
+#service supervisor start
+#supervisorctl start celeryd
+#supervisorctl start celerybeat
+
+celery -A cworker --workdir /opt/pygentic_ai/src worker -l INFO --detach
+celery -A cworker --workdir \
+/opt/pygentic_ai/src beat \
+--scheduler celery.beat.Scheduler \
+-l INFO \
+--detach
+
+sleep 10
+
+worker_ready() {
+    celery -A cworker --workdir /opt/pygentic_ai/src inspect ping
+}
+
+
+
+until worker_ready; do
+  >&2 echo 'Celery workers not available'
+  sleep 1
+done
+>&2 echo 'Celery workers is available'
+
+celery -A cworker --workdir /opt/pygentic_ai/src flower --basic-auth=${FLOWER_USERNAME}:${FLOWER_PASSWORD} --port=${CELERY_PORT} -l INFO

docker/ranked_jobs_ms/build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export DEBIAN_FRONTEND=noninteractive
+export TZ='America/New York'
+
+apt-get update && apt-get upgrade -y
+apt-get install openssh-client \
+    python3-dev \
+    python3-full \
+    python3-venv \
+    python3-pip \
+    git \
+    postgresql \
+    supervisor \
+    g++ \
+    gcc \
+    locales -y

docker/ranked_jobs_ms/python_build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+curl -LsSf https://astral.sh/uv/install.sh | sh
+PATH="/root/.local/bin/:$PATH"
+cd /opt/pygentic_ai || exit
+uv venv .venv
+source .venv/bin/activate
+for FILE in core_requirements dev_requirements
+do
+	uv pip compile --upgrade $FILE.in -o $FILE.txt
+done
+uv pip sync core_requirements.txt

docker/ranked_jobs_ms/python_start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+__dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+cd ${__dir}/../.. || exit
+source .venv/bin/activate
+cd src || exit
+gunicorn main:app -w ${WORKERS} -k uvicorn.workers.UvicornWorker \
+--timeout "${TIMEOUT}" \
+--forwarded-allow-ips "*" \
+-b 0.0.0.0:"${PORT}"

labs/Untitled.ipynb

@@ -0,0 +1,102 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "11b8593d-dea7-44e0-beee-c0c6ffea7d1a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "2b9388f3-c7e0-401d-8d0b-9ef43b78bd73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "work_dir = Path(__name__).resolve().parent.parent / \"src\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c0625c16-9e78-4525-aa84-2194480aaf0c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "WindowsPath('C:/dev/python/interview_eval_project/highmark_agentic_ai/src')"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "work_dir"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "481c534a-5120-457c-8f5e-3dbb2a5f1c3d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "backend = work_dir / \"backend\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "69c64859-7ad8-4e04-b974-ad54c8647bbf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if backend.exists():\n",
+    "    for folder in (\"core\", \"db\", \"server\", \"settings\", \"site\"):\n",
+    "        directory = backend / folder\n",
+    "        if not directory.exists():\n",
+    "            directory.mkdir(exist_ok=True)\n",
+    "        for file in (\"__init__\", \"core\", \"main\", \"utils\", \"consts\"):\n",
+    "            file_path = directory / file\n",
+    "            # file_path.touch(exist_ok=True)\n",
+    "            file_path.unlink(missing_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "50e9ff3d-f23a-402d-8bad-5f7f821d6b84",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Pydantic Agent AI",
+   "language": "python",
+   "name": "agentic_ai"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pyproject.toml

@@ -0,0 +1,288 @@
+[tool.ruff]
+fix = true
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+    "migrations",
+]
+
+line-length = 80
+
+[tool.ruff.lint]
+preview = true
+select = ["E", "F", "B"]
+#select = ["E", "F", "I", "PL", "I", "B"]
+ignore = ["B008", "E501"]  # black will take care of line-length violations
+ignore-init-module-imports = true
+unfixable = []
+
+[tool.ruff.lint.mccabe]
+max-complexity = 10
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F403", "F405", "E501"]
+"consts.py" = ["E501"]
+
+[tool.black]
+line-length = 80
+target-version = ['py39', 'py310', 'py311', 'py312', 'py313']
+extend-exclude = '''
+/(
+  # The following are specific to Black, you probably don't want those.
+  | blib2to3
+  | tests/data
+  | profiling
+)/
+'''
+[tool.isort]
+profile = "black"
+line_length = 80
+
+[tool.uv.sources]
+sqlmodel-crud-utilities = { git = "https://github.com/fsecada01/SQLModel-CRUD-Utilities", rev = "83e964f6e7b633e339e45ddcaaa49cd8617fa105" }
+#multi_line_output = 3
+#include_trailing_comma = true
+#force_grid_wrap = 0
+
+[project]
+name = "agentic_ai_service"
+authors = [
+    { name = "Francis Secada", email = "francis.secada@gmail.com" }
+]
+readme = "README.md"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+version = "1.0.0b0"
+dependencies = [
+    "aiofiles==24.1.0",
+    "aiomysql==0.2.0",
+    "amqp==5.3.1",
+    "annotated-types==0.7.0",
+    "anyio==4.8.0",
+    "appdirs==1.4.4",
+    "beautifulsoup4==4.12.3",
+    "billiard==4.2.1",
+    "celery==5.4.0",
+    "certifi==2024.12.14",
+    "click-didyoumean==0.3.1",
+    "click-plugins==1.1.1",
+    "click-repl==0.3.0",
+    "click==8.1.8",
+    "colorama==0.4.6",
+    "cssselect==1.2.0",
+    "fake-useragent==2.0.3",
+    "fastapi-restful==0.6.0",
+    "fastapi==0.115.6",
+    "fastcrud==0.15.5",
+    "flower==2.0.1",
+    "greenlet==3.1.1",
+    "h11==0.14.0",
+    "h2==4.1.0",
+    "hpack==4.0.0",
+    "html5lib==1.1",
+    "httpcore==1.0.7",
+    "httpx-html==0.11.0.dev0",
+    "httpx==0.28.1",
+    "humanize==4.11.0",
+    "hypercorn==0.17.3",
+    "hyperframe==6.0.1",
+    "idna==3.10",
+    "importlib-metadata==8.5.0",
+    "kombu==5.4.2",
+    "loguru==0.7.3",
+    "lxml-html-clean==0.4.1",
+    "lxml==5.3.0",
+    "mypy-extensions==1.0.0",
+    "parse==1.20.2",
+    "priority==2.0.0",
+    "prometheus-client==0.21.1",
+    "prompt-toolkit==3.0.48",
+    "psutil==5.9.8",
+    "psycopg==3.2.4",
+    "pydantic-core==2.27.2",
+    "pydantic==2.10.5",
+    "pyee==11.1.1",
+    "pymysql==1.1.1",
+    "pyppeteer==2.0.0",
+    "pyquery==2.0.1",
+    "python-dateutil==2.9.0.post0",
+    "python-decouple==3.8",
+    "python-dotenv==1.0.1",
+    "python-slugify==8.0.4",
+    "pytz==2024.2",
+    "redis==5.2.1",
+    "simplejson==3.19.3",
+    "six==1.17.0",
+    "sniffio==1.3.1",
+    "socksio==1.0.0",
+    "soupsieve==2.6",
+    "sqlalchemy-mixins==2.0.5",
+    "sqlalchemy-utils==0.41.2",
+    "sqlalchemy==2.0.37",
+    "sqlmodel-crud-utilities",
+    "sqlmodel==0.0.22",
+    "starlette==0.41.3",
+    "text-unidecode==1.3",
+    "tornado==6.4.2",
+    "tqdm==4.67.1",
+    "typing-extensions==4.12.2",
+    "typing-inspect==0.9.0",
+    "tzdata==2024.2",
+    "urllib3==1.26.20",
+    "uvicorn==0.34.0",
+    "vine==5.1.0",
+    "w3lib==2.2.1",
+    "wcwidth==0.2.13",
+    "webencodings==0.5.1",
+    "websockets==10.4",
+    "win32-setctime==1.2.0",
+    "wsproto==1.2.0",
+    "xmljson==0.2.1",
+    "xmltodict==0.14.2",
+    "zipp==3.21.0",
+]
+
+[dependency-groups]
+dev = [
+    "alembic==1.14.0",
+    "annotated-types==0.7.0",
+    "anyio==4.8.0",
+    "argon2-cffi-bindings==21.2.0",
+    "argon2-cffi==23.1.0",
+    "arrow==1.3.0",
+    "asttokens==3.0.0",
+    "async-lru==2.0.4",
+    "attrs==24.3.0",
+    "babel==2.16.0",
+    "beautifulsoup4==4.12.3",
+    "black==24.10.0",
+    "bleach==6.2.0",
+    "certifi==2024.12.14",
+    "cffi==1.17.1",
+    "cfgv==3.4.0",
+    "charset-normalizer==3.4.1",
+    "click==8.1.8",
+    "colorama==0.4.6",
+    "comm==0.2.2",
+    "debugpy==1.8.11",
+    "decorator==5.1.1",
+    "defusedxml==0.7.1",
+    "distlib==0.3.9",
+    "executing==2.1.0",
+    "fastapi-debug-toolbar==0.6.3",
+    "fastapi==0.115.6",
+    "fastjsonschema==2.21.1",
+    "filelock==3.16.1",
+    "fqdn==1.5.1",
+    "greenlet==3.1.1",
+    "h11==0.14.0",
+    "httpcore==1.0.7",
+    "httpx==0.28.1",
+    "identify==2.6.5",
+    "idna==3.10",
+    "ipykernel==6.29.5",
+    "ipython==8.31.0",
+    "isoduration==20.11.0",
+    "isort==5.13.2",
+    "jedi==0.19.2",
+    "jinja2==3.1.5",
+    "json5==0.10.0",
+    "jsonpointer==3.0.0",
+    "jsonschema-specifications==2024.10.1",
+    "jsonschema==4.23.0",
+    "jupyter-client==8.6.3",
+    "jupyter-core==5.7.2",
+    "jupyter-events==0.11.0",
+    "jupyter-lsp==2.2.5",
+    "jupyter-server-terminals==0.5.3",
+    "jupyter-server==2.15.0",
+    "jupyterlab-code-formatter==3.0.2",
+    "jupyterlab-pygments==0.3.0",
+    "jupyterlab-server==2.27.3",
+    "jupyterlab==4.3.4",
+    "mako==1.3.8",
+    "markupsafe==3.0.2",
+    "matplotlib-inline==0.1.7",
+    "mistune==3.1.0",
+    "mypy-extensions==1.0.0",
+    "nbclient==0.10.2",
+    "nbconvert==7.16.5",
+    "nbformat==5.10.4",
+    "nest-asyncio==1.6.0",
+    "nodeenv==1.9.1",
+    "notebook-shim==0.2.4",
+    "overrides==7.7.0",
+    "packaging==24.2",
+    "pandocfilters==1.5.1",
+    "parso==0.8.4",
+    "pathspec==0.12.1",
+    "platformdirs==4.3.6",
+    "pre-commit==4.0.1",
+    "prometheus-client==0.21.1",
+    "prompt-toolkit==3.0.48",
+    "psutil==5.9.8",
+    "pure-eval==0.2.3",
+    "pycparser==2.22",
+    "pydantic-core==2.27.2",
+    "pydantic-extra-types==2.10.2",
+    "pydantic-settings==2.7.1",
+    "pydantic==2.10.5",
+    "pygments==2.19.1",
+    "pyinstrument==5.0.0",
+    "python-dateutil==2.9.0.post0",
+    "python-dotenv==1.0.1",
+    "python-json-logger==3.2.1",
+    "pywin32==308",
+    "pywinpty==2.0.14",
+    "pyyaml==6.0.2",
+    "pyzmq==26.2.0",
+    "referencing==0.35.1",
+    "requests==2.32.3",
+    "rfc3339-validator==0.1.4",
+    "rfc3986-validator==0.1.1",
+    "rpds-py==0.22.3",
+    "ruff==0.9.2",
+    "send2trash==1.8.3",
+    "setuptools==75.8.0",
+    "six==1.17.0",
+    "sniffio==1.3.1",
+    "soupsieve==2.6",
+    "sqlalchemy==2.0.37",
+    "sqlparse==0.5.3",
+    "stack-data==0.6.3",
+    "starlette==0.41.3",
+    "terminado==0.18.1",
+    "tinycss2==1.4.0",
+    "tornado==6.4.2",
+    "traitlets==5.14.3",
+    "types-python-dateutil==2.9.0.20241206",
+    "typing-extensions==4.12.2",
+    "uri-template==1.3.0",
+    "urllib3==1.26.20",
+    "virtualenv==20.29.0",
+    "wcwidth==0.2.13",
+    "webcolors==24.11.1",
+    "webencodings==0.5.1",
+    "websocket-client==1.8.0",
+]