From 12485792d21d147c407d073af77bc26ea3bea502 Mon Sep 17 00:00:00 2001 From: Mabe Date: Wed, 23 Sep 2026 20:47:58 -0700 Subject: [PATCH] Add migration support with Alembic; implement migration scripts and configuration --- .gitignore | 3 +- alembic.ini | 152 +++++++++++++++++++++++++++++++++++++ cli.py | 2 + main.py | 21 +++++ migration/README | 1 + migration/alembic_scipt.py | 67 ++++++++++++++++ migration/env.py | 84 ++++++++++++++++++++ migration/script.py.mako | 28 +++++++ 8 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 alembic.ini create mode 100644 migration/README create mode 100644 migration/alembic_scipt.py create mode 100644 migration/env.py create mode 100644 migration/script.py.mako diff --git a/.gitignore b/.gitignore index 30b008b..8dda6fa 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,5 @@ coverage/ *.tmp *.bak *.old -.cache/ \ No newline at end of file +.cache/ +migration_status.flag diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..51682ef --- /dev/null +++ b/alembic.ini @@ -0,0 +1,152 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/migration + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# the special token `%(here)s` is available which indicates the absolute path +# to this configuration file. +# +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator (New in Alembic 1.16.0, supersedes version_path_separator); +# This indicates what character is used to +# split lists of file paths, including version_locations and prepend_sys_path +# within configparser files such as alembic.ini. +# +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +# See notes in "escaping characters in ini files" for guidelines on +# passwords +sqlalchemy.url = driver://user:pass@localhost/dbname + +# [post_write_hooks] +# This section defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, +# against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S \ No newline at end of file diff --git a/cli.py b/cli.py index 56c27c1..642f22c 100644 --- a/cli.py +++ b/cli.py @@ -188,4 +188,6 @@ print(r""" """) logging.info("Menu is starting") time.sleep(2.5) +while not os.path.exists("migration_status.flag"): + time.sleep(0.5) menu() diff --git a/main.py b/main.py index f231d79..f7918bd 100644 --- a/main.py +++ b/main.py @@ -4,6 +4,7 @@ import sys import os import auth.api_security as auth import time +import threading try: os.remove("api.log") @@ -28,6 +29,23 @@ except Exception as e: env = os.environ.copy() env["PYTHONIOENCODING"] = "utf-8" +#migration +def run_migration(): + global migration_done + process = subprocess.Popen( + ["python", "migration/alembic_scipt.py"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + process.wait() + migration_done = (process.returncode == 0) + with open("migration_status.flag", "w") as f: + f.write("done" if migration_done else "failed") + +thread = threading.Thread(target=run_migration) +thread.start() + +#cli and api log_file = open("api_server.log", "w", encoding="utf-8") api_process = subprocess.Popen( ["fastapi", "dev", "api_server.py"], @@ -40,6 +58,9 @@ cli_process = subprocess.Popen([sys.executable, "cli.py"]) api_process.wait() cli_process.wait() +time.sleep(60) +os.remove("migration_status.flag") + # automatic durataion validation check while True: time.sleep(300) diff --git a/migration/README b/migration/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/migration/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migration/alembic_scipt.py b/migration/alembic_scipt.py new file mode 100644 index 0000000..7111190 --- /dev/null +++ b/migration/alembic_scipt.py @@ -0,0 +1,67 @@ +import os +from alembic.config import Config +from alembic.script import ScriptDirectory +from alembic.runtime.migration import MigrationContext +from alembic import command +from sqlalchemy import create_engine +from alembic import command +from dotenv import load_dotenv +import logging + +#Logging + +logging.basicConfig( + filename='app.log', # Log to a file + level=logging.DEBUG, # Log everything (DEBUG and above) + format='%(asctime)s %(levelname)s %(message)s' +) + +logging.info('Application started') +try: + logging.debug('Debug info here') +except Exception as e: + logging.exception('An error occurred') + + +load_dotenv() +db_name = os.getenv("DB_NAME") +db_psswd = os.getenv("DB_PASSWORD") +db_user = os.getenv("DB_USER") +db_port = os.getenv("DB_PORT") or "3306" +db_host = os.getenv("DB_HOST") +if db_host == None: + os._exit(1) +engine_url = f"mariadb+mariadbconnector://{db_user}:{db_psswd}@{db_host}:{db_port}/{db_name}" + +base_dir = os.path.dirname(__file__) +ini_path = os.path.join(base_dir, "..", "alembic.ini") +alembic_cfg = Config(ini_path) +alembic_cfg.set_section_option(alembic_cfg.config_ini_section, "sqlalchemy.url", engine_url) +script = ScriptDirectory.from_config(alembic_cfg) + +engine = create_engine(engine_url) + +def check_db(): + with engine.connect() as conn: + context = MigrationContext.configure(conn) + current_rev = context.get_current_revision() + head_rev = script.get_current_head() + if current_rev != head_rev: + command.upgrade(alembic_cfg, "head") + return False + +def auto_lookup(): + command.revision(alembic_cfg, message="add tables and rows", autogenerate=True) + script = ScriptDirectory.from_config(alembic_cfg) + head_rev = script.get_revision(script.get_current_head()) + with open(head_rev.path) as f: + content = f.read() + if "pass\n # ### end Alembic commands ###" in content: + os.remove(head_rev.path) + command.stamp(alembic_cfg, "head") + logging.info("No changes DB detected") + else: + command.upgrade(alembic_cfg, "head") + logging.info("New DB changes, upgrade table") + +auto_lookup() \ No newline at end of file diff --git a/migration/env.py b/migration/env.py new file mode 100644 index 0000000..6852d2b --- /dev/null +++ b/migration/env.py @@ -0,0 +1,84 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +from models import Base + +metadata = Base.metadata + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() +if config.config_file_name is not None: + fileConfig(config.config_file_name) \ No newline at end of file diff --git a/migration/script.py.mako b/migration/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/migration/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"}