From 1442af6828db7a8062d400eabb4d77f4cd52969a Mon Sep 17 00:00:00 2001 From: Drew Bednar Date: Wed, 21 Sep 2022 20:30:31 -0400 Subject: [PATCH] Adding alembic and sql support --- .gitignore | 3 + .pylintrc | 2 + alembic.ini | 105 ++++++++++++++++++++++++++++++ alembic/README | 1 + alembic/env.py | 85 ++++++++++++++++++++++++ alembic/script.py.mako | 24 +++++++ alembic/versions/4e1bcbead27f_.py | 28 ++++++++ alembic/versions/b03fcdc9b301_.py | 36 ++++++++++ main.py | 24 +------ project/__init__.py | 18 +++++ project/celery_utils.py | 11 ++++ project/config.py | 47 +++++++++++++ project/database.py | 13 ++++ project/users/__init__.py | 9 +++ project/users/models.py | 18 +++++ project/users/tasks.py | 15 +++++ requirements.in | 2 + requirements.txt | 20 ++++++ tasks.py | 27 ++++++++ 19 files changed, 467 insertions(+), 21 deletions(-) create mode 100644 .pylintrc create mode 100644 alembic.ini create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/4e1bcbead27f_.py create mode 100644 alembic/versions/b03fcdc9b301_.py create mode 100644 project/__init__.py create mode 100644 project/celery_utils.py create mode 100644 project/config.py create mode 100644 project/database.py create mode 100644 project/users/__init__.py create mode 100644 project/users/models.py create mode 100644 project/users/tasks.py diff --git a/.gitignore b/.gitignore index 29dba5a..a3f8f3e 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,6 @@ dmypy.json # Cython debug symbols cython_debug/ + +# sqlite +*.sqlite3 diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..f921c98 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,2 @@ +[MASTER] +ignore-paths=^alembic\\.*$|^alembic/.*$ diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..7fa7c98 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,105 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..2500aa1 --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..646910b --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,85 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +from project import create_app +from project.config import settings +from project.database import Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +config.set_main_option("sqlalchemy.url", str(settings.DATABASE_URL)) + +# Used create_app to create a new fastapi_app instance to ensure the relevant models are loaded. +fastapi_app = create_app() + +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..55df286 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/4e1bcbead27f_.py b/alembic/versions/4e1bcbead27f_.py new file mode 100644 index 0000000..a990b93 --- /dev/null +++ b/alembic/versions/4e1bcbead27f_.py @@ -0,0 +1,28 @@ +"""empty message + +Revision ID: 4e1bcbead27f +Revises: +Create Date: 2022-09-21 20:11:34.057323 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "4e1bcbead27f" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/b03fcdc9b301_.py b/alembic/versions/b03fcdc9b301_.py new file mode 100644 index 0000000..d89769b --- /dev/null +++ b/alembic/versions/b03fcdc9b301_.py @@ -0,0 +1,36 @@ +"""empty message + +Revision ID: b03fcdc9b301 +Revises: 4e1bcbead27f +Create Date: 2022-09-23 15:50:30.792964 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "b03fcdc9b301" +down_revision = "4e1bcbead27f" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "users", + sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("username", sa.String(length=128), nullable=False), + sa.Column("email", sa.String(length=128), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("email"), + sa.UniqueConstraint("username"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("users") + # ### end Alembic commands ### diff --git a/main.py b/main.py index 353238f..0df34c1 100644 --- a/main.py +++ b/main.py @@ -1,22 +1,4 @@ -from celery import Celery -from fastapi import FastAPI +from project import create_app -app = FastAPI() - - -celery = Celery( - __name__, broker="redis://127.0.0.1:6379/0", backend="redis://127.0.0.1:6379/0" -) - - -@app.get("/") -async def root(): - return {"message": "Hello World"} - - -@celery.task -def divide(x, y): - import time - - time.sleep(5) - return x / y +app = create_app() +celery = app.celery_app diff --git a/project/__init__.py b/project/__init__.py new file mode 100644 index 0000000..1ee0803 --- /dev/null +++ b/project/__init__.py @@ -0,0 +1,18 @@ +from fastapi import FastAPI + +from project.celery_utils import create_celery + + +def create_app() -> FastAPI: + app = FastAPI() + app.celery_app = create_celery() + + from .users import users_router + + app.include_router(users_router) + + @app.get("/") + async def root(): + return {"message": "Hello World"} + + return app diff --git a/project/celery_utils.py b/project/celery_utils.py new file mode 100644 index 0000000..a97a25f --- /dev/null +++ b/project/celery_utils.py @@ -0,0 +1,11 @@ +from celery import current_app as celery_current_app + +from project.config import settings + + +def create_celery(): + """Clerey factory function.""" + celery_app = celery_current_app + celery_app.config_from_object(settings, namespace="CELERY") + + return celery_app diff --git a/project/config.py b/project/config.py new file mode 100644 index 0000000..c4273d0 --- /dev/null +++ b/project/config.py @@ -0,0 +1,47 @@ +import os +import pathlib +from functools import lru_cache + + +class BaseConfig: + BASE_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent + + DATABASE_URL: str = os.environ.get( + "DATABASE_URL", f"sqlite:///{BASE_DIR}/db.sqlite3" + ) + DATABASE_CONNECT_DICT: dict = {} + + CELERY_BROKER_URL: str = os.environ.get( + "CELERY_BROKER_URL", "redis://127.0.0.1:6379/0" + ) + CELERY_RESULT_BACKEND: str = os.environ.get( + "CELERY_RESULT_BACKEND", "redis://127.0.0.1:6379/0" + ) + + +class DevelopmentConfig(BaseConfig): + pass + + +class ProductionConfig(BaseConfig): + pass + + +class TestingConfig(BaseConfig): + pass + + +@lru_cache() +def get_settings(): + config_cls_dict = { + "development": DevelopmentConfig, + "production": ProductionConfig, + "testing": TestingConfig, + } + + config_name = os.environ.get("FASTAPI_CONFIG", "development") + config_cls = config_cls_dict[config_name] + return config_cls() + + +settings = get_settings() diff --git a/project/database.py b/project/database.py new file mode 100644 index 0000000..ad5b8c4 --- /dev/null +++ b/project/database.py @@ -0,0 +1,13 @@ +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +from project.config import settings + +# https://fastapi.tiangolo.com/tutorial/sql-databases/#create-the-sqlalchemy-engine +engine = create_engine( + settings.DATABASE_URL, connect_args=settings.DATABASE_CONNECT_DICT +) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +Base = declarative_base() diff --git a/project/users/__init__.py b/project/users/__init__.py new file mode 100644 index 0000000..4c574b2 --- /dev/null +++ b/project/users/__init__.py @@ -0,0 +1,9 @@ +from fastapi import APIRouter + +from project.celery_utils import create_celery + +users_router = APIRouter( + prefix="/users", +) + +from . import models, tasks # noqa diff --git a/project/users/models.py b/project/users/models.py new file mode 100644 index 0000000..f629fd8 --- /dev/null +++ b/project/users/models.py @@ -0,0 +1,18 @@ +from sqlalchemy import BigInteger, Column, String + +from project.database import Base + + +class User(Base): + + __tablename__ = "users" + + id = Column(BigInteger, primary_key=True, autoincrement=True) + username = Column(String(128), unique=True, nullable=False) + email = Column(String(128), unique=True, nullable=False) + + def __init__( + self, username, email, *args, **kwargs # pylint: disable=unused-argument + ) -> None: + self.username = username + self.email = email diff --git a/project/users/tasks.py b/project/users/tasks.py new file mode 100644 index 0000000..60e7ecf --- /dev/null +++ b/project/users/tasks.py @@ -0,0 +1,15 @@ +""" +Many resources on the web recommend using celery.task. This might cause circular imports since you'll have to import the Celery instance. +We used shared_task to make our code reusable, which, again, requires current_app in create_celery instead of creating a new Celery instance. +Now, we can copy this file anywhere in the app and it will work as expected. +""" + +from celery import shared_task + + +@shared_task +def divide(x, y): + import time + + time.sleep(5) + return x / y diff --git a/requirements.in b/requirements.in index 5f6d60d..06f1a64 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,7 @@ +alembic==1.8.1 celery==5.2.7 fastapi==0.79.0 flower==1.2.0 redis==4.3.4 +SQLAlchemy==1.4.40 uvicorn[standard]==0.18.2 diff --git a/requirements.txt b/requirements.txt index 87215c1..43e11b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ # # pip-compile requirements.in # +alembic==1.8.1 + # via -r requirements.in amqp==5.1.1 # via kombu anyio==3.6.1 @@ -37,6 +39,8 @@ fastapi==0.79.0 # via -r requirements.in flower==1.2.0 # via -r requirements.in +greenlet==1.1.3 + # via sqlalchemy h11==0.13.0 # via uvicorn httptools==0.5.0 @@ -45,8 +49,16 @@ humanize==4.4.0 # via flower idna==3.4 # via anyio +importlib-metadata==4.12.0 + # via alembic +importlib-resources==5.9.0 + # via alembic kombu==5.2.4 # via celery +mako==1.2.2 + # via alembic +markupsafe==2.1.1 + # via mako packaging==21.3 # via redis prometheus-client==0.14.1 @@ -71,6 +83,10 @@ six==1.16.0 # via click-repl sniffio==1.3.0 # via anyio +sqlalchemy==1.4.40 + # via + # -r requirements.in + # alembic starlette==0.19.1 # via fastapi tornado==6.2 @@ -96,3 +112,7 @@ websockets==10.3 # via uvicorn wrapt==1.14.1 # via deprecated +zipp==3.8.1 + # via + # importlib-metadata + # importlib-resources diff --git a/tasks.py b/tasks.py index 46da6f7..1a4ae72 100644 --- a/tasks.py +++ b/tasks.py @@ -38,3 +38,30 @@ def stop_redis(c): """Stops the Redis integration environent.""" print("Stopping Redis") c.run("docker compose -f docker-compose-redis.yml down") + + +@task +def create_migration(c): + """Creates a sqlalchemy migration.""" + c.run("alembic revision --autogenerate") + + +@task +def apply_migration(c, revision="head"): + """Applies an Alembic migration.""" + c.run(f"alembic upgrade {revision}") + + +@task(help={"username": "Username of the application", "email": "User's email"}) +def add_user(username="", email=""): + "Adds a dummy user to the application Database" + if not username or email: + raise ValueError("You must provide a username and email") + from project.database import SessionLocal + from project.users.models import User + + session = SessionLocal() + user = User(username=username, email=email) + session.add(user) + session.commit() + session.close()