diff --git a/README.md b/README.md index 540f057..c48a5ca 100644 --- a/README.md +++ b/README.md @@ -297,4 +297,17 @@ git push -f - PRs can contain multiple commits, they do not need to be squashed together before merging as long as each commit is atomic. Our repo is configured to only allow squash commits to `main` so the entire PR will appear as 1 commit on `main`, but the individual commits are preserved when viewing the PR. ## Secrets -Secrets are stored in the Environment Variable [file](https://www.notion.so/uwblueprintexecs/Environment-Variables-11910f3fb1dc80e4bc67d35c3d65d073?pvs=4) within the LLSC notion. \ No newline at end of file +Secrets are stored in the Environment Variable [file](https://www.notion.so/uwblueprintexecs/Environment-Variables-11910f3fb1dc80e4bc67d35c3d65d073?pvs=4) within the LLSC notion. + +## Migrations (mirrors [backend README](./backend/README.md)) + +We use Alembic for database schema migrations. We mainly use migration scripts to keep track of the incremental and in theory revertible changes that have occurred on the database. But, we don't need to rely on this to build the datebase itself, as `Base.metadata.create_all(bind=engine)` achieves that based on the current models. To create a new migration, run the following command after adding or editing models in `backend/app/models.py`: +```bash +cd backend +pdm run alembic revision --autogenerate -m "" +``` + +To apply the migration, run the following command: +```bash +pdm run alembic upgrade head +``` diff --git a/backend/README.md b/backend/README.md index 7c3e011..41ef9c6 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1 +1,75 @@ # llsc-backend + +## Setup (mirrors [base README](../README.md#setup)) +- Install pdm (this is a global installation, so location doesn't matter) +On macOS: +```bash +brew install pdm +``` +Otherwise, feel free to follow install instructions [here](https://pdm-project.org/latest/#installation) + +You will then need to go into each directory individually to install dependencies. + +FastAPI backend +```bash +cd backend +pdm install +``` + +To run the backend server locally (recommended for development), run the following command: +```bash +cd backend +pdm run dev +``` + +To check if the database has been started up, type the following: +```bash + docker ps | grep llsc_db +``` +This checks the list of docker containers and searchs for the container name `llsc_db` + + +## Formatting and Linting (mirrors [formatting in base README](../README.md#formatting-and-linting)) + +### Ruff + +We use Ruff for code linting and formatting in the backend. To check for and automatically fix linting issues: + +```bash +cd backend +pdm run ruff check --fix . +``` + +To format the code: +```bash +cd backend +pdm run ruff format . +``` + + +## Adding a new model +When adding a new model, make sure to add it to `app/models/__init__.py` so that the migration script can pick it up when autogenerating the new migration. + +In `app/models/__init__.py`, add the new model like so: +```python +from .Base import Base +... +from . import + +__all__ = ["Base", ... , ""] +``` +Then run the steps found in the [Migrations](#migrations) section to create a new migration. + + +## Migrations + +We use Alembic for database schema migrations. We mainly use migration scripts to keep track of the incremental and in theory revertible changes that have occurred on the database. To create a new migration, run the following command after adding or editing models in `backend/app/models.py`: +```bash +cd backend +pdm run alembic revision --autogenerate -m "" +``` + +To apply the migration, run the following command: +```bash +pdm run alembic upgrade head +``` diff --git a/backend/alembic.ini b/backend/alembic.ini index 18f8896..877c7d0 100644 --- a/backend/alembic.ini +++ b/backend/alembic.ini @@ -61,7 +61,8 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne # are written from script.py.mako # output_encoding = utf-8 -sqlalchemy.url = driver://user:pass@localhost/dbname +# Updated in env.py using the POSTGRES_DATABASE_URL environment variable +# sqlalchemy.url = [post_write_hooks] @@ -76,10 +77,10 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # black.options = -l 79 REVISION_SCRIPT_FILENAME # lint with attempts to fix using "ruff" - use the exec runner, execute a binary -# hooks = ruff -# ruff.type = exec -# ruff.executable = %(here)s/.venv/bin/ruff -# ruff.options = --fix REVISION_SCRIPT_FILENAME +hooks = ruff +ruff.type = exec +ruff.executable = %(here)s/.venv/bin/ruff +ruff.options = check --fix REVISION_SCRIPT_FILENAME # Logging configuration [loggers] diff --git a/backend/app/models/Base.py b/backend/app/models/Base.py new file mode 100644 index 0000000..d1da733 --- /dev/null +++ b/backend/app/models/Base.py @@ -0,0 +1,4 @@ +from sqlalchemy.orm import registry + +mapper_registry = registry() +Base = mapper_registry.generate_base() diff --git a/backend/app/models/Role.py b/backend/app/models/Role.py new file mode 100644 index 0000000..6d950d6 --- /dev/null +++ b/backend/app/models/Role.py @@ -0,0 +1,9 @@ +from sqlalchemy import Column, Integer, String + +from .Base import Base + + +class Role(Base): + __tablename__ = "roles" + id = Column(Integer, primary_key=True) + name = Column(String(80), nullable=False) diff --git a/backend/app/models/User.py b/backend/app/models/User.py new file mode 100644 index 0000000..bfe2ef7 --- /dev/null +++ b/backend/app/models/User.py @@ -0,0 +1,19 @@ +import uuid + +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship + +from .Base import Base + + +class User(Base): + __tablename__ = "users" + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + first_name = Column(String(80), nullable=False) + last_name = Column(String(80), nullable=False) + email = Column(String(120), unique=True, nullable=False) + role_id = Column(Integer, ForeignKey("roles.id"), nullable=False) + auth_id = Column(String, nullable=False) + + role = relationship("Role") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 4e49bd0..e51f027 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -1,18 +1,17 @@ -from sqlalchemy import SQLAlchemy +from alembic import command +from alembic.config import Config -db = SQLAlchemy() +# Make sure all models are here to reflect all current models +# when autogenerating new migration +from .Base import Base +from .Role import Role +from .User import User +# Used to avoid import errors for the models +__all__ = ["Base", "User", "Role"] -def init_app(app): - app.app_context().push() - db.init_app(app) - erase_db_and_sync = app.config["TESTING"] - - if erase_db_and_sync: - # drop tables - db.reflect() - db.drop_all() - - # recreate tables - db.create_all() +def run_migrations(): + alembic_cfg = Config("alembic.ini") + # Emulates `alembic upgrade head` to migrate up to latest revision + command.upgrade(alembic_cfg, "head") diff --git a/backend/app/server.py b/backend/app/server.py index 22eb334..5c43deb 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -1,10 +1,28 @@ +import logging +from contextlib import asynccontextmanager from typing import Union from dotenv import load_dotenv from fastapi import FastAPI +from . import models + load_dotenv() -app = FastAPI() + +log = logging.getLogger("uvicorn") + + +@asynccontextmanager +async def lifespan(_: FastAPI): + log.info("Starting up...") + models.run_migrations() + yield + log.info("Shutting down...") + + +# Source: https://stackoverflow.com/questions/77170361/ +# running-alembic-migrations-on-fastapi-startup +app = FastAPI(lifespan=lifespan) @app.get("/") diff --git a/backend/migrations/env.py b/backend/migrations/env.py index 9dd6c6c..cb165d2 100644 --- a/backend/migrations/env.py +++ b/backend/migrations/env.py @@ -1,8 +1,14 @@ +import os from logging.config import fileConfig from alembic import context +from dotenv import load_dotenv from sqlalchemy import engine_from_config, pool +from app.models import Base + +load_dotenv() + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -16,12 +22,14 @@ # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -target_metadata = None + +target_metadata = Base.metadata # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") # ... etc. +config.set_main_option("sqlalchemy.url", os.environ["POSTGRES_DATABASE_URL"]) def run_migrations_offline() -> None: @@ -55,8 +63,9 @@ def run_migrations_online() -> None: and associate a connection with the context. """ + alembic_config = config.get_section(config.config_ini_section, {}) connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), + alembic_config, prefix="sqlalchemy.", poolclass=pool.NullPool, ) diff --git a/backend/migrations/versions/4ba3479cb8df_create_user_table_and_roles.py b/backend/migrations/versions/4ba3479cb8df_create_user_table_and_roles.py new file mode 100644 index 0000000..45262f1 --- /dev/null +++ b/backend/migrations/versions/4ba3479cb8df_create_user_table_and_roles.py @@ -0,0 +1,50 @@ +"""create user table and roles + +Revision ID: 4ba3479cb8df +Revises: +Create Date: 2024-10-03 00:41:13.800838 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "4ba3479cb8df" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "roles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=80), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "users", + sa.Column("id", sa.String(), nullable=False), + sa.Column("first_name", sa.String(length=80), nullable=False), + sa.Column("last_name", sa.String(length=80), nullable=False), + sa.Column("email", sa.String(length=120), nullable=False), + sa.Column("role_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["role_id"], + ["roles.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("email"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("users") + op.drop_table("roles") + # ### end Alembic commands ### diff --git a/backend/migrations/versions/59bb2488a76b_insert_roles.py b/backend/migrations/versions/59bb2488a76b_insert_roles.py new file mode 100644 index 0000000..b740449 --- /dev/null +++ b/backend/migrations/versions/59bb2488a76b_insert_roles.py @@ -0,0 +1,37 @@ +"""insert roles + +Revision ID: 59bb2488a76b +Revises: 4ba3479cb8df +Create Date: 2024-10-16 16:55:42.324525 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "59bb2488a76b" +down_revision: Union[str, None] = "4ba3479cb8df" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.bulk_insert( + sa.table( + "roles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=80), nullable=False), + ), + [ + {"id": 1, "name": "participant"}, + {"id": 2, "name": "volunteer"}, + {"id": 3, "name": "admin"}, + ], + ) + + +def downgrade() -> None: + op.execute("DELETE FROM roles WHERE id IN (1, 2, 3)") diff --git a/backend/migrations/versions/79de0b981dd8_add_auth_id_to_user_model.py b/backend/migrations/versions/79de0b981dd8_add_auth_id_to_user_model.py new file mode 100644 index 0000000..6e3f787 --- /dev/null +++ b/backend/migrations/versions/79de0b981dd8_add_auth_id_to_user_model.py @@ -0,0 +1,30 @@ +"""Add auth_id to User model + +Revision ID: 79de0b981dd8 +Revises: 59bb2488a76b +Create Date: 2024-10-16 17:06:45.820859 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "79de0b981dd8" +down_revision: Union[str, None] = "59bb2488a76b" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("users", sa.Column("auth_id", sa.String(), nullable=False)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("users", "auth_id") + # ### end Alembic commands ### diff --git a/backend/migrations/versions/c9bc2b4d1036_update_users_id_to_uuid_with_default.py b/backend/migrations/versions/c9bc2b4d1036_update_users_id_to_uuid_with_default.py new file mode 100644 index 0000000..2c8cc4b --- /dev/null +++ b/backend/migrations/versions/c9bc2b4d1036_update_users_id_to_uuid_with_default.py @@ -0,0 +1,46 @@ +"""Update users id to UUID with default + +Revision ID: c9bc2b4d1036 +Revises: 79de0b981dd8 +Create Date: 2024-10-16 17:13:53.820521 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c9bc2b4d1036" +down_revision: Union[str, None] = "79de0b981dd8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "users", + "id", + existing_type=sa.VARCHAR(), + type_=sa.UUID(), + postgresql_using="id::uuid", + server_default=sa.text("gen_random_uuid()"), + existing_nullable=False, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "users", + "id", + existing_type=sa.UUID(), + type_=sa.VARCHAR(), + postgresql_using="id::text", + server_default=None, + existing_nullable=False, + ) + # ### end Alembic commands ### diff --git a/backend/pdm.lock b/backend/pdm.lock index 8060413..c461173 100644 --- a/backend/pdm.lock +++ b/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:2c6832080dcc206f03ea5f1d2a28462446baa6af5ec7721ed9b6d06b0809e1f4" +content_hash = "sha256:f51a6480b69b20a64cd23c57bce7d60a6fbef2c57d2f1a26dc2fb8363061c5dd" [[metadata.targets]] requires_python = "==3.12.*" @@ -831,6 +831,18 @@ files = [ {file = "protobuf-5.28.2.tar.gz", hash = "sha256:59379674ff119717404f7454647913787034f03fe7049cbef1d74a97bb4593f0"}, ] +[[package]] +name = "psycopg2" +version = "2.9.9" +requires_python = ">=3.7" +summary = "psycopg2 - Python-PostgreSQL Database Adapter" +groups = ["default"] +files = [ + {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, + {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, + {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, +] + [[package]] name = "pyasn1" version = "0.6.1" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1ae0fb8..7d1627e 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "firebase-admin>=6.5.0", "pytest>=8.3.3", "inflection>=0.5.1", + "psycopg2>=2.9.9", ] requires-python = "==3.12.*" readme = "README.md" @@ -26,6 +27,8 @@ distribution = false [tool.pdm.scripts] dev = "fastapi dev app/server.py" +revision = "alembic revision --autogenerate" +upgrade = "alembic upgrade head" [tool.ruff] target-version = "py312"