diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e89826bd33fbe0079f2fc6ed4ee3a492db90fad0..64dcc0c5ff3fdac4ca8fce1a5d1634eeac69c044 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,6 +46,17 @@ db_migrations_updated: - FLASK_APP=uffd flask db upgrade - FLASK_APP=uffd flask db migrate 2>&1 | grep -q 'No changes in schema detected' +test_db_migrations:sqlite: + stage: test + script: + - python3 check_migrations.py sqlite + +test_db_migrations:mysql: + stage: test + script: + - service mysql start + - python3 check_migrations.py mysql + linter:buster: image: registry.git.cccv.de/uffd/docker-images/buster stage: test diff --git a/check_migrations.py b/check_migrations.py new file mode 100755 index 0000000000000000000000000000000000000000..bd15129985d3a2f2a8d32f4f5d91040a45daa44b --- /dev/null +++ b/check_migrations.py @@ -0,0 +1,97 @@ +#!/usr/bin/python3 +import os +import sys +import logging +import datetime + +import flask_migrate + +from uffd import create_app, db +from uffd.user.models import User, Group +from uffd.mfa.models import RecoveryCodeMethod, TOTPMethod, WebauthnMethod +from uffd.role.models import Role, RoleGroup +from uffd.signup.models import Signup +from uffd.invite.models import Invite, InviteGrant, InviteSignup +from uffd.session.models import DeviceLoginConfirmation +from uffd.oauth2.models import OAuth2Grant, OAuth2Token, OAuth2DeviceLoginInitiation +from uffd.selfservice.models import PasswordToken, MailToken + +def run_test(dburi, revision): + config = { + 'TESTING': True, + 'DEBUG': True, + 'SQLALCHEMY_DATABASE_URI': dburi, + 'SECRET_KEY': 'DEBUGKEY', + 'LDAP_SERVICE_MOCK': True, + 'MAIL_SKIP_SEND': True, + 'SELF_SIGNUP': True, + 'ENABLE_INVITE': True, + 'ENABLE_PASSWORDRESET': True + } + app = create_app(config) + with app.test_request_context(): + flask_migrate.upgrade(revision='head') + # Add a few rows to all tables to make sure that the migrations work with data + user = User.query.first() + group = Group.query.first() + db.session.add(RecoveryCodeMethod(user=user)) + db.session.add(TOTPMethod(user=user, name='mytotp')) + db.session.add(WebauthnMethod(user=user, name='mywebauthn', cred=b'')) + role = Role(name='role', groups={group: RoleGroup(group=group)}) + db.session.add(role) + role.members.add(user) + db.session.add(Role(name='base', included_roles=[role], locked=True, is_default=True, moderator_group_dn=group.dn, groups={group: RoleGroup(group=group)})) + db.session.add(Signup(loginname='newuser', displayname='New User', mail='newuser@example.com', password='newpassword')) + db.session.add(Signup(loginname='testuser', displayname='Testuser', mail='testuser@example.com', password='testpassword', user_dn=user.dn)) + invite = Invite(valid_until=datetime.datetime.now(), roles=[role]) + db.session.add(invite) + invite.signups.append(InviteSignup(loginname='newuser', displayname='New User', mail='newuser@example.com', password='newpassword')) + invite.grants.append(InviteGrant(user_dn=user.dn)) + db.session.add(Invite(creator_dn=user.dn, valid_until=datetime.datetime.now())) + db.session.add(OAuth2Grant(user_dn=user.dn, client_id='testclient', code='testcode', redirect_uri='http://example.com/callback', expires=datetime.datetime.now())) + db.session.add(OAuth2Token(user_dn=user.dn, client_id='testclient', token_type='Bearer', access_token='testcode', refresh_token='testcode', expires=datetime.datetime.now())) + db.session.add(OAuth2DeviceLoginInitiation(oauth2_client_id='testclient', confirmations=[DeviceLoginConfirmation(user_dn=user.dn)])) + db.session.add(PasswordToken(loginname='testuser')) + db.session.add(MailToken(loginname='testuser', newmail='test@example.com')) + db.session.commit() + flask_migrate.downgrade(revision=revision) + flask_migrate.upgrade(revision='head') + +if __name__ == '__main__': + if len(sys.argv) != 2 or sys.argv[1] not in ['sqlite', 'mysql']: + print('usage: check_migrations.py {sqlite|mysql}') + exit(1) + dbtype = sys.argv[1] + revs = [s.split('_', 1)[0] for s in os.listdir('uffd/migrations/versions') if '_' in s and s.endswith('.py')] + ['base'] + logging.getLogger().setLevel(logging.INFO) + failures = 0 + for rev in revs: + logging.info(f'Testing "upgrade to head, add objects, downgrade to {rev}, upgrade to head"') + # Cleanup/drop database + if dbtype == 'sqlite': + try: + os.remove('/tmp/uffd_check_migrations_db.sqlite3') + except FileNotFoundError: + pass + dburi = 'sqlite:////tmp/uffd_check_migrations_db.sqlite3' + elif dbtype == 'mysql': + import MySQLdb + conn = MySQLdb.connect(user='root', unix_socket='/var/run/mysqld/mysqld.sock') + cur = conn.cursor() + try: + cur.execute('DROP DATABASE uffd') + except: + pass + cur.execute('CREATE DATABASE uffd') + conn.close() + dburi = 'mysql+mysqldb:///uffd?unix_socket=/var/run/mysqld/mysqld.sock' + try: + run_test(dburi, rev) + except Exception as ex: + failures += 1 + logging.error('Test failed', exc_info=ex) + if failures: + logging.info(f'{failures} tests failed') + exit(1) + logging.info('All tests succeeded') + exit(0) diff --git a/uffd/migrations/README b/uffd/migrations/README deleted file mode 100755 index 98e4f9c44effe479ed38c66ba922e7bcc672916f..0000000000000000000000000000000000000000 --- a/uffd/migrations/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/uffd/migrations/README.md b/uffd/migrations/README.md new file mode 100755 index 0000000000000000000000000000000000000000..3d7ebf9b7cfa9586244c13097f628684af1acd4a --- /dev/null +++ b/uffd/migrations/README.md @@ -0,0 +1,25 @@ +Database Migrations +=================== + +While we use Alembic in a single-database configuration, the migration scripts +are compatible with both SQLite and MySQL/MariaDB. + +Compatability with SQLite almost always requires `batch_alter_table` operations +to modify existing tables. These recreate the tables, copy the data and finally +replace the old with the newly creaed ones. Alembic is configured to +auto-generate those operations, but in most cases the generated code fails to +fully reflect all details of the original schema. This way some contraints +(i.e. `CHECK` contstrains on Enums) are lost. Define the full table and pass it +with `copy_from` to `batch_alter_table` to prevent this. + +Compatability with MySQL requires special care when changing primary keys and +when dealing with foreign keys. It often helps to temporarily remove foreign +key constraints concerning the table that is subject to change. When adding an +autoincrement id column as the new primary key of a table, recreate the table +with `batch_alter_table`. + +The `check_migrations.py` script verifies that upgrading and downgrading works +with both databases. While it is far from perfect, it catches many common +errors. It runs automatically as part of the CI pipeline. Make sure to update +the script when adding new tables and when making significant changes to +existing tables. diff --git a/uffd/migrations/versions/11ecc8f1ac3b_mysql_compat_fixes.py b/uffd/migrations/versions/11ecc8f1ac3b_mysql_compat_fixes.py new file mode 100644 index 0000000000000000000000000000000000000000..2376dcb25cad18ef1d4d4d9e5e5a87c52de01b87 --- /dev/null +++ b/uffd/migrations/versions/11ecc8f1ac3b_mysql_compat_fixes.py @@ -0,0 +1,67 @@ +"""MySQL compat fixes + +Revision ID: 11ecc8f1ac3b +Revises: bf71799b7b9e +Create Date: 2021-09-13 04:15:07.479295 + +""" +from alembic import op +import sqlalchemy as sa + +revision = '11ecc8f1ac3b' +down_revision = 'bf71799b7b9e' +branch_labels = None +depends_on = None + +def upgrade(): + meta = sa.MetaData(bind=op.get_bind()) + table = sa.Table('device_login_confirmation', meta, + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('initiation_id', sa.Integer(), nullable=False), + sa.Column('user_dn', sa.String(length=128), nullable=False), + sa.Column('code0', sa.String(length=32), nullable=False), + sa.Column('code1', sa.String(length=32), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_device_login_confirmation')), + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + table = sa.Table('invite_signup', meta, + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('invite_id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_invite_signup')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + meta = sa.MetaData(bind=op.get_bind()) + # Previously "fk_device_login_confirmation_initiation_id_" was named + # "fk_device_login_confirmation_initiation_id_device_login_initiation" + # but this was too long for MySQL. + table = sa.Table('device_login_confirmation', meta, + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('initiation_id', sa.Integer(), nullable=False), + sa.Column('user_dn', sa.String(length=128), nullable=False), + sa.Column('code0', sa.String(length=32), nullable=False), + sa.Column('code1', sa.String(length=32), nullable=False), + sa.ForeignKeyConstraint(['initiation_id'], ['device_login_initiation.id'], name='fk_device_login_confirmation_initiation_id_'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_device_login_confirmation')), + sa.UniqueConstraint('initiation_id', 'code0', name='uq_device_login_confirmation_initiation_id_code0'), + sa.UniqueConstraint('initiation_id', 'code1', name='uq_device_login_confirmation_initiation_id_code1'), + sa.UniqueConstraint('user_dn', name=op.f('uq_device_login_confirmation_user_dn')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + + # Previously "fk_invite_signup_id_signup" was named + # "fk_invite_signup_signup_id_signup" by mistake. + table = sa.Table('invite_signup', meta, + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('invite_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['id'], ['signup.id'], name=op.f('fk_invite_signup_id_signup')), + sa.ForeignKeyConstraint(['invite_id'], ['invite.id'], name=op.f('fk_invite_signup_invite_id_invite')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_invite_signup')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + +def downgrade(): + pass diff --git a/uffd/migrations/versions/54b2413586fd_invite_pk_change.py b/uffd/migrations/versions/54b2413586fd_invite_pk_change.py index 5d927cf0a16513fd0ba5468304a7986331492b1d..a59e2ca69336fd8027a967e9436e250605b4a6b8 100644 --- a/uffd/migrations/versions/54b2413586fd_invite_pk_change.py +++ b/uffd/migrations/versions/54b2413586fd_invite_pk_change.py @@ -44,10 +44,17 @@ def upgrade(): sa.Column('disabled', sa.Boolean(name=op.f('ck_invite_disabled')), nullable=False), sa.PrimaryKeyConstraint('token', name=op.f('pk_invite')) ) - with op.batch_alter_table('invite', copy_from=table) as batch_op: + with op.batch_alter_table('invite_grant', schema=None) as batch_op: + batch_op.drop_constraint('fk_invite_grant_invite_token_invite', type_='foreignkey') + with op.batch_alter_table('invite_roles', schema=None) as batch_op: + batch_op.drop_constraint('fk_invite_roles_invite_token_invite', type_='foreignkey') + with op.batch_alter_table('invite_signup', schema=None) as batch_op: + batch_op.drop_constraint('fk_invite_signup_invite_token_invite', type_='foreignkey') + with op.batch_alter_table('invite', copy_from=table, recreate='always') as batch_op: batch_op.drop_constraint(batch_op.f('pk_invite'), type_='primary') - batch_op.add_column(sa.Column('id', sa.Integer(), autoincrement=True, nullable=False)) + batch_op.add_column(sa.Column('id', sa.Integer(), nullable=True)) batch_op.create_primary_key(batch_op.f('pk_invite'), ['id']) + batch_op.alter_column('id', autoincrement=True, nullable=False, existing_type=sa.Integer()) batch_op.create_unique_constraint(batch_op.f('uq_invite_token'), ['token']) with op.batch_alter_table('invite_grant', schema=None) as batch_op: batch_op.add_column(sa.Column('invite_id', sa.Integer(), nullable=True)) @@ -62,27 +69,27 @@ def upgrade(): with op.batch_alter_table('invite_grant', schema=None) as batch_op: batch_op.alter_column('invite_id', existing_type=sa.INTEGER(), nullable=False) - batch_op.drop_constraint('fk_invite_grant_invite_token_invite', type_='foreignkey') batch_op.create_foreign_key(batch_op.f('fk_invite_grant_invite_id_invite'), 'invite', ['invite_id'], ['id']) batch_op.drop_column('invite_token') with op.batch_alter_table('invite_roles', schema=None) as batch_op: batch_op.drop_constraint(batch_op.f('pk_invite_roles'), type_='primary') batch_op.create_primary_key(batch_op.f('pk_invite_roles'), ['invite_id', 'role_id']) - batch_op.drop_constraint('fk_invite_roles_invite_token_invite', type_='foreignkey') batch_op.create_foreign_key(batch_op.f('fk_invite_roles_invite_id_invite'), 'invite', ['invite_id'], ['id']) batch_op.drop_column('invite_token') with op.batch_alter_table('invite_signup', schema=None) as batch_op: batch_op.alter_column('invite_id', existing_type=sa.INTEGER(), nullable=False) - batch_op.drop_constraint('fk_invite_signup_invite_token_invite', type_='foreignkey') batch_op.create_foreign_key(batch_op.f('fk_invite_signup_invite_id_invite'), 'invite', ['invite_id'], ['id']) batch_op.drop_column('invite_token') def downgrade(): with op.batch_alter_table('invite_signup', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('fk_invite_signup_invite_id_invite'), type_='foreignkey') batch_op.add_column(sa.Column('invite_token', sa.VARCHAR(length=128), nullable=True)) with op.batch_alter_table('invite_roles', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('fk_invite_roles_invite_id_invite'), type_='foreignkey') batch_op.add_column(sa.Column('invite_token', sa.VARCHAR(length=128), nullable=True)) with op.batch_alter_table('invite_grant', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('fk_invite_grant_invite_id_invite'), type_='foreignkey') batch_op.add_column(sa.Column('invite_token', sa.VARCHAR(length=128), nullable=True)) op.execute(invite_grant.update().values(invite_token=sa.select([invite.c.token]).where(invite.c.id==invite_grant.c.invite_id).as_scalar())) @@ -91,20 +98,14 @@ def downgrade(): with op.batch_alter_table('invite_signup', schema=None) as batch_op: batch_op.alter_column('invite_token', existing_type=sa.VARCHAR(length=128), nullable=False) - batch_op.drop_constraint(batch_op.f('fk_invite_signup_invite_id_invite'), type_='foreignkey') - batch_op.create_foreign_key('fk_invite_signup_invite_token_invite', 'invite', ['invite_token'], ['token']) batch_op.drop_column('invite_id') with op.batch_alter_table('invite_roles', schema=None) as batch_op: batch_op.alter_column('invite_token', existing_type=sa.VARCHAR(length=128), nullable=False) batch_op.drop_constraint(batch_op.f('pk_invite_roles'), type_='primary') batch_op.create_primary_key(batch_op.f('pk_invite_roles'), ['invite_token', 'role_id']) - batch_op.drop_constraint(batch_op.f('fk_invite_roles_invite_id_invite'), type_='foreignkey') - batch_op.create_foreign_key('fk_invite_roles_invite_token_invite', 'invite', ['invite_token'], ['token']) batch_op.drop_column('invite_id') with op.batch_alter_table('invite_grant', schema=None) as batch_op: batch_op.alter_column('invite_token', existing_type=sa.VARCHAR(length=128), nullable=False) - batch_op.drop_constraint(batch_op.f('fk_invite_grant_invite_id_invite'), type_='foreignkey') - batch_op.create_foreign_key('fk_invite_grant_invite_token_invite', 'invite', ['invite_token'], ['token']) batch_op.drop_column('invite_id') # CHECK constraints get lost when reflecting from the actual table @@ -121,8 +122,15 @@ def downgrade(): sa.PrimaryKeyConstraint('id', name=op.f('pk_invite')), sa.UniqueConstraint('token', name=op.f('uq_invite_token')) ) - with op.batch_alter_table('invite', copy_from=table) as batch_op: + with op.batch_alter_table('invite', copy_from=table, recreate='always') as batch_op: batch_op.drop_constraint(batch_op.f('uq_invite_token'), type_='unique') + batch_op.alter_column('id', autoincrement=False, existing_type=sa.Integer()) batch_op.drop_constraint(batch_op.f('pk_invite'), type_='primary') batch_op.drop_column('id') batch_op.create_primary_key(batch_op.f('pk_invite'), ['token']) + with op.batch_alter_table('invite_signup', schema=None) as batch_op: + batch_op.create_foreign_key('fk_invite_signup_invite_token_invite', 'invite', ['invite_token'], ['token']) + with op.batch_alter_table('invite_roles', schema=None) as batch_op: + batch_op.create_foreign_key('fk_invite_roles_invite_token_invite', 'invite', ['invite_token'], ['token']) + with op.batch_alter_table('invite_grant', schema=None) as batch_op: + batch_op.create_foreign_key('fk_invite_grant_invite_token_invite', 'invite', ['invite_token'], ['token']) diff --git a/uffd/migrations/versions/a8c6b6e91c28_device_login.py b/uffd/migrations/versions/a8c6b6e91c28_device_login.py index 8d594b4afc7903cc09abcf1c30cabe69264f3ca2..efdbc3018b5980a588c5bf490f18f77ebeb82cde 100644 --- a/uffd/migrations/versions/a8c6b6e91c28_device_login.py +++ b/uffd/migrations/versions/a8c6b6e91c28_device_login.py @@ -33,7 +33,8 @@ def upgrade(): sa.Column('user_dn', sa.String(length=128), nullable=False), sa.Column('code0', sa.String(length=32), nullable=False), sa.Column('code1', sa.String(length=32), nullable=False), - sa.ForeignKeyConstraint(['initiation_id'], ['device_login_initiation.id'], name=op.f('fk_device_login_confirmation_initiation_id_device_login_initiation')), + # name would be fk_device_login_confirmation_initiation_id_device_login_initiation, but that is too long for MySQL + sa.ForeignKeyConstraint(['initiation_id'], ['device_login_initiation.id'], name=op.f('fk_device_login_confirmation_initiation_id_')), sa.PrimaryKeyConstraint('id', name=op.f('pk_device_login_confirmation')), sa.UniqueConstraint('initiation_id', 'code0', name=op.f('uq_device_login_confirmation_initiation_id_code0')), sa.UniqueConstraint('initiation_id', 'code1', name=op.f('uq_device_login_confirmation_initiation_id_code1')), diff --git a/uffd/migrations/versions/bad6fc529510_added_rolegroup_requires_mfa_and_cleanup.py b/uffd/migrations/versions/bad6fc529510_added_rolegroup_requires_mfa_and_cleanup.py index d18822db070c65f2daa9fd9aaa95cdb54300b9bf..c97861859c4bd5d9c8f642f813672555c7d4df85 100644 --- a/uffd/migrations/versions/bad6fc529510_added_rolegroup_requires_mfa_and_cleanup.py +++ b/uffd/migrations/versions/bad6fc529510_added_rolegroup_requires_mfa_and_cleanup.py @@ -17,23 +17,52 @@ depends_on = None def upgrade(): meta = sa.MetaData(bind=op.get_bind()) table = sa.Table('role-group', meta, - sa.Column('role_id', sa.Integer(), nullable=False), - sa.Column('dn', sa.String(128), nullable=False), + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('dn', sa.String(length=128), nullable=True), + sa.Column('role_id', sa.Integer(), nullable=True), sa.ForeignKeyConstraint(['role_id'], ['role.id'], name=op.f('fk_role-group_role_id_role')), - sa.PrimaryKeyConstraint('role_id', 'dn', name=op.f('pk_role-group')) + sa.PrimaryKeyConstraint('id', name=op.f('pk_role-group')), + sa.UniqueConstraint('dn', 'role_id', name=op.f('uq_role-group_dn')) ) - with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: - batch_op.alter_column('dn', new_column_name='group_dn', nullable=False) + with op.batch_alter_table(table.name, copy_from=table) as batch_op: + batch_op.alter_column('id', autoincrement=False, existing_type=sa.Integer()) + batch_op.drop_constraint(batch_op.f('pk_role-group'), type_='primary') + batch_op.drop_constraint(batch_op.f('uq_role-group_dn'), type_='unique') + batch_op.drop_column('id') + batch_op.alter_column('dn', new_column_name='group_dn', nullable=False, existing_type=sa.String(128)) + batch_op.alter_column('role_id', nullable=False, existing_type=sa.Integer()) batch_op.add_column(sa.Column('requires_mfa', sa.Boolean(name=op.f('ck_role-group_requires_mfa')), nullable=False, default=False)) + batch_op.create_primary_key(batch_op.f('pk_role-group'), ['role_id', 'group_dn']) def downgrade(): meta = sa.MetaData(bind=op.get_bind()) table = sa.Table('role-group', meta, sa.Column('role_id', sa.Integer(), nullable=False), sa.Column('group_dn', sa.String(128), nullable=False), - sa.ForeignKeyConstraint(['role_id'], ['role.id'], name=op.f('fk_role-group_role_id_role')) + sa.Column('requires_mfa', sa.Boolean(name=op.f('ck_role-group_requires_mfa')), nullable=False, default=False), + sa.ForeignKeyConstraint(['role_id'], ['role.id'], name=op.f('fk_role-group_role_id_role')), + sa.PrimaryKeyConstraint('role_id', 'group_dn', name=op.f('pk_role-group')) ) with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: - batch_op.add_column(sa.Column('id', sa.INTEGER(), nullable=False, autoincrement=True, primary_key=True)) - batch_op.alter_column('group_dn', new_column_name='dn', nullable=True) - batch_op.add_column(sa.Column('requires_mfa', sa.Boolean(name=op.f('ck_role-group_requires_mfa')), nullable=False, default=False)) + # For some reason MySQL does not allow us to drop the primary key if the foreignkey on role_id exists + batch_op.drop_constraint(batch_op.f('fk_role-group_role_id_role'), type_='foreignkey') + batch_op.drop_constraint(batch_op.f('pk_role-group'), type_='primary') + batch_op.drop_column('requires_mfa') + batch_op.alter_column('role_id', nullable=True, existing_type=sa.Integer()) + batch_op.alter_column('group_dn', new_column_name='dn', nullable=True, existing_type=sa.String(128)) + batch_op.add_column(sa.Column('id', sa.Integer(), nullable=True)) + batch_op.create_primary_key(batch_op.f('pk_role-group'), ['id']) + batch_op.alter_column('id', autoincrement=True, nullable=False, existing_type=sa.Integer()) + # For some reason MySQL ignores this statement + #batch_op.create_unique_constraint(op.f('uq_role-group_dn'), ['dn', 'role_id']) + meta = sa.MetaData(bind=op.get_bind()) + table = sa.Table('role-group', meta, + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('dn', sa.String(length=128), nullable=True), + sa.Column('role_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['role_id'], ['role.id'], name=op.f('fk_role-group_role_id_role')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_role-group')), + sa.UniqueConstraint('dn', 'role_id', name=op.f('uq_role-group_dn')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass diff --git a/uffd/migrations/versions/bf71799b7b9e_add_id_to_signup_table.py b/uffd/migrations/versions/bf71799b7b9e_add_id_to_signup_table.py index 68aa43bfa37d9bf56e151a3253b9fa8ba40a2dbc..70252cec576792571c34343bb84c99810b2e3067 100644 --- a/uffd/migrations/versions/bf71799b7b9e_add_id_to_signup_table.py +++ b/uffd/migrations/versions/bf71799b7b9e_add_id_to_signup_table.py @@ -14,6 +14,18 @@ branch_labels = None depends_on = None def upgrade(): + meta = sa.MetaData(bind=op.get_bind()) + invite_signup = sa.Table('invite_signup', meta, + sa.Column('token', sa.String(length=128), nullable=False), + sa.Column('invite_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['invite_id'], ['invite.id'], name=op.f('fk_invite_signup_invite_id_invite')), + sa.ForeignKeyConstraint(['token'], ['signup.token'], name=op.f('fk_invite_signup_token_signup')), + sa.PrimaryKeyConstraint('token', name=op.f('pk_invite_signup')) + ) + with op.batch_alter_table(invite_signup.name, copy_from=invite_signup) as batch_op: + batch_op.add_column(sa.Column('id', sa.Integer(), nullable=True)) + batch_op.drop_constraint('fk_invite_signup_token_signup', 'foreignkey') + meta = sa.MetaData(bind=op.get_bind()) signup = sa.Table('signup', meta, sa.Column('token', sa.String(length=128), nullable=False), @@ -27,20 +39,10 @@ def upgrade(): sa.PrimaryKeyConstraint('token', name=op.f('pk_signup')) ) with op.batch_alter_table(signup.name, copy_from=signup, recreate='always') as batch_op: - batch_op.add_column(sa.Column('id', sa.Integer(), autoincrement=True, nullable=False)) batch_op.drop_constraint('pk_signup', 'primary') - batch_op.create_primary_key('pk_signup', ['id']) - - meta = sa.MetaData(bind=op.get_bind()) - invite_signup = sa.Table('invite_signup', meta, - sa.Column('token', sa.String(length=128), nullable=False), - sa.Column('invite_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['invite_id'], ['invite.id'], name=op.f('fk_invite_signup_invite_id_invite')), - sa.ForeignKeyConstraint(['token'], ['signup.token'], name=op.f('fk_invite_signup_token_signup')), - sa.PrimaryKeyConstraint('token', name=op.f('pk_invite_signup')) - ) - with op.batch_alter_table(invite_signup.name, copy_from=invite_signup, recreate='always') as batch_op: batch_op.add_column(sa.Column('id', sa.Integer(), nullable=True)) + batch_op.create_primary_key('pk_signup', ['id']) + batch_op.alter_column('id', autoincrement=True, nullable=False, existing_type=sa.Integer()) meta = sa.MetaData(bind=op.get_bind()) signup = sa.Table('signup', meta, @@ -60,13 +62,12 @@ def upgrade(): sa.Column('token', sa.String(length=128), nullable=False), sa.Column('invite_id', sa.Integer(), nullable=False), sa.ForeignKeyConstraint(['invite_id'], ['invite.id'], name=op.f('fk_invite_signup_invite_id_invite')), - sa.ForeignKeyConstraint(['token'], ['signup.token'], name=op.f('fk_invite_signup_token_signup')), sa.PrimaryKeyConstraint('token', name=op.f('pk_invite_signup')) ) op.execute(invite_signup.update().values(id=sa.select([signup.c.id]).where(signup.c.token==invite_signup.c.token).limit(1).as_scalar())) - with op.batch_alter_table(invite_signup.name, copy_from=invite_signup, recreate='always') as batch_op: - batch_op.drop_constraint('fk_invite_signup_token_signup', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_invite_signup_signup_id_signup'), 'signup', ['id'], ['id']) + with op.batch_alter_table(invite_signup.name, copy_from=invite_signup) as batch_op: + batch_op.alter_column('id', nullable=False, existing_type=sa.Integer()) + batch_op.create_foreign_key(batch_op.f('fk_invite_signup_id_signup'), 'signup', ['id'], ['id']) batch_op.drop_constraint('pk_invite_signup', 'primary') batch_op.drop_column('token') batch_op.create_primary_key('pk_invite_signup', ['id']) @@ -80,8 +81,27 @@ def downgrade(): sa.ForeignKeyConstraint(['id'], ['signup.id'], name=op.f('fk_invite_signup_id_signup')), sa.PrimaryKeyConstraint('id', name=op.f('pk_invite_signup')) ) - with op.batch_alter_table(invite_signup.name, copy_from=invite_signup, recreate='always') as batch_op: + with op.batch_alter_table(invite_signup.name, copy_from=invite_signup) as batch_op: batch_op.add_column(sa.Column('token', sa.VARCHAR(length=128), nullable=True)) + batch_op.drop_constraint('fk_invite_signup_id_signup', type_='foreignkey') + + meta = sa.MetaData(bind=op.get_bind()) + signup = sa.Table('signup', meta, + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('token', sa.String(length=128), nullable=False), + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('loginname', sa.Text(), nullable=True), + sa.Column('displayname', sa.Text(), nullable=True), + sa.Column('mail', sa.Text(), nullable=True), + sa.Column('pwhash', sa.Text(), nullable=True), + sa.Column('user_dn', sa.String(length=128), nullable=True), + sa.Column('type', sa.String(length=50), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_signup')) + ) + with op.batch_alter_table(signup.name, copy_from=signup) as batch_op: + batch_op.alter_column('id', autoincrement=False, existing_type=sa.Integer()) + batch_op.drop_constraint('pk_signup', 'primary') + batch_op.create_primary_key('pk_signup', ['token']) meta = sa.MetaData(bind=op.get_bind()) signup = sa.Table('signup', meta, @@ -105,9 +125,8 @@ def downgrade(): sa.PrimaryKeyConstraint('id', name=op.f('pk_invite_signup')) ) op.execute(invite_signup.update().values(token=sa.select([signup.c.token]).where(signup.c.id==invite_signup.c.id).limit(1).as_scalar())) - with op.batch_alter_table(invite_signup.name, copy_from=invite_signup, recreate='always') as batch_op: - batch_op.drop_constraint('fk_invite_signup_id_signup', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_invite_signup_signup_token_signup'), 'signup', ['token'], ['token']) + with op.batch_alter_table(invite_signup.name, copy_from=invite_signup) as batch_op: + batch_op.create_foreign_key(batch_op.f('fk_invite_signup_token_signup'), 'signup', ['token'], ['token']) batch_op.drop_constraint('pk_invite_signup', 'primary') batch_op.drop_column('id') batch_op.create_primary_key('pk_invite_signup', ['token']) @@ -125,7 +144,5 @@ def downgrade(): sa.Column('type', sa.String(length=50), nullable=True), sa.PrimaryKeyConstraint('id', name=op.f('pk_signup')) ) - with op.batch_alter_table(signup.name, copy_from=signup, recreate='always') as batch_op: - batch_op.drop_constraint('pk_signup', 'primary') - batch_op.create_primary_key('pk_signup', ['token']) + with op.batch_alter_table(signup.name, copy_from=signup) as batch_op: batch_op.drop_column('id') diff --git a/uffd/migrations/versions/cbca20cf64d9_constraint_name_fixes.py b/uffd/migrations/versions/cbca20cf64d9_constraint_name_fixes.py index cc237505b4e7e2fa71c6a488fb94de7bd56ca33c..541e1dbf1f3d0a2edc980cb34aa83e909d4a0e16 100644 --- a/uffd/migrations/versions/cbca20cf64d9_constraint_name_fixes.py +++ b/uffd/migrations/versions/cbca20cf64d9_constraint_name_fixes.py @@ -19,10 +19,63 @@ def upgrade(): # The only difference is that all contraints are named according to the newly # defined naming conventions. This enables changing constraints in future # migrations. - meta = sa.MetaData(bind=op.get_bind()) + # # We call batch_alter_table without any operations to have it recreate all # tables with the column/constraint definitions from "table" and populate it # with the data from the original table. + + # First recreate tables that have (unnamed) foreign keys without any foreign keys + meta = sa.MetaData(bind=op.get_bind()) + table = sa.Table('invite_grant', meta, + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('invite_token', sa.String(length=128), nullable=False), + sa.Column('user_dn', sa.String(length=128), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_invite_grant')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + table = sa.Table('invite_roles', meta, + sa.Column('invite_token', sa.String(length=128), nullable=False), + sa.Column('role_id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('invite_token', 'role_id', name=op.f('pk_invite_roles')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + table = sa.Table('invite_signup', meta, + sa.Column('token', sa.String(length=128), nullable=False), + sa.Column('invite_token', sa.String(length=128), nullable=False), + sa.PrimaryKeyConstraint('token', name=op.f('pk_invite_signup')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + table = sa.Table('role-group', meta, + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('dn', sa.String(length=128), nullable=True), + sa.Column('role_id', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_role-group')), + sa.UniqueConstraint('dn', 'role_id', name=op.f('uq_role-group_dn')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + table = sa.Table('role-inclusion', meta, + sa.Column('role_id', sa.Integer(), nullable=False), + sa.Column('included_role_id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('role_id', 'included_role_id', name=op.f('pk_role-inclusion')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + table = sa.Table('role-user', meta, + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('dn', sa.String(length=128), nullable=True), + sa.Column('role_id', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_role-user')), + sa.UniqueConstraint('dn', 'role_id', name=op.f('uq_role-user_dn')) + ) + with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + pass + + # Then recreate all tables with properly named constraints and readd foreign key constraints + meta = sa.MetaData(bind=op.get_bind()) table = sa.Table('invite', meta, sa.Column('token', sa.String(length=128), nullable=False), sa.Column('created', sa.DateTime(), nullable=False), @@ -182,5 +235,6 @@ def upgrade(): pass def downgrade(): - # upgrade only adds names to all constraints, no need to undo anything - pass + # upgrade only adds names to all constraints, no need to undo much + with op.batch_alter_table('oauth2grant', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_oauth2grant_code'), ['code'], unique=False) diff --git a/uffd/migrations/versions/e9a67175e179_add_id_to_selfservice_tokens.py b/uffd/migrations/versions/e9a67175e179_add_id_to_selfservice_tokens.py index 934df36d31107d0b6d5c643f703235129c2647cb..51d7bffd5973fe3ff5b863b1c732ca40e3fcc94f 100644 --- a/uffd/migrations/versions/e9a67175e179_add_id_to_selfservice_tokens.py +++ b/uffd/migrations/versions/e9a67175e179_add_id_to_selfservice_tokens.py @@ -23,9 +23,10 @@ def upgrade(): sa.PrimaryKeyConstraint('token', name=op.f('pk_mailToken')) ) with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: - batch_op.add_column(sa.Column('id', sa.Integer(), autoincrement=True, nullable=False)) batch_op.drop_constraint('pk_mailToken', 'primary') + batch_op.add_column(sa.Column('id', sa.Integer(), nullable=True)) batch_op.create_primary_key('pk_mailToken', ['id']) + batch_op.alter_column('id', autoincrement=True, nullable=False, existing_type=sa.Integer()) table = sa.Table('passwordToken', meta, sa.Column('token', sa.String(length=128), nullable=False), sa.Column('created', sa.DateTime(), nullable=True), @@ -33,9 +34,10 @@ def upgrade(): sa.PrimaryKeyConstraint('token', name=op.f('pk_passwordToken')) ) with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: - batch_op.add_column(sa.Column('id', sa.Integer(), autoincrement=True, nullable=False)) batch_op.drop_constraint('pk_passwordToken', 'primary') + batch_op.add_column(sa.Column('id', sa.Integer(), nullable=True)) batch_op.create_primary_key('pk_passwordToken', ['id']) + batch_op.alter_column('id', autoincrement=True, nullable=False, existing_type=sa.Integer()) def downgrade(): meta = sa.MetaData(bind=op.get_bind()) @@ -47,7 +49,8 @@ def downgrade(): sa.Column('newmail', sa.String(length=255), nullable=True), sa.PrimaryKeyConstraint('token', name=op.f('pk_mailToken')) ) - with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: + with op.batch_alter_table(table.name, copy_from=table) as batch_op: + batch_op.alter_column('id', autoincrement=False, existing_type=sa.Integer()) batch_op.drop_constraint('pk_mailToken', 'primary') batch_op.create_primary_key('pk_mailToken', ['token']) batch_op.drop_column('id') @@ -58,8 +61,8 @@ def downgrade(): sa.Column('loginname', sa.String(length=32), nullable=True), sa.PrimaryKeyConstraint('token', name=op.f('pk_passwordToken')) ) - with op.batch_alter_table(table.name, copy_from=table, recreate='always') as batch_op: - batch_op.add_column(sa.Column('id', sa.Integer(), autoincrement=True, nullable=False)) + with op.batch_alter_table(table.name, copy_from=table) as batch_op: + batch_op.alter_column('id', autoincrement=False, existing_type=sa.Integer()) batch_op.drop_constraint('pk_passwordToken', 'primary') batch_op.create_primary_key('pk_passwordToken', ['token']) batch_op.drop_column('id') diff --git a/uffd/session/models.py b/uffd/session/models.py index ab243e0fcdbac2bc6dd55e85371a93c06fdc857e..70d38fb9947ae4122611f0d08dccc754bd9d86c7 100644 --- a/uffd/session/models.py +++ b/uffd/session/models.py @@ -114,7 +114,8 @@ class DeviceLoginConfirmation(db.Model): __tablename__ = 'device_login_confirmation' id = Column(Integer(), primary_key=True, autoincrement=True) - initiation_id = Column(Integer(), ForeignKey('device_login_initiation.id'), nullable=False) + initiation_id = Column(Integer(), ForeignKey('device_login_initiation.id', + name='fk_device_login_confirmation_initiation_id_'), nullable=False) initiation = relationship('DeviceLoginInitiation', back_populates='confirmations') user_dn = Column(String(128), nullable=False, unique=True) user = DBRelationship('user_dn', User)