diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a3cd3bf2a33a79e3e5f3f4e0bbcc623bebeb07af..75c6e1ff6659e3ffdafe74b814666de236fe784e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -88,7 +88,7 @@ linter:bullseye:
reports:
codequality: codeclimate.json
-unittests:buster:
+unittests:buster:sqlite:
image: registry.git.cccv.de/uffd/docker-images/buster
stage: test
needs: []
@@ -111,7 +111,19 @@ unittests:buster:
junit: report.xml
coverage: '/^TOTAL.*\s+(\d+\%)$/'
-unittests:bullseye:
+unittests:buster:mysql:
+ image: registry.git.cccv.de/uffd/docker-images/buster
+ stage: test
+ needs: []
+ script:
+ - service mysql start
+ - TEST_WITH_MYSQL=1 python3 -m pytest --junitxml=report.xml
+ artifacts:
+ when: always
+ reports:
+ junit: report.xml
+
+unittests:bullseye:sqlite:
image: registry.git.cccv.de/uffd/docker-images/bullseye
stage: test
needs: []
@@ -122,6 +134,18 @@ unittests:bullseye:
reports:
junit: report.xml
+unittests:bullseye:mysql:
+ image: registry.git.cccv.de/uffd/docker-images/bullseye
+ stage: test
+ needs: []
+ script:
+ - service mariadb start
+ - TEST_WITH_MYSQL=1 python3 -m pytest --junitxml=report.xml
+ artifacts:
+ when: always
+ reports:
+ junit: report.xml
+
html5validator:
stage: test
needs: []
diff --git a/README.md b/README.md
index aecc542917badbd777a5d07cd71c9b90d0d525f3..83505f73f973e9567d237310ac7ea20c24871b0b 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ Please note that we refer to Debian packages here and **not** pip packages.
- python3-flask-babel
- python3-argon2
- python3-itsdangerous (also a dependency of python3-flask)
-- python3-mysqldb or python3-pymysql for MySQL/MariaDB support
+- python3-mysqldb or python3-pymysql for MariaDB support
Some of the dependencies (especially fido2) changed their API in recent versions, so make sure to install the versions from Debian Buster or Bullseye.
For development, you can also use virtualenv with the supplied `requirements.txt`.
@@ -69,10 +69,12 @@ deb https://packages.cccv.de/uffd bullseye main
Then download [cccv-archive-key.gpg](cccv-archive-key.gpg) and add it to the trusted repository keys in `/etc/apt/trusted.gpg.d/`.
Afterwards run `apt update && apt install uffd` to install the package.
-The Debian package uses uwsgi to run uffd and ships an `uffd-admin` to execute flask commands in the correct context.
+The Debian package uses uwsgi to run uffd and ships an `uffd-admin` script to execute flask commands in the correct context.
If you upgrade, make sure to run `flask db upgrade` after every update! The Debian package takes care of this by itself using uwsgi pre start hooks.
For an example uwsgi config, see our [uswgi.ini](uwsgi.ini). You might find our [nginx include file](nginx.include.conf) helpful to setup a web server in front of uwsgi.
+Uffd supports SQLite and MariaDB. To use MariaDB, create the database with the options `CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin` and make sure to add the `?charset=utf8mb4` parameter to `SQLALCHEMY_DATABASE_URI`.
+
## Python Coding Style Conventions
PEP 8 without double new lines, tabs instead of spaces and a max line length of 160 characters.
diff --git a/check_migrations.py b/check_migrations.py
index eb8d8bc046a9649377e786b125dd4434cab47ffa..4c7d0af0e785fdbe91c04673b30c873cef5ce95e 100755
--- a/check_migrations.py
+++ b/check_migrations.py
@@ -99,12 +99,12 @@ if __name__ == '__main__':
conn = MySQLdb.connect(user='root', unix_socket='/var/run/mysqld/mysqld.sock')
cur = conn.cursor()
try:
- cur.execute('DROP DATABASE uffd')
+ cur.execute('DROP DATABASE uffd_tests')
except:
pass
- cur.execute('CREATE DATABASE uffd')
+ cur.execute('CREATE DATABASE uffd_tests CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin')
conn.close()
- dburi = 'mysql+mysqldb:///uffd?unix_socket=/var/run/mysqld/mysqld.sock'
+ dburi = 'mysql+mysqldb:///uffd_tests?unix_socket=/var/run/mysqld/mysqld.sock&charset=utf8mb4'
try:
run_test(dburi, rev)
except Exception as ex:
diff --git a/tests/utils.py b/tests/utils.py
index 55a6164e19d48039238febada65329d06fecc1fa..832b203cd40064a31324645ca0db57c1be99178a 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -40,6 +40,17 @@ class AppTestCase(unittest.TestCase):
except FileNotFoundError:
pass
config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/uffd-migration-test-db.sqlite3'
+ if os.environ.get('TEST_WITH_MYSQL'):
+ import MySQLdb
+ conn = MySQLdb.connect(user='root', unix_socket='/var/run/mysqld/mysqld.sock')
+ cur = conn.cursor()
+ try:
+ cur.execute('DROP DATABASE uffd_tests')
+ except:
+ pass
+ cur.execute('CREATE DATABASE uffd_tests CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin')
+ conn.close()
+ config['SQLALCHEMY_DATABASE_URI'] = 'mysql+mysqldb:///uffd_tests?unix_socket=/var/run/mysqld/mysqld.sock&charset=utf8mb4'
self.app = create_app(config)
self.setUpApp()
diff --git a/uffd/database.py b/uffd/database.py
index 83d5446d958898b1429f0119e66cfba91642b44a..38ed8951d1683f33f352d8ea68d00928eed3d527 100644
--- a/uffd/database.py
+++ b/uffd/database.py
@@ -33,6 +33,20 @@ def enable_sqlite_foreign_key_support(dbapi_connection, connection_record):
def customize_db_engine(engine):
if engine.name == 'sqlite':
event.listen(engine, 'connect', enable_sqlite_foreign_key_support)
+ elif engine.name in ('mysql', 'mariadb'):
+ @event.listens_for(engine, 'connect')
+ def receive_connect(dbapi_connection, connection_record): # pylint: disable=unused-argument
+ cursor = dbapi_connection.cursor()
+ cursor.execute('SHOW VARIABLES LIKE "character_set_connection"')
+ character_set_connection = cursor.fetchone()[1]
+ if character_set_connection != 'utf8mb4':
+ raise Exception(f'Unsupported connection charset "{character_set_connection}". Make sure to add "?charset=utf8mb4" to SQLALCHEMY_DATABASE_URI!')
+ cursor.execute('SHOW VARIABLES LIKE "collation_database"')
+ collation_database = cursor.fetchone()[1]
+ if collation_database != 'utf8mb4_nopad_bin':
+ raise Exception(f'Unsupported database collation "{collation_database}". Create the database with "CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin"!')
+ cursor.execute('SET NAMES utf8mb4 COLLATE utf8mb4_nopad_bin')
+ cursor.close()
class SQLAlchemyJSON(JSONEncoder):
def default(self, o):
diff --git a/uffd/migrations/env.py b/uffd/migrations/env.py
index 23663ff2f54e6c4425953537976b175246c8a9e6..fe8568acea6d574f17351d5aa612baedb1402ccd 100755
--- a/uffd/migrations/env.py
+++ b/uffd/migrations/env.py
@@ -3,6 +3,7 @@ from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
import logging
+import click
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
@@ -74,6 +75,14 @@ def run_migrations_online():
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args)
+ if engine.name in ('mysql', 'mariadb'):
+ character_set_connection = connection.execute('SHOW VARIABLES LIKE "character_set_connection"').fetchone()[1]
+ if character_set_connection != 'utf8mb4':
+ raise click.ClickException(f'Unsupported connection charset "{character_set_connection}". Make sure to add "?charset=utf8mb4" to SQLALCHEMY_DATABASE_URI!')
+ collation_database = connection.execute('SHOW VARIABLES LIKE "collation_database"').fetchone()[1]
+ if collation_database != 'utf8mb4_nopad_bin':
+ raise click.ClickException(f'Unsupported database collation "{collation_database}". Create the database with "CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin"!')
+ connection.execute('SET NAMES utf8mb4 COLLATE utf8mb4_nopad_bin')
try:
with context.begin_transaction():