Force charset/collation on MariaDB and enable CI tests

Uffd now requires that MariaDB databases have utf8mb4 charset and
utf8mb4_nopad_bin collation. The collation was chosen for consistency with
SQLite's BINARY collation.

.gitlab-ci.yml

@@ -88,7 +88,7 @@ linter:bullseye:
     reports:
       codequality: codeclimate.json
 
-unittests:buster:
+unittests:buster:sqlite:
   image: registry.git.cccv.de/uffd/docker-images/buster
   stage: test
   needs: []
@@ -111,7 +111,19 @@ unittests:buster:
       junit: report.xml
   coverage: '/^TOTAL.*\s+(\d+\%)$/'
 
-unittests:bullseye:
+unittests:buster:mysql:
+  image: registry.git.cccv.de/uffd/docker-images/buster
+  stage: test
+  needs: []
+  script:
+  - service mysql start
+  - TEST_WITH_MYSQL=1 python3 -m pytest --junitxml=report.xml
+  artifacts:
+    when: always
+    reports:
+      junit: report.xml
+
+unittests:bullseye:sqlite:
   image: registry.git.cccv.de/uffd/docker-images/bullseye
   stage: test
   needs: []
@@ -122,6 +134,18 @@ unittests:bullseye:
     reports:
       junit: report.xml
 
+unittests:bullseye:mysql:
+  image: registry.git.cccv.de/uffd/docker-images/bullseye
+  stage: test
+  needs: []
+  script:
+  - service mariadb start
+  - TEST_WITH_MYSQL=1 python3 -m pytest --junitxml=report.xml
+  artifacts:
+    when: always
+    reports:
+      junit: report.xml
+
 html5validator:
   stage: test
   needs: []

README.md

@@ -19,7 +19,7 @@ Please note that we refer to Debian packages here and **not** pip packages.
 - python3-flask-babel
 - python3-argon2
 - python3-itsdangerous (also a dependency of python3-flask)
-- python3-mysqldb or python3-pymysql for MySQL/MariaDB support
+- python3-mysqldb or python3-pymysql for MariaDB support
 
 Some of the dependencies (especially fido2) changed their API in recent versions, so make sure to install the versions from Debian Buster or Bullseye.
 For development, you can also use virtualenv with the supplied `requirements.txt`.
@@ -69,10 +69,12 @@ deb https://packages.cccv.de/uffd bullseye main
 Then download [cccv-archive-key.gpg](cccv-archive-key.gpg) and add it to the trusted repository keys in `/etc/apt/trusted.gpg.d/`.
 Afterwards run `apt update && apt install uffd` to install the package.
 
-The Debian package uses uwsgi to run uffd and ships an `uffd-admin` to execute flask commands in the correct context.
+The Debian package uses uwsgi to run uffd and ships an `uffd-admin` script to execute flask commands in the correct context.
 If you upgrade, make sure to run `flask db upgrade` after every update! The Debian package takes care of this by itself using uwsgi pre start hooks.
 For an example uwsgi config, see our [uswgi.ini](uwsgi.ini). You might find our [nginx include file](nginx.include.conf) helpful to setup a web server in front of uwsgi.
 
+Uffd supports SQLite and MariaDB. To use MariaDB, create the database with the options `CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin` and make sure to add the `?charset=utf8mb4` parameter to `SQLALCHEMY_DATABASE_URI`.
+
 ## Python Coding Style Conventions
 
 PEP 8 without double new lines, tabs instead of spaces and a max line length of 160 characters.

check_migrations.py

@@ -99,12 +99,12 @@ if __name__ == '__main__':
 			conn = MySQLdb.connect(user='root', unix_socket='/var/run/mysqld/mysqld.sock')
 			cur = conn.cursor()
 			try:
-				cur.execute('DROP DATABASE uffd')
+				cur.execute('DROP DATABASE uffd_tests')
 			except:
 				pass
-			cur.execute('CREATE DATABASE uffd')
+			cur.execute('CREATE DATABASE uffd_tests CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin')
 			conn.close()
-			dburi = 'mysql+mysqldb:///uffd?unix_socket=/var/run/mysqld/mysqld.sock'
+			dburi = 'mysql+mysqldb:///uffd_tests?unix_socket=/var/run/mysqld/mysqld.sock&charset=utf8mb4'
 		try:
 			run_test(dburi, rev)
 		except Exception as ex:

tests/utils.py

@@ -40,6 +40,17 @@ class AppTestCase(unittest.TestCase):
 			except FileNotFoundError:
 				pass
 			config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/uffd-migration-test-db.sqlite3'
+		if os.environ.get('TEST_WITH_MYSQL'):
+			import MySQLdb
+			conn = MySQLdb.connect(user='root', unix_socket='/var/run/mysqld/mysqld.sock')
+			cur = conn.cursor()
+			try:
+				cur.execute('DROP DATABASE uffd_tests')
+			except:
+				pass
+			cur.execute('CREATE DATABASE uffd_tests CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin')
+			conn.close()
+			config['SQLALCHEMY_DATABASE_URI'] = 'mysql+mysqldb:///uffd_tests?unix_socket=/var/run/mysqld/mysqld.sock&charset=utf8mb4'
 		self.app = create_app(config)
 		self.setUpApp()
 

uffd/database.py

@@ -33,6 +33,20 @@ def enable_sqlite_foreign_key_support(dbapi_connection, connection_record):
 def customize_db_engine(engine):
 	if engine.name == 'sqlite':
 		event.listen(engine, 'connect', enable_sqlite_foreign_key_support)
+	elif engine.name in ('mysql', 'mariadb'):
+		@event.listens_for(engine, 'connect')
+		def receive_connect(dbapi_connection, connection_record): # pylint: disable=unused-argument
+			cursor = dbapi_connection.cursor()
+			cursor.execute('SHOW VARIABLES LIKE "character_set_connection"')
+			character_set_connection = cursor.fetchone()[1]
+			if character_set_connection != 'utf8mb4':
+				raise Exception(f'Unsupported connection charset "{character_set_connection}". Make sure to add "?charset=utf8mb4" to SQLALCHEMY_DATABASE_URI!')
+			cursor.execute('SHOW VARIABLES LIKE "collation_database"')
+			collation_database  = cursor.fetchone()[1]
+			if collation_database != 'utf8mb4_nopad_bin':
+				raise Exception(f'Unsupported database collation "{collation_database}". Create the database with "CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin"!')
+			cursor.execute('SET NAMES utf8mb4 COLLATE utf8mb4_nopad_bin')
+			cursor.close()
 
 class SQLAlchemyJSON(JSONEncoder):
 	def default(self, o):

uffd/migrations/env.py

@@ -3,6 +3,7 @@ from alembic import context
 from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig
 import logging
+import click
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -74,6 +75,14 @@ def run_migrations_online():
                       target_metadata=target_metadata,
                       process_revision_directives=process_revision_directives,
                       **current_app.extensions['migrate'].configure_args)
+    if engine.name in ('mysql', 'mariadb'):
+        character_set_connection = connection.execute('SHOW VARIABLES LIKE "character_set_connection"').fetchone()[1]
+        if character_set_connection != 'utf8mb4':
+            raise click.ClickException(f'Unsupported connection charset "{character_set_connection}". Make sure to add "?charset=utf8mb4" to SQLALCHEMY_DATABASE_URI!')
+        collation_database = connection.execute('SHOW VARIABLES LIKE "collation_database"').fetchone()[1]
+        if collation_database != 'utf8mb4_nopad_bin':
+            raise click.ClickException(f'Unsupported database collation "{collation_database}". Create the database with "CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin"!')
+        connection.execute('SET NAMES utf8mb4 COLLATE utf8mb4_nopad_bin')
 
     try:
         with context.begin_transaction():