From 91ba4a6fc5477cdc9965fe3b076f7bcc5f75453c Mon Sep 17 00:00:00 2001
From: Julian Rother <julian@cccv.de>
Date: Fri, 4 Nov 2022 20:30:36 +0100
Subject: [PATCH] Force charset/collation on MariaDB and enable CI tests
Uffd now requires that MariaDB databases have utf8mb4 charset and
utf8mb4_nopad_bin collation. The collation was chosen for consistency with
SQLite's BINARY collation.
---
.gitlab-ci.yml | 28 ++++++++++++++++++++++++++--
README.md | 6 ++++--
check_migrations.py | 6 +++---
tests/utils.py | 11 +++++++++++
uffd/database.py | 14 ++++++++++++++
uffd/migrations/env.py | 9 +++++++++
6 files changed, 67 insertions(+), 7 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a3cd3bf2..75c6e1ff 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -88,7 +88,7 @@ linter:bullseye:
reports:
codequality: codeclimate.json
-unittests:buster:
+unittests:buster:sqlite:
image: registry.git.cccv.de/uffd/docker-images/buster
stage: test
needs: []
@@ -111,7 +111,19 @@ unittests:buster:
junit: report.xml
coverage: '/^TOTAL.*\s+(\d+\%)$/'
-unittests:bullseye:
+unittests:buster:mysql:
+ image: registry.git.cccv.de/uffd/docker-images/buster
+ stage: test
+ needs: []
+ script:
+ - service mysql start
+ - TEST_WITH_MYSQL=1 python3 -m pytest --junitxml=report.xml
+ artifacts:
+ when: always
+ reports:
+ junit: report.xml
+
+unittests:bullseye:sqlite:
image: registry.git.cccv.de/uffd/docker-images/bullseye
stage: test
needs: []
@@ -122,6 +134,18 @@ unittests:bullseye:
reports:
junit: report.xml
+unittests:bullseye:mysql:
+ image: registry.git.cccv.de/uffd/docker-images/bullseye
+ stage: test
+ needs: []
+ script:
+ - service mariadb start
+ - TEST_WITH_MYSQL=1 python3 -m pytest --junitxml=report.xml
+ artifacts:
+ when: always
+ reports:
+ junit: report.xml
+
html5validator:
stage: test
needs: []
diff --git a/README.md b/README.md
index aecc5429..83505f73 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ Please note that we refer to Debian packages here and **not** pip packages.
- python3-flask-babel
- python3-argon2
- python3-itsdangerous (also a dependency of python3-flask)
-- python3-mysqldb or python3-pymysql for MySQL/MariaDB support
+- python3-mysqldb or python3-pymysql for MariaDB support
Some of the dependencies (especially fido2) changed their API in recent versions, so make sure to install the versions from Debian Buster or Bullseye.
For development, you can also use virtualenv with the supplied `requirements.txt`.
@@ -69,10 +69,12 @@ deb https://packages.cccv.de/uffd bullseye main
Then download [cccv-archive-key.gpg](cccv-archive-key.gpg) and add it to the trusted repository keys in `/etc/apt/trusted.gpg.d/`.
Afterwards run `apt update && apt install uffd` to install the package.
-The Debian package uses uwsgi to run uffd and ships an `uffd-admin` to execute flask commands in the correct context.
+The Debian package uses uwsgi to run uffd and ships an `uffd-admin` script to execute flask commands in the correct context.
If you upgrade, make sure to run `flask db upgrade` after every update! The Debian package takes care of this by itself using uwsgi pre start hooks.
For an example uwsgi config, see our [uswgi.ini](uwsgi.ini). You might find our [nginx include file](nginx.include.conf) helpful to setup a web server in front of uwsgi.
+Uffd supports SQLite and MariaDB. To use MariaDB, create the database with the options `CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin` and make sure to add the `?charset=utf8mb4` parameter to `SQLALCHEMY_DATABASE_URI`.
+
## Python Coding Style Conventions
PEP 8 without double new lines, tabs instead of spaces and a max line length of 160 characters.
diff --git a/check_migrations.py b/check_migrations.py
index eb8d8bc0..4c7d0af0 100755
--- a/check_migrations.py
+++ b/check_migrations.py
@@ -99,12 +99,12 @@ if __name__ == '__main__':
conn = MySQLdb.connect(user='root', unix_socket='/var/run/mysqld/mysqld.sock')
cur = conn.cursor()
try:
- cur.execute('DROP DATABASE uffd')
+ cur.execute('DROP DATABASE uffd_tests')
except:
pass
- cur.execute('CREATE DATABASE uffd')
+ cur.execute('CREATE DATABASE uffd_tests CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin')
conn.close()
- dburi = 'mysql+mysqldb:///uffd?unix_socket=/var/run/mysqld/mysqld.sock'
+ dburi = 'mysql+mysqldb:///uffd_tests?unix_socket=/var/run/mysqld/mysqld.sock&charset=utf8mb4'
try:
run_test(dburi, rev)
except Exception as ex:
diff --git a/tests/utils.py b/tests/utils.py
index 55a6164e..832b203c 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -40,6 +40,17 @@ class AppTestCase(unittest.TestCase):
except FileNotFoundError:
pass
config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/uffd-migration-test-db.sqlite3'
+ if os.environ.get('TEST_WITH_MYSQL'):
+ import MySQLdb
+ conn = MySQLdb.connect(user='root', unix_socket='/var/run/mysqld/mysqld.sock')
+ cur = conn.cursor()
+ try:
+ cur.execute('DROP DATABASE uffd_tests')
+ except:
+ pass
+ cur.execute('CREATE DATABASE uffd_tests CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin')
+ conn.close()
+ config['SQLALCHEMY_DATABASE_URI'] = 'mysql+mysqldb:///uffd_tests?unix_socket=/var/run/mysqld/mysqld.sock&charset=utf8mb4'
self.app = create_app(config)
self.setUpApp()
diff --git a/uffd/database.py b/uffd/database.py
index 83d5446d..38ed8951 100644
--- a/uffd/database.py
+++ b/uffd/database.py
@@ -33,6 +33,20 @@ def enable_sqlite_foreign_key_support(dbapi_connection, connection_record):
def customize_db_engine(engine):
if engine.name == 'sqlite':
event.listen(engine, 'connect', enable_sqlite_foreign_key_support)
+ elif engine.name in ('mysql', 'mariadb'):
+ @event.listens_for(engine, 'connect')
+ def receive_connect(dbapi_connection, connection_record): # pylint: disable=unused-argument
+ cursor = dbapi_connection.cursor()
+ cursor.execute('SHOW VARIABLES LIKE "character_set_connection"')
+ character_set_connection = cursor.fetchone()[1]
+ if character_set_connection != 'utf8mb4':
+ raise Exception(f'Unsupported connection charset "{character_set_connection}". Make sure to add "?charset=utf8mb4" to SQLALCHEMY_DATABASE_URI!')
+ cursor.execute('SHOW VARIABLES LIKE "collation_database"')
+ collation_database = cursor.fetchone()[1]
+ if collation_database != 'utf8mb4_nopad_bin':
+ raise Exception(f'Unsupported database collation "{collation_database}". Create the database with "CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin"!')
+ cursor.execute('SET NAMES utf8mb4 COLLATE utf8mb4_nopad_bin')
+ cursor.close()
class SQLAlchemyJSON(JSONEncoder):
def default(self, o):
diff --git a/uffd/migrations/env.py b/uffd/migrations/env.py
index 23663ff2..fe8568ac 100755
--- a/uffd/migrations/env.py
+++ b/uffd/migrations/env.py
@@ -3,6 +3,7 @@ from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
import logging
+import click
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
@@ -74,6 +75,14 @@ def run_migrations_online():
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args)
+ if engine.name in ('mysql', 'mariadb'):
+ character_set_connection = connection.execute('SHOW VARIABLES LIKE "character_set_connection"').fetchone()[1]
+ if character_set_connection != 'utf8mb4':
+ raise click.ClickException(f'Unsupported connection charset "{character_set_connection}". Make sure to add "?charset=utf8mb4" to SQLALCHEMY_DATABASE_URI!')
+ collation_database = connection.execute('SHOW VARIABLES LIKE "collation_database"').fetchone()[1]
+ if collation_database != 'utf8mb4_nopad_bin':
+ raise click.ClickException(f'Unsupported database collation "{collation_database}". Create the database with "CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin"!')
+ connection.execute('SET NAMES utf8mb4 COLLATE utf8mb4_nopad_bin')
try:
with context.begin_transaction():
--
GitLab