Cleanup CI tests

Turns check_migrations.py into a normal test case. Speeds up pipeline by
making html5validator use the artifacts from tests:buster:sqlite instead of
running the tests on its own.

.gitlab-ci.yml

@@ -47,19 +47,6 @@ db_migrations_updated:
   - FLASK_APP=uffd FLASK_ENV=testing flask db upgrade
   - FLASK_APP=uffd FLASK_ENV=testing flask db migrate 2>&1 | grep -q 'No changes in schema detected'
 
-test_db_migrations:sqlite:
-  stage: test
-  needs: []
-  script:
-  - python3 check_migrations.py sqlite
-
-test_db_migrations:mysql:
-  stage: test
-  needs: []
-  script:
-  - service mysql start
-  - python3 check_migrations.py mysql
-
 linter:buster:
   image: registry.git.cccv.de/uffd/docker-images/buster
   stage: test
@@ -88,12 +75,16 @@ linter:bullseye:
     reports:
       codequality: codeclimate.json
 
-unittests:buster:sqlite:
+tests:buster:sqlite:
   image: registry.git.cccv.de/uffd/docker-images/buster
   stage: test
   needs: []
   script:
-  - python3-coverage run --include 'uffd/*.py' -m pytest --junitxml=report.xml || touch failed
+  - rm -rf pages
+  - mkdir -p pages
+  - cp -r uffd/static pages/static
+  - DUMP_PAGES=pages python3-coverage run --include 'uffd/*.py' -m pytest --junitxml=report.xml || touch failed
+  - sed -i -e 's/href="\/static\//href=".\/static\//g' -e 's/src="\/static\//src=".\/static\//g' pages/*.html || true
   - python3-coverage report -m
   - python3-coverage html
   - python3-coverage xml
@@ -103,6 +94,7 @@ unittests:buster:sqlite:
     paths:
     - htmlcov/index.html
     - htmlcov
+    - pages
     expose_as: 'Coverage Report'
     reports:
       coverage_report:
@@ -111,7 +103,7 @@ unittests:buster:sqlite:
       junit: report.xml
   coverage: '/^TOTAL.*\s+(\d+\%)$/'
 
-unittests:buster:mysql:
+tests:buster:mysql:
   image: registry.git.cccv.de/uffd/docker-images/buster
   stage: test
   needs: []
@@ -123,7 +115,7 @@ unittests:buster:mysql:
     reports:
       junit: report.xml
 
-unittests:bullseye:sqlite:
+tests:bullseye:sqlite:
   image: registry.git.cccv.de/uffd/docker-images/bullseye
   stage: test
   needs: []
@@ -134,7 +126,7 @@ unittests:bullseye:sqlite:
     reports:
       junit: report.xml
 
-unittests:bullseye:mysql:
+tests:bullseye:mysql:
   image: registry.git.cccv.de/uffd/docker-images/bullseye
   stage: test
   needs: []
@@ -148,13 +140,9 @@ unittests:bullseye:mysql:
 
 html5validator:
   stage: test
-  needs: []
+  needs:
+  - job: tests:buster:sqlite
   script:
-  - rm -rf pages
-  - mkdir -p pages
-  - cp -r uffd/static pages/static
-  - DUMP_PAGES=pages python3 -m unittest discover tests/views
-  - sed -i -e 's/href="\/static\//href=".\/static\//g' -e 's/src="\/static\//src=".\/static\//g' pages/*.html
   - html5validator --root pages 2>&1 | tee html5validator.log
   artifacts:
     when: on_failure
@@ -176,18 +164,18 @@ trans_de:
 test:package:pip:buster:
   image: registry.git.cccv.de/uffd/docker-images/buster
   stage: test
+  needs:
+  - job: build:pip
   script:
   - pip3 install dist/*.tar.gz
-  dependencies:
-  - build:pip
 
 test:package:pip:bullseye:
   image: registry.git.cccv.de/uffd/docker-images/bullseye
   stage: test
+  needs:
+  - job: build:pip
   script:
   - pip3 install dist/*.tar.gz
-  dependencies:
-  - build:pip
 
 # Since we want to test if the package installs correctly on a fresh Debian
 # install (has correct dependencies, etc.), we don't use uffd/docker-images
@@ -195,6 +183,8 @@ test:package:pip:bullseye:
 test:package:apt:buster:
   image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/debian:buster
   stage: test
+  needs:
+  - job: build:apt
   before_script: []
   script:
   - apt -y update
@@ -204,12 +194,12 @@ test:package:apt:buster:
   - service nginx start || ( service nginx status; nginx -t; exit 1; )
   - uffd-admin routes
   - curl -Lv 127.0.0.1:5000
-  dependencies:
-  - build:apt
 
 test:package:apt:bullseye:
   image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/debian:bullseye
   stage: test
+  needs:
+  - job: build:apt
   before_script: []
   script:
   - apt -y update
@@ -219,8 +209,6 @@ test:package:apt:bullseye:
   - service nginx start || ( service nginx status; nginx -t; exit 1; )
   - uffd-admin routes
   - curl -Lv 127.0.0.1:5000
-  dependencies:
-  - build:apt
 
 .publish:
   stage: deploy

check_migrations.py → tests/migrations/test_fuzzy.py

-#!/usr/bin/python3
 import os
 import sys
-import logging
 import datetime
 
-import flask_migrate
-
-from uffd import create_app, db
+from uffd.database import db
 from uffd.models import (
 	User, UserEmail, Group,
 	RecoveryCodeMethod, TOTPMethod, WebauthnMethod,
@@ -19,37 +15,32 @@ from uffd.models import (
 	PasswordToken,
 )
 
-def run_test(dburi, revision):
-	config = {
-		'TESTING': True,
-		'DEBUG': True,
-		'SQLALCHEMY_DATABASE_URI': dburi,
-		'SECRET_KEY': 'DEBUGKEY',
-		'MAIL_SKIP_SEND': True,
-		'SELF_SIGNUP': True,
-		'ENABLE_INVITE': True,
-		'ENABLE_PASSWORDRESET': True,
-		'LDAP_SERVICE_MOCK': True,
-		'OAUTH2_CLIENTS': {
+from tests.utils import MigrationTestCase
+
+class TestFuzzy(MigrationTestCase):
+	def setUpApp(self):
+		self.app.config['LDAP_SERVICE_MOCK'] = True
+		self.app.config['OAUTH2_CLIENTS'] = {
 			'test': {
 				'service_name': 'test',
 				'client_secret': 'testsecret',
 				'redirect_uris': ['http://localhost:5004/oauthproxy/callback'],
 				'logout_urls': ['http://localhost:5004/oauthproxy/logout']
 			}
-		},
-		'API_CLIENTS_2': {
+		}
+		self.app.config['API_CLIENTS_2'] = {
 			'test': {
 				'service_name': 'test',
 				'client_secret': 'testsecret',
 				'scopes': ['checkpassword', 'getusers', 'getmails']
 			},
-		},
 		}
-	app = create_app(config)
-	with app.test_request_context():
-		flask_migrate.upgrade(revision='head')
-		# Add a few rows to all tables to make sure that the migrations work with data
+
+	# Runs every upgrade/downgrade script with data. To do this we first upgrade
+	# to head, create data, then downgrade, upgrade, downgrade for every revision.
+	def test_migrations_fuzzy(self):
+		self.upgrade('head')
+		# Users and groups were created by 878b25c4fae7_ldap_to_db because we set LDAP_SERVICE_MOCK to True
 		user = User.query.first()
 		group = Group.query.first()
 		db.session.add(RecoveryCodeMethod(user=user))
@@ -74,44 +65,8 @@ def run_test(dburi, revision):
 		db.session.add(OAuth2DeviceLoginInitiation(client=oauth2_client, confirmations=[DeviceLoginConfirmation(user=user)]))
 		db.session.add(PasswordToken(user=user))
 		db.session.commit()
-		flask_migrate.downgrade(revision=revision)
-		flask_migrate.upgrade(revision='head')
-
-if __name__ == '__main__':
-	if len(sys.argv) != 2 or sys.argv[1] not in ['sqlite', 'mysql']:
-		print('usage: check_migrations.py {sqlite|mysql}')
-		exit(1)
-	dbtype = sys.argv[1]
-	revs = [s.split('_', 1)[0] for s in os.listdir('uffd/migrations/versions') if '_' in s and s.endswith('.py')] + ['base']
-	logging.getLogger().setLevel(logging.INFO)
-	failures = 0
+		revs = [s.split('_', 1)[0] for s in os.listdir('uffd/migrations/versions') if '_' in s and s.endswith('.py')]
 		for rev in revs:
-		logging.info(f'Testing "upgrade to head, add objects, downgrade to {rev}, upgrade to head"')
-		# Cleanup/drop database
-		if dbtype == 'sqlite':
-			try:
-				os.remove('/tmp/uffd_check_migrations_db.sqlite3')
-			except FileNotFoundError:
-				pass
-			dburi = 'sqlite:////tmp/uffd_check_migrations_db.sqlite3'
-		elif dbtype == 'mysql':
-			import MySQLdb
-			conn = MySQLdb.connect(user='root', unix_socket='/var/run/mysqld/mysqld.sock')
-			cur = conn.cursor()
-			try:
-				cur.execute('DROP DATABASE uffd_tests')
-			except:
-				pass
-			cur.execute('CREATE DATABASE uffd_tests CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin')
-			conn.close()
-			dburi = 'mysql+mysqldb:///uffd_tests?unix_socket=/var/run/mysqld/mysqld.sock&charset=utf8mb4'
-		try:
-			run_test(dburi, rev)
-		except Exception as ex:
-			failures += 1
-			logging.error('Test failed', exc_info=ex)
-	if failures:
-		logging.info(f'{failures} tests failed')
-		exit(1)
-	logging.info('All tests succeeded')
-	exit(0)
+			self.downgrade('-1')
+			self.upgrade('+1')
+			self.downgrade('-1')