diff --git a/README.md b/README.md index 7cb76413b045e0810e59b8f57ff2d54346b9a643..aecc542917badbd777a5d07cd71c9b90d0d525f3 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Please note that we refer to Debian packages here and **not** pip packages. - python3-flask-migrate - python3-qrcode - python3-fido2 (version 0.5.0 or 0.9.1, optional) +- python3-prometheus-client (optional, needed for metrics) - python3-oauthlib - python3-flask-babel - python3-argon2 @@ -111,6 +112,24 @@ The only OAuth2 scope supported is `profile`. The userinfo endpoint returns json `id` is the numeric (Unix) user id, `name` the display name and `nickname` the loginname of the user. +## Metrics + +Uffd can export metrics in a prometheus compatible way. It needs python3-prometheus-client for this feature to work. +Metrics can be accessed via `/metrics` and `/api/v1/metrics_prometheus`. +Those endpoints are protected via api credentials. Add prometheus in the uffd UI as a service and create an +api client with the `metrics` permission. Then you can access the metrics like that: + +``` +$ curl localhost:5000/api/v1/metrics_prometheus --user api-user:api-password +# HELP python_info Python platform information +# TYPE python_info gauge +python_info{implementation="CPython",major="3",minor="9",patchlevel="2",version="3.9.2"} 1.0 +# HELP uffd_version_info Various version infos +# TYPE uffd_version_info gauge +uffd_version_info{version="local"} 1.0 +[..] +``` + ## Translation The web frontend is initially written in English and translated in the following Languages: diff --git a/debian/control b/debian/control index ae3744ceb9a28e343fe72422911c36855c2112da..0e498d6a15f3d2fe548a96bc6204fcc630f7fc79 100644 --- a/debian/control +++ b/debian/control @@ -32,4 +32,5 @@ Depends: Recommends: nginx, python3-mysqldb, + python3-prometheus-client, Description: Web-based user management and single sign-on software diff --git a/setup.py b/setup.py index 45f8ca6346f9ea87abcfcedc33ffac5bec9cce28..f88514ddc02872ea5c1165329885ba9df182dcb9 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ setup( 'alembic==1.0.0', 'argon2-cffi==18.3.0', 'itsdangerous==0.24', + 'prometheus-client==0.9', # The main dependencies on their own lead to version collisions and pip is # not very good at resolving them, so we pin the versions from Debian Buster diff --git a/tests/test_api.py b/tests/test_api.py index 1e2213870454a0c60b3505d0e4acd0b6c57b2ea3..7734da840a7cf4d6db8969d9c64d5a90b3ced030 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -281,3 +281,12 @@ class TestAPIRemailerResolve(UffdTestCase): self.assertEqual(r.status_code, 400) r = self.client.get(path=url_for('api.resolve_remailer', foo='bar'), headers=[basic_auth('test', 'test')], follow_redirects=True) self.assertEqual(r.status_code, 400) + +class TestAPIMetricsPrometheus(UffdTestCase): + def setUpDB(self): + db.session.add(APIClient(service=Service(name='test'), auth_username='test', auth_password='test', perm_metrics=True)) + + def test(self): + r = self.client.get(path=url_for('api.prometheus_metrics'), headers=[basic_auth('test', 'test')]) + self.assertEqual(r.status_code, 200) + self.assertTrue("uffd_version_info" in r.data.decode()) diff --git a/uffd/migrations/versions/b8fbefca3675_added_api_permission_for_metrics.py b/uffd/migrations/versions/b8fbefca3675_added_api_permission_for_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..ca86b66a621d01fe35b67268cc5acb676cf375f3 --- /dev/null +++ b/uffd/migrations/versions/b8fbefca3675_added_api_permission_for_metrics.py @@ -0,0 +1,51 @@ +"""added api permission for metrics + +Revision ID: b8fbefca3675 +Revises: f2eb2c52a61f +Create Date: 2022-08-22 21:30:19.265531 + +""" +from alembic import op +import sqlalchemy as sa + +revision = 'b8fbefca3675' +down_revision = 'f2eb2c52a61f' +branch_labels = None +depends_on = None + +def upgrade(): + meta = sa.MetaData(bind=op.get_bind()) + api_client = sa.Table('api_client', meta, + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('service_id', sa.Integer(), nullable=False), + sa.Column('auth_username', sa.String(length=40), nullable=False), + sa.Column('auth_password', sa.Text(), nullable=False), + sa.Column('perm_users', sa.Boolean(), nullable=False), + sa.Column('perm_checkpassword', sa.Boolean(), nullable=False), + sa.Column('perm_mail_aliases', sa.Boolean(), nullable=False), + sa.Column('perm_remailer', sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint(['service_id'], ['service.id'], name=op.f('fk_api_client_service_id_service'), onupdate='CASCADE', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_api_client')), + sa.UniqueConstraint('auth_username', name=op.f('uq_api_client_auth_username')) + ) + with op.batch_alter_table('api_client', copy_from=api_client) as batch_op: + batch_op.add_column(sa.Column('perm_metrics', sa.Boolean(), nullable=False, server_default=sa.false())) + +def downgrade(): + meta = sa.MetaData(bind=op.get_bind()) + api_client = sa.Table('api_client', meta, + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('service_id', sa.Integer(), nullable=False), + sa.Column('auth_username', sa.String(length=40), nullable=False), + sa.Column('auth_password', sa.Text(), nullable=False), + sa.Column('perm_users', sa.Boolean(), nullable=False), + sa.Column('perm_checkpassword', sa.Boolean(), nullable=False), + sa.Column('perm_mail_aliases', sa.Boolean(), nullable=False), + sa.Column('perm_remailer', sa.Boolean(), nullable=False), + sa.Column('perm_metrics', sa.Boolean(), nullable=False, server_default=sa.false()), + sa.ForeignKeyConstraint(['service_id'], ['service.id'], name=op.f('fk_api_client_service_id_service'), onupdate='CASCADE', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_api_client')), + sa.UniqueConstraint('auth_username', name=op.f('uq_api_client_auth_username')) + ) + with op.batch_alter_table('api_client', copy_from=api_client) as batch_op: + batch_op.drop_column('perm_metrics') diff --git a/uffd/models/api.py b/uffd/models/api.py index f770ee1a89ad8743beeef5c27857c0e6f26ea855..c3dfd5e011ed9727663923cf6f4cf9ca44d17fbd 100644 --- a/uffd/models/api.py +++ b/uffd/models/api.py @@ -18,6 +18,7 @@ class APIClient(db.Model): perm_checkpassword = Column(Boolean(), default=False, nullable=False) perm_mail_aliases = Column(Boolean(), default=False, nullable=False) perm_remailer = Column(Boolean(), default=False, nullable=False) + perm_metrics = Column(Boolean(), default=False, nullable=False) @classmethod def permission_exists(cls, name): diff --git a/uffd/models/invite.py b/uffd/models/invite.py index ad550b256f8bba4e46f6f9eee833b319396ec2de..27c1db2bbd15a4a4a432908b14cdd984c97c26bd 100644 --- a/uffd/models/invite.py +++ b/uffd/models/invite.py @@ -3,6 +3,7 @@ import datetime from flask_babel import gettext as _ from flask import current_app from sqlalchemy import Column, String, Integer, ForeignKey, DateTime, Boolean +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship from uffd.utils import token_urlfriendly @@ -30,11 +31,11 @@ class Invite(db.Model): signups = relationship('InviteSignup', back_populates='invite', lazy=True, cascade='all, delete-orphan') grants = relationship('InviteGrant', back_populates='invite', lazy=True, cascade='all, delete-orphan') - @property + @hybrid_property def expired(self): - return datetime.datetime.utcnow().replace(second=0, microsecond=0) > self.valid_until + return self.valid_until < datetime.datetime.utcnow().replace(second=0, microsecond=0) - @property + @hybrid_property def voided(self): return self.single_use and self.used diff --git a/uffd/templates/service/api.html b/uffd/templates/service/api.html index 50f9d7f971364f2afa57f4a457fe0e451b65668c..abe7ad6861b3edd6a321dba15aeeea00453c9bac 100644 --- a/uffd/templates/service/api.html +++ b/uffd/templates/service/api.html @@ -51,6 +51,10 @@ <i class="fas fa-exclamation-triangle text-warning" data-toggle="tooltip" data-placement="top" title="{{ _('This option has no effect: Remailer config options are unset') }}"></i> {% endif %} </div> + <div class="form-check"> + <input class="form-check-input" type="checkbox" id="client-perm-metrics" name="perm_metrics" value="1" aria-label="enabled" {{ 'checked' if client.perm_metrics }}> + <label class="form-check-label" for="client-perm-metrics"><b>metrics</b>: {{_('Access uffd metrics')}}</label> + </div> </div> </form> diff --git a/uffd/views/__init__.py b/uffd/views/__init__.py index 4f5bdb08fdb0ae7a282d4e57df70e1b84f39e45f..1ee14190f7396badd7c833e361f4481bba8e7350 100644 --- a/uffd/views/__init__.py +++ b/uffd/views/__init__.py @@ -36,3 +36,5 @@ def init_app(app): app.register_blueprint(api.bp) app.register_blueprint(mail.bp) app.register_blueprint(rolemod.bp) + + app.add_url_rule("/metrics", view_func=api.prometheus_metrics) diff --git a/uffd/views/api.py b/uffd/views/api.py index 506bf5d9db591bf25fb01fd440146b2fe5610e4c..0bfa71e775348ecca23da62b08653133fcc24da3 100644 --- a/uffd/views/api.py +++ b/uffd/views/api.py @@ -1,9 +1,11 @@ import functools -from flask import Blueprint, jsonify, request, abort +from flask import Blueprint, jsonify, request, abort, Response from uffd.database import db -from uffd.models import User, ServiceUser, Group, Mail, MailReceiveAddress, MailDestinationAddress, APIClient +from uffd.models import ( + User, ServiceUser, Group, Mail, MailReceiveAddress, MailDestinationAddress, APIClient, + RecoveryCodeMethod, TOTPMethod, WebauthnMethod, Invite, Role, Service ) from .session import login_ratelimit bp = Blueprint('api', __name__, template_folder='templates', url_prefix='/api/v1/') @@ -158,3 +160,54 @@ def resolve_remailer(): if not service_user: return jsonify(address=None) return jsonify(address=service_user.real_email) + +@bp.route('/metrics_prometheus', methods=['GET']) +@apikey_required('metrics') +def prometheus_metrics(): + import pkg_resources #pylint: disable=import-outside-toplevel + from prometheus_client.core import CollectorRegistry, CounterMetricFamily, InfoMetricFamily #pylint: disable=import-outside-toplevel + from prometheus_client import PLATFORM_COLLECTOR, generate_latest, CONTENT_TYPE_LATEST #pylint: disable=import-outside-toplevel + + class UffdCollector(): + def collect(self): #pylint: disable=no-self-use + try: + uffd_version = str(pkg_resources.get_distribution('uffd').version) + except pkg_resources.DistributionNotFound: + uffd_version = "unknown" + yield InfoMetricFamily('uffd_version', 'Various version infos', value={"version": uffd_version}) + + user_metric = CounterMetricFamily('uffd_users_total', 'Number of users', labels=['user_type']) + user_metric.add_metric(['regular'], value=User.query.filter_by(is_service_user=False).count()) + user_metric.add_metric(['service'], User.query.filter_by(is_service_user=True).count()) + yield user_metric + + mfa_auth_metric = CounterMetricFamily('uffd_users_auth_mfa_total', 'mfa stats', labels=['mfa_type']) + mfa_auth_metric.add_metric(['recoverycode'], value=RecoveryCodeMethod.query.count()) + mfa_auth_metric.add_metric(['totp'], value=TOTPMethod.query.count()) + mfa_auth_metric.add_metric(['webauthn'], value=WebauthnMethod.query.count()) + yield mfa_auth_metric + + yield CounterMetricFamily('uffd_roles_total', 'Number of roles', value=Role.query.count()) + + role_members_metric = CounterMetricFamily('uffd_role_members_total', 'Members of a role', labels=['role_name']) + for role in Role.query.all(): + role_members_metric.add_metric([role.name], value=len(role.members)) + yield role_members_metric + + group_metric = CounterMetricFamily('uffd_groups_total', 'Total number of groups', value=Group.query.count()) + yield group_metric + + invite_metric = CounterMetricFamily('uffd_invites_total', 'Number of invites', labels=['invite_state']) + invite_metric.add_metric(['used'], value=Invite.query.filter_by(used=True).count()) + invite_metric.add_metric(['expired'], value=Invite.query.filter_by(expired=True).count()) + invite_metric.add_metric(['disabled'], value=Invite.query.filter_by(disabled=True).count()) + invite_metric.add_metric(['voided'], value=Invite.query.filter_by(voided=True).count()) + invite_metric.add_metric([], value=Invite.query.count()) + yield invite_metric + + yield CounterMetricFamily('uffd_services_total', 'Number of services', value=Service.query.count()) + + registry = CollectorRegistry(auto_describe=True) + registry.register(PLATFORM_COLLECTOR) + registry.register(UffdCollector()) + return Response(response=generate_latest(registry=registry),content_type=CONTENT_TYPE_LATEST) diff --git a/uffd/views/service.py b/uffd/views/service.py index dbedf510577ba3a0cb6b161dafaeb0dfd446a659..cc4ee8d70b0d5a2f39a3df1d997e6c9e2d7c8b42 100644 --- a/uffd/views/service.py +++ b/uffd/views/service.py @@ -157,6 +157,7 @@ def api_submit(service_id, id=None): client.perm_checkpassword = request.form.get('perm_checkpassword') == '1' client.perm_mail_aliases = request.form.get('perm_mail_aliases') == '1' client.perm_remailer = request.form.get('perm_remailer') == '1' + client.perm_metrics = request.form.get('perm_metrics') == '1' db.session.commit() return redirect(url_for('service.show', id=service.id))