diff --git a/files/gitlab-ldap-sync.py b/files/gitlab-ldap-sync.py new file mode 100644 index 0000000000000000000000000000000000000000..8f8c0fcab53e00b34e93e7f70a8d548ef4cb97ba --- /dev/null +++ b/files/gitlab-ldap-sync.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 + +# Gitlab LDAP Synchronization +# +# Adds users to Gitlab groups that match the corresponding 'ldapmapping' +# filter and removes Gitlab users with LDAP identity that do not match the +# filter. However this script does not create Gitlab groups. +# +# Creates Gitlab users for all LDAP users with Gitlab access (based on 'base' +# and 'user_filter' config variables). Updates admin status of Gitlab users +# based on 'admin_group' membership. +# +# Other user attributes like display name and email are NOT updated, because +# the API does not allow this reliably (due to bugs in Gitlab). Configure +# 'omniauth_sync_profile_from_provider' and 'omniauth_sync_profile_attributes' +# to update these attributes on each login instead. Gitlab automatically +# blocks users that disappeared from LDAP/the configured LDAP group with a +# special "ldap_blocked" state. Due to bugs in Gitlab, users in this state +# are not always automatically unblocked when they reappear in LDAP. This +# script does not unblock those users, because Gitlab does not allow +# unblocking of "ldap_blocked" users via the API (not even via the +# Admin-Interface). +# +# The config is read from "/usr/local/etc/gitlab-ldap-sync.json". It contains +# three top-level objects: ldap, groups, ldap_sync +# +# 'ldap_sync' contains the Gitlab API details (api_url and api_token) for this +# script. 'ldap' contains the content of 'ldap_servers' from the Gitlab +# configuration. +# +# 'groups' is a list of group definitions. Only group definitions that contain +# a 'ldapmapping' (LDAP filter string) are synchronized. The attributes 'name' +# and 'parent' (optional) are used to identitfy the Gitlab group. + +import sys +import ssl +import secrets +import json +import logging + +import ldap3 +import gitlab +from systemd.journal import JournalHandler + +def connect_ldap(host, port, ca_file, bind_dn, bind_passwd): + tls = ldap3.Tls(validate=ssl.CERT_REQUIRED, ca_certs_file=ca_file) + server = ldap3.Server(host, port=port, use_ssl=True, get_info=ldap3.ALL, tls=tls) + conn = ldap3.Connection(server, bind_dn, bind_passwd, auto_bind=True) + old_search = conn.search + def search(*args, **kwargs): + kwargs.update({'attributes': [ldap3.ALL_ATTRIBUTES, ldap3.ALL_OPERATIONAL_ATTRIBUTES]}) + return old_search(*args, **kwargs) + conn.search = search + return conn + +def sync_user(gl, dn, username, name, email, is_admin=False, ldap_provider='ldapmain', dry_run=True): + logging.debug('Synchronizing user %s', dn) + user = (gl.users.list(provider=ldap_provider, extern_uid=dn) or [None])[0] + if user is None: + logging.debug('No corresponding Gitlab user found') + logging.info('Creating user %s', dn) + attrs = {'provider': ldap_provider, 'extern_uid': dn} + attrs['username'] = username + attrs['name'] = name + attrs['email'] = email + attrs['admin'] = is_admin + attrs['password'] = secrets.token_hex(50) + attrs['skip_confirmation'] = True + # Emulate gitlab's username mangling if it is already taken + tries = 5 + for i in range(1, tries + 1): + try: + if not dry_run: + logging.debug('Trying to create new user "%s"', attrs['username']) + gl.users.create(attrs) + logging.debug('User creation successful') + return + except gitlab.exceptions.GitlabCreateError as e: + if i < tries: + logging.debug('User creation with username "%s" failed: %s', attrs['username'], e.error_message) + else: + logging.exception('User creation with username "%s" failed', attrs['username']) + attrs['username'] = username + '%d'%(i + 1) + logging.debug('Giving up user creation') + return + logging.debug('Found corresponding Gitlab user "%s", synchronizing attributes', user.username) + if user.is_admin != is_admin: + logging.info('Changing admin status of user %s (%s -> %s)', dn, user.is_admin, is_admin) + if not dry_run: + user.admin = is_admin + user.save() + if user.is_admin != is_admin: + logging.warning('Changing admin status of user %s failed silently', dn) + +def sync_group_members(gl, group, member_dns, ldap_provider='ldapmain', dry_run=True): + logging.debug('Synchronizing members of group %s', group.full_path) + existing_member_dns = [] + logging.debug('Checking current Gitlab group members') + members = group.members.list(all=True) + owner_count = len([member for member in members if member.access_level == gitlab.OWNER_ACCESS]) + logging.debug('Group has %d owners', owner_count) + for member in members: + user = gl.users.get(member.id) + identities = {item['provider']: item['extern_uid'] for item in user.identities} + if ldap_provider not in identities: + logging.debug('User "%s" has no ldap identity, skipping', user.username) + continue + dn = identities[ldap_provider] + existing_member_dns.append(dn) + if dn not in member_dns: + logging.info('Removing user %s from group %s', dn, group.full_path) + if not dry_run: + member.delete() + elif member.access_level != gitlab.MASTER_ACCESS: + # We cannot change the access level of the last owner of a top-level group + if member.access_level == gitlab.OWNER_ACCESS: + if owner_count == 1 and not group.parent_id: + logging.info('Not updating access level of %s in group %s, because it is the last owner', dn, group.full_path) + continue + owner_count -= 1 + logging.info('Updating access level of %s in group %s', dn, group.full_path) + try: + if not dry_run: + member.access_level = gitlab.MASTER_ACCESS + member.save() + except gitlab.exceptions.GitlabUpdateError as e: + logging.exception('Access level update of %s in group %s failed', dn, group.full_path) + for dn in member_dns: + if dn in existing_member_dns: + continue + user = (gl.users.list(provider=ldap_provider, extern_uid=dn) or [None])[0] + logging.info('Adding user %s to group %s', dn, group.full_path) + if user is None: + logging.warning('Could not add user %s to group %s: No Gitlab user found', dn, group.full_path) + continue + try: + if not dry_run: + group.members.create({'user_id': user.id, 'access_level': gitlab.MASTER_ACCESS}) + except gitlab.exceptions.GitlabCreateError as e: + if e.response_code == 500: + logging.info('Adding user %s to group %s failed with Internal Server Error', dn, group.full_path) + logging.info('This is regularly caused by adding a user that is already an inherited member with a higher access level and therefore ignored') + else: + logging.exception('Adding user %s to group %s failed', dn, group.full_path) + +def load_config(path): + logging.debug('Loading config from %s', path) + with open(path, 'r') as f: + config = json.load(f) + groups = {} + for item in config['groups']: + full_path = item['name'] + if 'parent' in item: + full_path = item['parent'] + '/' + full_path + if full_path in groups: + logging.error('Group config contains duplicate group "%s"', full_path) + logging.info('Ignoring duplicate group entry') + continue + groups[full_path] = item + config['groups'] = groups + return config + +def main(config_path, dry_run=True): + if dry_run: + logging.warning('Gitlab LDAP sync running in dry_run mode') + config = load_config(config_path) + gl = gitlab.Gitlab(config['ldap_sync']['api_url'], config['ldap_sync']['api_token'], ssl_verify=True) + conn = connect_ldap(host=config['ldap']['main']['host'], port=config['ldap']['main']['port'], + ca_file=config['ldap']['main']['ca_file'], + bind_dn=config['ldap']['main']['bind_dn'], + bind_passwd=config['ldap']['main']['password']) + logging.info('Starting user synchronization') + admin_group_dn = 'cn=%s,%s'%(config['ldap']['main']['admin_group'], config['ldap']['main']['group_base']) + logging.debug('Fetching all LDAP users') + conn.search(config['ldap']['main']['base'], config['ldap']['main']['user_filter']) + for entry in conn.response: + dn = entry['dn'] + email = entry['attributes']['mail'][0] + name = entry['attributes']['cn'][0] + username = entry['attributes'][config['ldap']['main']['uid']][0] + is_admin = admin_group_dn in entry['attributes']['memberOf'] + try: + sync_user(gl, dn=dn, username=username, name=name, email=email, is_admin=is_admin, ldap_provider='ldapmain', dry_run=dry_run) + except gitlab.exceptions.GitlabOperationError as e: + logging.exception('Error during synchronization of user %s', dn) + logging.info('User synchronization finished') + + logging.info('Starting group synchronization') + for group in gl.groups.list(all=True): + group_config = config['groups'].get(group.full_path) + if group_config is None or 'ldapmapping' not in group_config: + continue + ldap_filter = '(&{}{})'.format(group_config['ldapmapping'], config['ldap']['main']['user_filter']) + conn.search(config['ldap']['main']['base'], ldap_filter) + member_dns = [entry['dn'] for entry in conn.response] + try: + sync_group_members(gl, group, member_dns, ldap_provider='ldapmain', dry_run=dry_run) + except gitlab.exceptions.GitlabOperationError as e: + logging.exception('Error during synchronization of group %s', group.full_path) + logging.info('Group synchronization finished') + +if __name__ == "__main__": + journal_handler = JournalHandler(SYSLOG_IDENTIFIER='gitlab-ldap-sync', level=logging.INFO) + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.DEBUG) + stderr_handler = logging.StreamHandler(sys.stderr) + stderr_handler.setLevel(logging.WARNING) + logging.basicConfig(level=logging.INFO, handlers=[journal_handler, stdout_handler, stderr_handler]) + + try: + main(config_path='/usr/local/etc/gitlab-ldap-sync.json', dry_run=False) + except Exception as e: + logging.exception('Error') + diff --git a/tasks/gitlab.yml b/tasks/gitlab.yml index 203fc082dea15371d68e15a4254549d6f94d1947..5d0ba67049ee1513a1c3e975e24dca2bedf8b0a7 100644 --- a/tasks/gitlab.yml +++ b/tasks/gitlab.yml @@ -30,19 +30,34 @@ description: "{{ item.description | default(omit) }}" with_items: "{{ gitlab.groups }}" -- name: setup gitlab ldap group membership sync +- name: setup gitlab ldap sync when: gitlab.ldap.enabled block: + - name: create gitlab-ldap-sync user + user: + name: gitlab-ldap-sync + shell: /bin/bash + system: true + groups: + - ssl-cert + - name: copy ldap sync config + copy: + content: "{{ {'groups': gitlab.groups, 'ldap': gitlab.ldap.servers, 'ldap_sync': {'api_url': gitlab.external_url, 'api_token': gitlab.api_token}}|to_nice_json }}" + dest: /usr/local/etc/gitlab-ldap-sync.json + mode: 0600 + owner: gitlab-ldap-sync + group: root - name: copy ldap sync script - template: - src: gitlab-ldap-groupsync.py.j2 - dest: /usr/local/bin/gitlab-ldap-groupsync.py - mode: 0700 + copy: + src: gitlab-ldap-sync.py + dest: /usr/local/bin/gitlab-ldap-sync.py + mode: 0755 owner: root group: root - - name: add ldap group sync cronjob + - name: add ldap sync cronjob cron: - name: gitlab ldap group membership sync - job: "/usr/local/bin/gitlab-ldap-groupsync.py > /dev/null 2>&1" + name: gitlab ldap sync + job: "/usr/local/bin/gitlab-ldap-sync.py > /dev/null" minute: "12,27,42,57" + user: gitlab-ldap-sync diff --git a/templates/gitlab-ldap-groupsync.py.j2 b/templates/gitlab-ldap-groupsync.py.j2 deleted file mode 100644 index c5e467a6bc792bde3b8c2f5cc14d0b4dd231e74f..0000000000000000000000000000000000000000 --- a/templates/gitlab-ldap-groupsync.py.j2 +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 - -import logging -import sys -import ssl -from systemd.journal import JournalHandler -from ldap3 import Server, Connection, Tls, ALL, ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES -import gitlab - -# this is needed to parse json as python code... -true = True -false = False - -config = {} -config['groups'] = {{ gitlab.groups|to_json(ensure_ascii=False) }} -config['ldap'] = {{ gitlab.ldap.servers|to_json(ensure_ascii=False) }} - -def getLogger(): - log = logging.getLogger() - log.setLevel(logging.DEBUG) - log.addHandler(JournalHandler(SYSLOG_IDENTIFIER='gitlab-ldap-groupsync', level=logging.INFO)) - stdoutHandler = logging.StreamHandler(sys.stdout) - stdoutHandler.setLevel(logging.DEBUG) - log.addHandler(stdoutHandler) - stderrHandler = logging.StreamHandler(sys.stderr) - stderrHandler.setLevel(logging.WARNING) - log.addHandler(stderrHandler) - return log - -def fixLdapConnection(conn): - old_search = conn.search - def search(*args, **kwargs): - kwargs.update({'attributes': [ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES]}) - return old_search(*args, **kwargs) - conn.search = search - return conn - -def configGroupByPath(path): - for configGroup in config['groups']: - configGroupPath = '{}/{}'.format(configGroup.get('parent', ''), configGroup['name']).lstrip('/') - if configGroupPath == path: - return configGroup - return None - -def getLdapMembers(connection, ldapfilter): - combinedFilter = '(&{}{})'.format(ldapfilter, config['ldap']['main']['user_filter']) - connection.search(config['ldap']['main']['base'], combinedFilter) - result = [] - for entry in connection.entries: - result += [entry.entry_dn] - return result - -def gitlabSync(logger, gitlabApi, ldapConnection): - for gitlabGroup in gitlabApi.groups.list(all=True): - logger.debug('') - configGroup = configGroupByPath(gitlabGroup.full_path) - if not configGroup or not configGroup.get('ldapmapping'): - continue - logger.debug('Handling group "%s"', gitlabGroup.full_path) - ldapMembers = getLdapMembers(ldapConnection, configGroup['ldapmapping']) - logger.debug('ldapmapping for group: %s', configGroup['ldapmapping']) - logger.debug('ldap members for group: %s', ldapMembers) - for member in gitlabGroup.members.list(): - gitlabUser = gitlabApi.users.get(member.id) - try: - uid = None - for identity in gitlabUser.identities: - if identity['provider'] == 'ldapmain': - uid = identity['extern_uid'] - if not uid: - raise AttributeError - except AttributeError: - logger.debug('User %s is no ldap user, skipping', member.username) - continue - if not uid in ldapMembers: - logger.warning('User %s should not be a member of %s, removing', uid, gitlabGroup.full_path) - member.delete() - else: - # we remove the uid from ldapMembers so we can add all left over uids as members later on - ldapMembers.remove(uid) - if member.access_level != gitlab.MASTER_ACCESS: - member.access_level = gitlab.MASTER_ACCESS - try: - member.save() - except gitlab.exceptions.GitlabUpdateError as e: - logger.critical('Updating user "%s" failed:', uid) - logger.critical(e, exc_info=True) - for uid in ldapMembers: - logger.warning('User %s should be member of %s, adding', uid, gitlabGroup.full_path) - newMember = gitlabApi.users.list(extern_uid=uid, provider='ldapmain') - if len(newMember) != 1: - logger.error('User "%s" was not found in gitlab. Most often this means the user has never logged into gitlab. Api result: %s', uid, newMember) - continue - newMember = newMember[0] - try: - gitlabGroup.members.create({'user_id': newMember.id, 'access_level': gitlab.MASTER_ACCESS}) - except gitlab.exceptions.GitlabCreateError as e: - logger.critical('Adding user "%s" failed:', uid) - logger.critical(e, exc_info=True) - -def main(): - logger = getLogger() - logger.info('Starting sync') - try: - logger.debug('Connecting to Gitlab') - gitlabApi = gitlab.Gitlab(url='{{ gitlab.external_url }}', private_token='{{ gitlab.api_token }}', ssl_verify=True) - - logger.debug('Connecting to LDAP') - ldapTls = Tls(validate=ssl.CERT_REQUIRED, ca_certs_file=config['ldap']['main']['ca_file']) - ldapServer = Server(config['ldap']['main']['host'], port=config['ldap']['main']['port'], use_ssl=(config['ldap']['main']['encryption']=='simple_tls'), get_info=ALL, tls=ldapTls) - ldapConnection = fixLdapConnection(Connection(ldapServer, config['ldap']['main']['bind_dn'], config['ldap']['main']['password'], auto_bind=True)) - logger.debug('Setup done, iterating gitlab groups...') - - gitlabSync(logger, gitlabApi, ldapConnection) - - logger.info('Sync done') - except Exception as e: - logger.critical(e, exc_info=True) - - -if __name__ == "__main__": - main()