replaced group sync with more generic ldap sync script

Changes:
* Creates Gitlab user accounts for all LDAP users with Gitlab access
* Updates admin status of Gitlab users based on LDAP admin_group membership
* Reliably updates member access levels in groups with more than 20 members (issue with paging of Gitlab API)
* The cronjob now outputs log messages of level WARNING or higher

files/gitlab-ldap-sync.py

+#!/usr/bin/env python3
+
+# Gitlab LDAP Synchronization
+#
+# Adds users to Gitlab groups that match the corresponding 'ldapmapping'
+# filter and removes Gitlab users with LDAP identity that do not match the
+# filter. However this script does not create Gitlab groups.
+#
+# Creates Gitlab users for all LDAP users with Gitlab access (based on 'base'
+# and 'user_filter' config variables). Updates admin status of Gitlab users
+# based on 'admin_group' membership.
+#
+# Other user attributes like display name and email are NOT updated, because
+# the API does not allow this reliably (due to bugs in Gitlab). Configure
+# 'omniauth_sync_profile_from_provider' and 'omniauth_sync_profile_attributes'
+# to update these attributes on each login instead. Gitlab automatically
+# blocks users that disappeared from LDAP/the configured LDAP group with a
+# special "ldap_blocked" state. Due to bugs in Gitlab, users in this state
+# are not always automatically unblocked when they reappear in LDAP. This
+# script does not unblock those users, because Gitlab does not allow
+# unblocking of "ldap_blocked" users via the API (not even via the
+# Admin-Interface).
+#
+# The config is read from "/usr/local/etc/gitlab-ldap-sync.json". It contains
+# three top-level objects: ldap, groups, ldap_sync
+# 
+# 'ldap_sync' contains the Gitlab API details (api_url and api_token) for this
+# script. 'ldap' contains the content of 'ldap_servers' from the Gitlab
+# configuration.
+#
+# 'groups' is a list of group definitions. Only group definitions that contain
+# a 'ldapmapping' (LDAP filter string) are synchronized. The attributes 'name'
+# and 'parent' (optional) are used to identitfy the Gitlab group.
+
+import sys
+import ssl
+import secrets
+import json
+import logging
+
+import ldap3
+import gitlab
+from systemd.journal import JournalHandler
+
+def connect_ldap(host, port, ca_file, bind_dn, bind_passwd):
+	tls = ldap3.Tls(validate=ssl.CERT_REQUIRED, ca_certs_file=ca_file)
+	server = ldap3.Server(host, port=port, use_ssl=True, get_info=ldap3.ALL, tls=tls)
+	conn = ldap3.Connection(server, bind_dn, bind_passwd, auto_bind=True)
+	old_search = conn.search
+	def search(*args, **kwargs):
+		kwargs.update({'attributes': [ldap3.ALL_ATTRIBUTES, ldap3.ALL_OPERATIONAL_ATTRIBUTES]})
+		return old_search(*args, **kwargs)
+	conn.search = search
+	return conn
+
+def sync_user(gl, dn, username, name, email, is_admin=False, ldap_provider='ldapmain', dry_run=True):
+	logging.debug('Synchronizing user %s', dn)
+	user = (gl.users.list(provider=ldap_provider, extern_uid=dn) or [None])[0]
+	if user is None:
+		logging.debug('No corresponding Gitlab user found')
+		logging.info('Creating user %s', dn)
+		attrs = {'provider': ldap_provider, 'extern_uid': dn}
+		attrs['username'] = username
+		attrs['name'] = name
+		attrs['email'] = email
+		attrs['admin'] = is_admin
+		attrs['password'] = secrets.token_hex(50)
+		attrs['skip_confirmation'] = True
+		# Emulate gitlab's username mangling if it is already taken
+		tries = 5
+		for i in range(1, tries + 1):
+			try:
+				if not dry_run:
+					logging.debug('Trying to create new user "%s"', attrs['username'])
+					gl.users.create(attrs)
+				logging.debug('User creation successful')
+				return
+			except gitlab.exceptions.GitlabCreateError as e:
+				if i < tries:
+					logging.debug('User creation with username "%s" failed: %s', attrs['username'], e.error_message)
+				else:
+					logging.exception('User creation with username "%s" failed', attrs['username'])
+			attrs['username'] = username + '%d'%(i + 1)
+		logging.debug('Giving up user creation')
+		return
+	logging.debug('Found corresponding Gitlab user "%s", synchronizing attributes', user.username)
+	if user.is_admin != is_admin:
+		logging.info('Changing admin status of user %s (%s -> %s)', dn, user.is_admin, is_admin)
+		if not dry_run:
+			user.admin = is_admin
+			user.save()
+			if user.is_admin != is_admin:
+				logging.warning('Changing admin status of user %s failed silently', dn)
+
+def sync_group_members(gl, group, member_dns, ldap_provider='ldapmain', dry_run=True):
+	logging.debug('Synchronizing members of group %s', group.full_path)
+	existing_member_dns = []
+	logging.debug('Checking current Gitlab group members')
+	members = group.members.list(all=True)
+	owner_count = len([member for member in members if member.access_level == gitlab.OWNER_ACCESS])
+	logging.debug('Group has %d owners', owner_count)
+	for member in members:
+		user = gl.users.get(member.id)
+		identities = {item['provider']: item['extern_uid'] for item in user.identities}
+		if ldap_provider not in identities:
+			logging.debug('User "%s" has no ldap identity, skipping', user.username)
+			continue
+		dn = identities[ldap_provider]
+		existing_member_dns.append(dn)
+		if dn not in member_dns:
+			logging.info('Removing user %s from group %s', dn, group.full_path)
+			if not dry_run:
+				member.delete()
+		elif member.access_level != gitlab.MASTER_ACCESS:
+			# We cannot change the access level of the last owner of a top-level group
+			if member.access_level == gitlab.OWNER_ACCESS:
+				if owner_count == 1 and not group.parent_id:
+					logging.info('Not updating access level of %s in group %s, because it is the last owner', dn, group.full_path)
+					continue
+				owner_count -= 1
+			logging.info('Updating access level of %s in group %s', dn, group.full_path)
+			try:
+				if not dry_run:
+					member.access_level = gitlab.MASTER_ACCESS
+					member.save()
+			except gitlab.exceptions.GitlabUpdateError as e:
+				logging.exception('Access level update of %s in group %s failed', dn, group.full_path)
+	for dn in member_dns:
+		if dn in existing_member_dns:
+			continue
+		user = (gl.users.list(provider=ldap_provider, extern_uid=dn) or [None])[0]
+		logging.info('Adding user %s to group %s', dn, group.full_path)
+		if user is None:
+			logging.warning('Could not add user %s to group %s: No Gitlab user found', dn, group.full_path)
+			continue
+		try:
+			if not dry_run:
+				group.members.create({'user_id': user.id, 'access_level': gitlab.MASTER_ACCESS})
+		except gitlab.exceptions.GitlabCreateError as e:
+			if e.response_code == 500:
+				logging.info('Adding user %s to group %s failed with Internal Server Error', dn, group.full_path)
+				logging.info('This is regularly caused by adding a user that is already an inherited member with a higher access level and therefore ignored')
+			else:
+				logging.exception('Adding user %s to group %s failed', dn, group.full_path)
+
+def load_config(path):
+	logging.debug('Loading config from %s', path)
+	with open(path, 'r') as f:
+		config = json.load(f)
+	groups = {}
+	for item in config['groups']:
+		full_path = item['name']
+		if 'parent' in item:
+			full_path = item['parent'] + '/' + full_path
+		if full_path in groups:
+			logging.error('Group config contains duplicate group "%s"', full_path)
+			logging.info('Ignoring duplicate group entry')
+			continue
+		groups[full_path] = item
+	config['groups'] = groups
+	return config
+
+def main(config_path, dry_run=True):
+	if dry_run:
+		logging.warning('Gitlab LDAP sync running in dry_run mode')
+	config = load_config(config_path)
+	gl = gitlab.Gitlab(config['ldap_sync']['api_url'], config['ldap_sync']['api_token'], ssl_verify=True)
+	conn = connect_ldap(host=config['ldap']['main']['host'], port=config['ldap']['main']['port'],
+	                    ca_file=config['ldap']['main']['ca_file'],
+	                    bind_dn=config['ldap']['main']['bind_dn'],
+	                    bind_passwd=config['ldap']['main']['password'])
+	logging.info('Starting user synchronization')
+	admin_group_dn = 'cn=%s,%s'%(config['ldap']['main']['admin_group'], config['ldap']['main']['group_base'])
+	logging.debug('Fetching all LDAP users')
+	conn.search(config['ldap']['main']['base'], config['ldap']['main']['user_filter'])
+	for entry in conn.response:
+		dn = entry['dn']
+		email = entry['attributes']['mail'][0]
+		name = entry['attributes']['cn'][0]
+		username = entry['attributes'][config['ldap']['main']['uid']][0]
+		is_admin = admin_group_dn in entry['attributes']['memberOf']
+		try:
+			sync_user(gl, dn=dn, username=username, name=name, email=email, is_admin=is_admin, ldap_provider='ldapmain', dry_run=dry_run)
+		except gitlab.exceptions.GitlabOperationError as e:
+			logging.exception('Error during synchronization of user %s', dn)
+	logging.info('User synchronization finished')
+
+	logging.info('Starting group synchronization')
+	for group in gl.groups.list(all=True):
+		group_config = config['groups'].get(group.full_path)
+		if group_config is None or 'ldapmapping' not in group_config:
+			continue
+		ldap_filter = '(&{}{})'.format(group_config['ldapmapping'], config['ldap']['main']['user_filter'])
+		conn.search(config['ldap']['main']['base'], ldap_filter)
+		member_dns = [entry['dn'] for entry in conn.response]
+		try:
+			sync_group_members(gl, group, member_dns, ldap_provider='ldapmain', dry_run=dry_run)
+		except gitlab.exceptions.GitlabOperationError as e:
+			logging.exception('Error during synchronization of group %s', group.full_path)
+	logging.info('Group synchronization finished')
+
+if __name__ == "__main__":
+	journal_handler = JournalHandler(SYSLOG_IDENTIFIER='gitlab-ldap-sync', level=logging.INFO)
+	stdout_handler = logging.StreamHandler(sys.stdout)
+	stdout_handler.setLevel(logging.DEBUG)
+	stderr_handler = logging.StreamHandler(sys.stderr)
+	stderr_handler.setLevel(logging.WARNING)
+	logging.basicConfig(level=logging.INFO, handlers=[journal_handler, stdout_handler, stderr_handler])
+
+	try:
+		main(config_path='/usr/local/etc/gitlab-ldap-sync.json', dry_run=False)
+	except Exception as e:
+		logging.exception('Error')
+

tasks/gitlab.yml

@@ -30,19 +30,34 @@
     description: "{{ item.description | default(omit) }}"
   with_items: "{{ gitlab.groups }}"
 
-- name: setup gitlab ldap group membership sync
+- name: setup gitlab ldap sync
   when: gitlab.ldap.enabled
   block:
+  - name: create gitlab-ldap-sync user
+    user:
+      name: gitlab-ldap-sync
+      shell: /bin/bash
+      system: true
+      groups:
+        - ssl-cert
+  - name: copy ldap sync config
+    copy:
+      content: "{{ {'groups': gitlab.groups, 'ldap': gitlab.ldap.servers, 'ldap_sync': {'api_url': gitlab.external_url, 'api_token': gitlab.api_token}}|to_nice_json }}"
+      dest: /usr/local/etc/gitlab-ldap-sync.json
+      mode: 0600
+      owner: gitlab-ldap-sync
+      group: root
   - name: copy ldap sync script
-    template:
-      src: gitlab-ldap-groupsync.py.j2
-      dest: /usr/local/bin/gitlab-ldap-groupsync.py
-      mode: 0700
+    copy:
+      src: gitlab-ldap-sync.py
+      dest: /usr/local/bin/gitlab-ldap-sync.py
+      mode: 0755
       owner: root
       group: root
-  - name: add ldap group sync cronjob
+  - name: add ldap sync cronjob
     cron:
-      name: gitlab ldap group membership sync
-      job: "/usr/local/bin/gitlab-ldap-groupsync.py > /dev/null 2>&1"
+      name: gitlab ldap sync
+      job: "/usr/local/bin/gitlab-ldap-sync.py > /dev/null"
       minute: "12,27,42,57"
+      user: gitlab-ldap-sync
 

templates/gitlab-ldap-groupsync.py.j2

-#!/usr/bin/env python3
-
-import logging
-import sys
-import ssl
-from systemd.journal import JournalHandler
-from ldap3 import Server, Connection, Tls, ALL, ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES
-import gitlab
-
-# this is needed to parse json as python code...
-true = True
-false = False
-
-config = {}
-config['groups'] = {{ gitlab.groups|to_json(ensure_ascii=False) }}
-config['ldap'] = {{ gitlab.ldap.servers|to_json(ensure_ascii=False) }}
-
-def getLogger():
-	log = logging.getLogger()
-	log.setLevel(logging.DEBUG)
-	log.addHandler(JournalHandler(SYSLOG_IDENTIFIER='gitlab-ldap-groupsync', level=logging.INFO))
-	stdoutHandler = logging.StreamHandler(sys.stdout)
-	stdoutHandler.setLevel(logging.DEBUG)
-	log.addHandler(stdoutHandler)
-	stderrHandler = logging.StreamHandler(sys.stderr)
-	stderrHandler.setLevel(logging.WARNING)
-	log.addHandler(stderrHandler)
-	return log
-
-def fixLdapConnection(conn):
-	old_search = conn.search
-	def search(*args, **kwargs):
-		kwargs.update({'attributes': [ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES]})
-		return old_search(*args, **kwargs)
-	conn.search = search
-	return conn
-
-def configGroupByPath(path):
-	for configGroup in config['groups']:
-		configGroupPath = '{}/{}'.format(configGroup.get('parent', ''), configGroup['name']).lstrip('/')
-		if configGroupPath == path:
-			return configGroup
-	return None
-
-def getLdapMembers(connection, ldapfilter):
-	combinedFilter = '(&{}{})'.format(ldapfilter, config['ldap']['main']['user_filter'])
-	connection.search(config['ldap']['main']['base'], combinedFilter)
-	result = []
-	for entry in connection.entries:
-		result += [entry.entry_dn]
-	return result
-
-def gitlabSync(logger, gitlabApi, ldapConnection):
-	for gitlabGroup in gitlabApi.groups.list(all=True):
-		logger.debug('')
-		configGroup = configGroupByPath(gitlabGroup.full_path)
-		if not configGroup or not configGroup.get('ldapmapping'):
-			continue
-		logger.debug('Handling group "%s"', gitlabGroup.full_path)
-		ldapMembers = getLdapMembers(ldapConnection, configGroup['ldapmapping'])
-		logger.debug('ldapmapping for group: %s', configGroup['ldapmapping'])
-		logger.debug('ldap members for group: %s', ldapMembers)
-		for member in gitlabGroup.members.list():
-			gitlabUser = gitlabApi.users.get(member.id)
-			try:
-				uid = None
-				for identity in gitlabUser.identities:
-					if identity['provider'] == 'ldapmain':
-						uid = identity['extern_uid']
-				if not uid:
-					raise AttributeError
-			except AttributeError:
-				logger.debug('User %s is no ldap user, skipping', member.username)
-				continue
-			if not uid in ldapMembers:
-				logger.warning('User %s should not be a member of %s, removing', uid, gitlabGroup.full_path)
-				member.delete()
-			else:
-				# we remove the uid from ldapMembers so we can add all left over uids as members later on
-				ldapMembers.remove(uid)
-				if member.access_level != gitlab.MASTER_ACCESS:
-					member.access_level = gitlab.MASTER_ACCESS
-					try:
-						member.save()
-					except gitlab.exceptions.GitlabUpdateError as e:
-						logger.critical('Updating user "%s" failed:', uid)
-						logger.critical(e, exc_info=True)
-		for uid in ldapMembers:
-			logger.warning('User %s should be member of %s, adding', uid, gitlabGroup.full_path)
-			newMember = gitlabApi.users.list(extern_uid=uid, provider='ldapmain')
-			if len(newMember) != 1:
-				logger.error('User "%s" was not found in gitlab. Most often this means the user has never logged into gitlab. Api result: %s', uid, newMember)
-				continue
-			newMember = newMember[0]
-			try:
-				gitlabGroup.members.create({'user_id': newMember.id, 'access_level': gitlab.MASTER_ACCESS})
-			except gitlab.exceptions.GitlabCreateError as e:
-				logger.critical('Adding user "%s" failed:', uid)
-				logger.critical(e, exc_info=True)
-
-def main():
-	logger = getLogger()
-	logger.info('Starting sync')
-	try:
-		logger.debug('Connecting to Gitlab')
-		gitlabApi = gitlab.Gitlab(url='{{ gitlab.external_url }}', private_token='{{ gitlab.api_token }}', ssl_verify=True)
-
-		logger.debug('Connecting to LDAP')
-		ldapTls = Tls(validate=ssl.CERT_REQUIRED, ca_certs_file=config['ldap']['main']['ca_file'])
-		ldapServer = Server(config['ldap']['main']['host'], port=config['ldap']['main']['port'], use_ssl=(config['ldap']['main']['encryption']=='simple_tls'), get_info=ALL, tls=ldapTls)
-		ldapConnection = fixLdapConnection(Connection(ldapServer, config['ldap']['main']['bind_dn'], config['ldap']['main']['password'], auto_bind=True))
-		logger.debug('Setup done, iterating gitlab groups...')
-
-		gitlabSync(logger, gitlabApi, ldapConnection)
-
-		logger.info('Sync done')
-	except Exception as e:
-		logger.critical(e, exc_info=True)
-
-
-if __name__ == "__main__":
-	main()