diff --git a/src/core/management/commands/test_schedule_import.py b/src/core/management/commands/test_schedule_import.py new file mode 100644 index 0000000000000000000000000000000000000000..839f672d1bbb29069bbbb275b4e91ffc815e1486 --- /dev/null +++ b/src/core/management/commands/test_schedule_import.py @@ -0,0 +1,30 @@ +import argparse +import json + +from django.conf import settings +from django.core.exceptions import SuspiciousOperation +from django.core.management.base import BaseCommand +from django.db import transaction + +from ...models.conference import Conference + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--data-file', '-f', type=argparse.FileType('r'), help='the data file to load') + + def handle(self, *args, **options): + datafile = options.get('data_file') + data = json.load(datafile) + + conf = Conference.objects.get(pk=settings.SELECTED_CONFERENCE_ID) + try: + with transaction.atomic(): + assembly = conf.assemblies.create(slug='import') + source = conf.schedule_sources.create(assembly=assembly) + activity = source.load_data(data) + raise SuspiciousOperation('fail deliberately to rollback any possible changes') + except SuspiciousOperation: + pass + + print(json.dumps(activity, indent=2)) diff --git a/src/core/models/events.py b/src/core/models/events.py index a79c519c2c14b1cd5e6f3ced0f0d62140fde457d..b18f3374a65b8d8fde517c39263286ca276a491c 100644 --- a/src/core/models/events.py +++ b/src/core/models/events.py @@ -362,6 +362,7 @@ class Event(TaggedItemMixin, BackendMixin, models.Model): allow_kind: bool = False, allow_track: bool = False, room_lookup=None, + speaker_lookup=None, ): """ Loads an Event instance from the given dictionary. @@ -375,6 +376,7 @@ class Event(TaggedItemMixin, BackendMixin, models.Model): :param allow_kind: Flag indicating whether the 'kind' attribute may be set (e.g. to 'official'). :param allow_track: Flag indicating whether the 'track' attribute may be set. :param room_lookup: Lookup function to get the event's room from the data given, without this, a 'room' key will be ignored. + :param speaker_lookup: Lookup function to get a conference user object for the data given, without this, speakers will be ignored. :returns: a new event or the existing one with fields updated from the given data :rtype: Event @@ -451,6 +453,18 @@ class Event(TaggedItemMixin, BackendMixin, models.Model): if pop_used_keys: del data['room'] + if 'speakers' in data: + if speaker_lookup is not None: + obj.save() # TODO: check if we can do this better, but we need the model saved for adding speakers + for item in data['speakers']: + speaker = speaker_lookup(item) + if speaker is not None: + obj.ensure_speaker(speaker) + else: + raise RuntimeWarning('Event.from_dict() got data with "speakers" but no speaker_lookup was provided.') + if pop_used_keys: + del data['speakers'] + if pop_used_keys: obj.additional_data = data elif 'additional_data' in data: @@ -640,6 +654,28 @@ class Event(TaggedItemMixin, BackendMixin, models.Model): # we require a configured BigBlueButton integration return settings.INTEGRATIONS_BBB + def ensure_speaker(self, user: PlatformUser): + entry, created = self.participants.get_or_create(participant=user) + if created: + logger.info( + 'Added participant "%s" (%s) to event "%s" (%s), intended to be a speaker.', + user.username, + user.id, + self.name, + self.pk, + ) + + entry.role = EventParticipant.Role.SPEAKER + entry.is_public = True + entry.is_accepted = True + entry.save() + logger.info( + 'Forced participant #%s (%s) of event %s to accepted+public+speaker.', + entry.id, + user.username, + self.pk, + ) + def _EventAttachment_upload_path(instance, filename): # file will be uploaded to MEDIA_ROOT/user_<id>/<filename> diff --git a/src/core/models/schedules.py b/src/core/models/schedules.py index 80eddbb083d22fda04a817b3bc04d1f10f163abd..8dc1d8c4adb7f138f10247935a85772ba07b3f00 100644 --- a/src/core/models/schedules.py +++ b/src/core/models/schedules.py @@ -1,5 +1,7 @@ import logging from datetime import timedelta +from hashlib import sha1 +from typing import TYPE_CHECKING, Dict, List, Optional from uuid import UUID, uuid4 from django.core.exceptions import ObjectDoesNotExist @@ -9,10 +11,14 @@ from django.utils.translation import gettext_lazy as _ from ..fields import ConferenceReference from ..schedules import ScheduleTypeManager -from ..utils import mask_url, str2bool +from ..utils import mail2uuid, mask_url, str2bool from .assemblies import Assembly -from .events import Event, EventAttachment, EventParticipant +from .events import Event, EventAttachment from .rooms import Room +from .users import PlatformUser + +if TYPE_CHECKING: + pass logger = logging.getLogger(__name__) @@ -112,6 +118,61 @@ class ScheduleSource(models.Model): deadline = latest_import.start + self.import_timeout return timezone.now() < deadline + def _get_or_create_speaker( + self, + name: str, + mail_guid: Optional[str | UUID] = None, + addresses: Optional[List[str]] = None, + ): + if not name: + raise ValueError('You need to provide a name for the speaker.') + + if mail_guid is None and addresses is None: + raise ValueError('You need to provide at least mail_guid or addresses (better: both).') + + if mail_guid and not isinstance(mail_guid, UUID): + mail_guid = UUID(mail_guid) + + speaker_username = '_speaker_' + str(mail_guid or sha1('\n'.join(sorted(addresses)))) + + # try to find by the username + if candidate := PlatformUser.objects.filter(username=speaker_username, user_type=PlatformUser.Type.SPEAKER).first(): + return candidate, False + + # basically, this is a hail-mary-type attempt: we try to find a PlatformUser who + # has a verified email matching the given mail_guid or any of the supplied addresses + candidates = [] # type: List[PlatformUser] + for cm in self.conference.users.select_related('user').filter(user__user_type__in=PlatformUser.PERSON_TYPES).iterator(): # type: ConferenceMember + for addr in cm.user.get_verified_mail_addresses(): + if mail_guid and mail_guid == mail2uuid(addr): + candidates.append(cm.user) + break # exit the inner for-loop + if addresses and addr in addresses: + candidates.append(cm.user) + break # exit the inner for-loop + + # the good case: we found something \o/ + if len(candidates) == 1: + return candidates[0], False + + # the very bad case: we found too many + if len(candidates) > 1: + raise ValueError('Multiple candidate speakers found: ' + '; '.join(x.pk for x in candidates)) + + # the expected case: nothing found, create a new one + name_split = name.split(' ') + user_kwargs = { + 'user_type': PlatformUser.Type.SPEAKER, + 'username': speaker_username, + 'show_name': True, + 'last_name': name_split.pop(), + 'first_name': ' '.join(name_split), + } + new_user = PlatformUser(**user_kwargs) + new_user.save() + + return new_user, True + def get_or_create_mapping(self, mapping_type, source_id, create_local_object=True, source_uuid=None, hints: dict | None = None): """ Fetches the local object mapped by the given type and id. @@ -160,6 +221,13 @@ class ScheduleSource(models.Model): lo = Event(conference=self.conference, pk=source_uuid, assembly=assembly) + elif mapping_type == ScheduleSourceMapping.MappingType.SPEAKER: + lo, _ = self._get_or_create_speaker( + mail_guid=source_uuid, + name=hints.get('name'), + addresses=hints.get('addresses'), + ) + mapping = self.mappings.create( mapping_type=mapping_type, source_id=source_id, @@ -190,6 +258,10 @@ class ScheduleSource(models.Model): if item_type == 'event': hints['room_lookup'] = from_dict_args.get('room_lookup') hints['room_name'] = item.get('room') + hints['speaker_lookup'] = from_dict_args.get('speaker_lookup') + elif item_type == 'speaker': + hints['name'] = item.get('public_name') or item.get('name') + hints['addresses'] = item.get('addresses') mapping, new_mapping = self.get_or_create_mapping( mapping_type=item_type, @@ -331,6 +403,7 @@ class ScheduleSource(models.Model): activity = [] events = {} rooms = {} + speakers = {} # derive some flags cfg = self.import_configuration or {} @@ -338,7 +411,7 @@ class ScheduleSource(models.Model): replace_conference_slug_prefix = cfg.get('replace_conference_slug_prefix') allow_track = cfg.get('import_tracks') or False - # note down all existing rooms and events so that we can call out the missing ones + # note down all existing rooms, events and speakers so that we can call out the missing ones if self.assembly: expected_rooms = list(self.assembly.rooms.values_list('id', flat=True)) else: @@ -352,6 +425,70 @@ class ScheduleSource(models.Model): mapping_type=ScheduleSourceMapping.MappingType.EVENT, ).values_list('local_id', flat=True) ) + expected_speakers = list( + self.mappings.filter( + mapping_type=ScheduleSourceMapping.MappingType.SPEAKER, + ).values_list('local_id', flat=True) + ) + + def speaker_lookup(speaker_info: Dict[str, str]): + """ + Try to match the given speaker dict to a PlatformUser, if necessary creating a virtual one in the process. + Returns None if the speaker shall be skipped (explicitly, using ScheduleSourceMapping.skip=True). + + Example: + ```json + { + "id": 4711, + "guid": "c25334d0-9539-55e3-92b4-f559c384522b", + "name": "Hub Team", + "links": [ + { + "url": "https://git.cccv.de/hub/hub", + "title": "Quellcode" + } + ], + "biography": "Das Projekt-Team vom Hub, der Daten-Integrationsplattform von Congress & Camp.", + "avatar_url": "https://www.ccc.de/images/events.png", + "public_name": "Hub Team" + } + ``` + """ + + # sanity check: verify that required attributes are present + if any(x not in speaker_info for x in ['id', 'public_name']): + raise ValueError('Missing required attribute in speaker_info.') + + speaker_id = speaker_info.get('id') + try: + action = self._load_dataitem( + activity=activity, + item=speaker_info, + item_source_id=speaker_id, + item_type='speaker', + expected_items=expected_speakers, + items=speakers, + from_dict_args={}, + ) + + if action == 'skipped': + # if the speaker has been skipped throw it into the mapping table anyway + spk_mapping = self.mappings.get(mapping_type=ScheduleSourceMapping.MappingType.SPEAKER, source_id=speaker_id) + assert spk_mapping.skip + speakers[speaker_id] = spk_mapping.local_object + + except Exception as err: + activity.append( + { + 'action': 'error', + 'type': 'speaker', + 'source_id': speaker_id, + 'local_id': None, + 'message': str(err), + } + ) + + return speakers[speaker_id] # first, load the rooms (as they're needed for events) for r_id, r in data['rooms'].items(): @@ -405,6 +542,7 @@ class ScheduleSource(models.Model): 'allow_kind': self.assembly.is_official if self.assembly else False, # TODO: lookup assembly's room if not given 'allow_track': allow_track, # TODO 'room_lookup': lambda r_source_id: rooms.get(r_source_id), + 'speaker_lookup': speaker_lookup, }, ) except Exception as err: @@ -523,12 +661,10 @@ class ScheduleSourceMapping(models.Model): return attachment if self.mapping_type == self.MappingType.SPEAKER: - participant = EventParticipant.objects.prefetch_related('event').get(pk=self.local_id) - if self.schedule_source.assembly_id is not None and participant.event.assembly_id != self.schedule_source.assembly_id: - raise LocalObjectAccessViolation('Assembly of EventParticipant does not match.') - if participant.role != EventParticipant.Role.SPEAKER: - raise LocalObjectAccessViolation('Participant selected is not a speaker.') - return participant + speaker = PlatformUser.objects.get(pk=self.local_id) + if not speaker.is_person: + raise LocalObjectAccessViolation("Referenced speaker's PlatformUser is not a person.") + return speaker # we don't know about that mapping type, bail out raise LocalObjectAccessViolation('Unknown mapping.') diff --git a/src/core/models/users.py b/src/core/models/users.py index 1939e84ed6337a66f23add6623466be4e372aa6e..d1ee7f4a852dd24ccf14efa264477ef1464c6d8e 100644 --- a/src/core/models/users.py +++ b/src/core/models/users.py @@ -2,7 +2,7 @@ import logging import re from pathlib import Path from typing import TYPE_CHECKING, Any -from uuid import uuid4 +from uuid import UUID, uuid4 from timezone_field import TimeZoneField @@ -29,7 +29,7 @@ from ..utils import download_from_url from ..validators import ImageDimensionValidator if TYPE_CHECKING: - from core.models import ConferenceMember + from core.models import Assembly, Conference, ConferenceMember logger = logging.getLogger(__name__) @@ -332,6 +332,70 @@ class PlatformUser(AbstractUser): except ObjectDoesNotExist: return None + @classmethod + def from_dict( + cls, + data: dict, + conference: 'Conference', + assembly: 'Assembly' = None, + existing=None, + pop_used_keys: bool = False, + ): + """ + Loads an PlatformUser instance from the given dictionary - used by the Schedule import mechanism. + An existing user can be provided which's data is overwritten (in parts). + However, for sanity reasons, this will only operate on user_type=SPEAKER. + + :param data: a dictionary containing PlatformUser attributes' names as key (and their values) + :param conference: the Conference instance to which this PlatformUser shall be joined + :param assembly: ignored + :param existing: an instance of PlatformUser (or None to get a new one) + :type existing: PlatformUser + :param pop_used_keys: Remove 'used' keys from the provided data. This can be used to detect additional/errornous fields. + + :returns: a new user or the existing one with fields updated from the given data, in both cases joined to the conference + :rtype: PlatformUser + """ + assert isinstance(data, dict), 'Data must be a dictionary.' + if not existing: + raise NotImplementedError('Creating a PlatformUser with .from_dict() is not supported.') + if existing.user_type != PlatformUser.Type.SPEAKER: + raise NotImplementedError('Updating a PlatformUser which is not a SPEAKER is not supported (yet).') + + given_uuid = UUID(data.pop('guid')) if 'guid' in data else None + if existing: + obj = existing + else: + obj = cls() + if given_uuid is not None: + obj.pk = given_uuid + + # join the new user into the conference + cm, _ = conference.users.get_or_create(user=obj) + cm.description = data.get('description') + cm.save() + + # add all known addresses as (unverified and non-public) communication channels + for addr in data.get('addresses', []): + obj.communication_channels.get_or_create( + channel=UserCommunicationChannel.Channel.MAIL, + address=addr, + defaults={ + 'is_verified': False, + 'show_public': False, + }, + ) + + # load avatar, if an URL is given (and has not changed since last time) + if avatar_url := data.get('avatar_url'): + if obj.avatar_url != avatar_url: + try: + obj.load_avatar_from_url(avatar_url) + obj.avatar_url = avatar_url + except Exception: + # just log the error, if the speaker has no avatar (yet), we don't care too much + logger.exception('Failed to download avatar for (new) speaker user %s: %s', obj.pk, avatar_url) + def __create_slug(self, extension=''): """ recursive function to get a free slug based on the username and an optional extension string.