From b843aa0c4fdcc997dc74ff1a6ae633f139550866 Mon Sep 17 00:00:00 2001
From: Teal <git@teal.is>
Date: Tue, 4 Mar 2025 22:52:29 +0100
Subject: [PATCH] Improve conference and import management
When adding a new conference, we now automatically run the import job and let the user choose relevant stages. Users can also manage relevant stages from the conference edit view.
Added tracking of import histories. The app now knows when an import has been started and ended, and errors are logged into the DB as well and exposed to the frontend.
Added an interstitial page for fetching conference data, plus a way to manually trigger imports if no data was found.
---
.../admin/conferences_controller.rb | 46 +++++-
app/jobs/conference_import_job.rb | 25 ++++
app/jobs/fetch_conference_data_job.rb | 71 +++++++--
app/models/conference.rb | 67 +++++++++
app/models/import_history.rb | 10 ++
app/views/admin/conferences/edit.html.erb | 16 +-
.../admin/conferences/import_error.html.erb | 26 ++++
.../admin/conferences/import_history.html.erb | 138 ++++++++++++++++++
.../conferences/import_progress.html.erb | 67 +++++++++
.../select_relevant_stages.html.erb | 93 ++++++++++++
config/cronotab.rb | 11 +-
config/routes.rb | 12 +-
...ort_timestamps_and_error_to_conferences.rb | 7 +
.../20250304204807_create_import_histories.rb | 16 ++
db/schema.rb | 58 +++++++-
test/fixtures/import_histories.yml | 21 +++
test/models/import_history_test.rb | 7 +
17 files changed, 673 insertions(+), 18 deletions(-)
create mode 100644 app/jobs/conference_import_job.rb
create mode 100644 app/models/import_history.rb
create mode 100644 app/views/admin/conferences/import_error.html.erb
create mode 100644 app/views/admin/conferences/import_history.html.erb
create mode 100644 app/views/admin/conferences/import_progress.html.erb
create mode 100644 app/views/admin/conferences/select_relevant_stages.html.erb
create mode 100644 db/migrate/20250304204757_add_import_timestamps_and_error_to_conferences.rb
create mode 100644 db/migrate/20250304204807_create_import_histories.rb
create mode 100644 test/fixtures/import_histories.yml
create mode 100644 test/models/import_history_test.rb
diff --git a/app/controllers/admin/conferences_controller.rb b/app/controllers/admin/conferences_controller.rb
index 56adf5b..53ea681 100644
--- a/app/controllers/admin/conferences_controller.rb
+++ b/app/controllers/admin/conferences_controller.rb
@@ -2,7 +2,7 @@ module Admin
class ConferencesController < ApplicationController
before_action :authenticate_user!
before_action :authorize_permission
- before_action :set_conference, only: [ :edit, :update, :destroy ]
+ before_action :set_conference, except: [ :index, :new, :create ]
def index
@conferences = Conference.all
@@ -16,7 +16,8 @@ module Admin
@conference = Conference.new(conference_params)
if @conference.save
- redirect_to admin_conferences_path, notice: "Conference was successfully created."
+ FetchConferenceDataJob.perform_later(@conference.slug)
+ redirect_to import_progress_admin_conference_path(@conference), notice: "Conference was successfully created and data import has started."
else
render :new, status: :unprocessable_entity
end
@@ -38,6 +39,47 @@ module Admin
redirect_to admin_conferences_path, notice: "Conference was successfully deleted."
end
+ def import_progress
+ end
+
+ def import_status
+ render json: {
+ status: @conference.import_status,
+ started_at: @conference.last_import_started_at,
+ completed_at: @conference.last_import_completed_at,
+ error_summary: @conference.import_error_summary,
+ has_error: @conference.last_import_error.present?
+ }
+ end
+
+ def import_error
+ render layout: false if request.xhr?
+ end
+
+ def import_history
+ @import_histories = @conference.import_histories.order(created_at: :desc).limit(20)
+ end
+
+ def retry_import
+ FetchConferenceDataJob.perform_later(@conference.slug)
+ redirect_to import_progress_admin_conference_path(@conference), notice: "Data import has been started."
+ end
+
+ def select_relevant_stages
+ @stages = @conference.stages
+ end
+
+ def update_relevant_stages
+ @conference.relevant_stage_ids = params[:conference][:relevant_stage_ids]
+
+ if @conference.save
+ redirect_to admin_conferences_path, notice: "Relevant stages updated successfully."
+ else
+ @stages = @conference.stages
+ render :select_relevant_stages
+ end
+ end
+
private
def authorize_permission
diff --git a/app/jobs/conference_import_job.rb b/app/jobs/conference_import_job.rb
new file mode 100644
index 0000000..54f42ae
--- /dev/null
+++ b/app/jobs/conference_import_job.rb
@@ -0,0 +1,25 @@
+class ConferenceImportJob < ApplicationJob
+ queue_as :default
+
+ def perform(conference_slug)
+ conference = Conference.find_by(slug: conference_slug)
+ return unless conference
+
+ if conference.import_in_progress? && conference.last_import_started_at > 5.minutes.ago
+ Rails.logger.info "Skipping import for #{conference_slug} as another import is already in progress"
+ return
+ end
+
+ if conference.import_completed? && conference.last_import_completed_at > 5.minutes.ago
+ Rails.logger.info "Skipping import for #{conference_slug} as it was recently imported"
+ return
+ end
+
+ if conference.retry_count >= 3 && conference.last_import_started_at > 1.hour.ago
+ Rails.logger.info "Skipping import for #{conference_slug} due to too many recent failures"
+ return
+ end
+
+ FetchConferenceDataJob.perform_later(conference_slug)
+ end
+end
diff --git a/app/jobs/fetch_conference_data_job.rb b/app/jobs/fetch_conference_data_job.rb
index 402a055..b2eefb6 100644
--- a/app/jobs/fetch_conference_data_job.rb
+++ b/app/jobs/fetch_conference_data_job.rb
@@ -3,15 +3,68 @@ class FetchConferenceDataJob < ApplicationJob
def perform(conference_slug)
conference = Conference.find_by(slug: conference_slug)
- # Convert import job class to class path format
- job_class_path = "#{conference.import_job_class.camelize}"
- if Object.const_defined?(job_class_path)
- job_class = job_class_path.constantize
- job_class.perform_now(conference_slug)
- else
- Rails.logger.error "No custom jobs for conference #{conference_slug} found"
+
+ if conference.import_in_progress? && conference.last_import_started_at > 5.minutes.ago
+ Rails.logger.info "Skipping import for #{conference_slug} as another import is already in progress"
+ return
+ end
+
+ import_history = ImportHistory.create!(
+ conference: conference,
+ started_at: Time.current,
+ status: :started
+ )
+
+ conference.update(last_import_started_at: Time.current, last_import_error: nil)
+
+ begin
+ stages_before = conference.stages.count
+ sessions_before = conference.sessions.count
+ speakers_before = conference.speakers.count
+
+ job_class_path = "#{conference.import_job_class.camelize}"
+ if Object.const_defined?(job_class_path)
+ job_class = job_class_path.constantize
+ job_class.perform_now(conference_slug)
+ else
+ raise "No custom jobs for conference #{conference_slug} found"
+ end
+
+ stages_after = conference.stages.count
+ sessions_after = conference.sessions.count
+ speakers_after = conference.speakers.count
+
+ import_history.update(
+ completed_at: Time.current,
+ status: :completed,
+ stages_count: stages_after - stages_before,
+ sessions_count: sessions_after - sessions_before,
+ speakers_count: speakers_after - speakers_before
+ )
+
+ conference.update(last_import_completed_at: Time.current)
+ rescue => e
+ error_message = "#{e.class.name}: #{e.message}"
+ backtrace = e.backtrace.join("\n")
+ full_error = "#{error_message}\n\n#{backtrace}"
+
+ import_history.update(
+ status: :failed,
+ error: full_error
+ )
+
+ conference.update(last_import_error: full_error)
+
+ Rails.logger.error("Import failed for #{conference_slug}: #{error_message}")
+ Rails.logger.error(backtrace)
+
+ if conference.should_retry_import?
+ backoff_time = [ 30, 60, 120 ][conference.retry_count].seconds
+ FetchConferenceDataJob.set(wait: backoff_time).perform_later(conference_slug)
+ Rails.logger.info("Scheduled retry for #{conference_slug} in #{backoff_time} seconds")
+ end
+
+ raise e
end
- rescue StandardError => e
- Rails.logger.error "Failed to enqueue job for conference #{conference_slug}: #{e.message}"
end
end
diff --git a/app/models/conference.rb b/app/models/conference.rb
index 95a1914..ba65359 100644
--- a/app/models/conference.rb
+++ b/app/models/conference.rb
@@ -3,6 +3,7 @@ class Conference < ApplicationRecord
has_many :speakers
has_many :stages
has_many :revision_sets
+ has_many :import_histories
serialize :data, coder: JSON
@@ -11,6 +12,72 @@ class Conference < ApplicationRecord
has_many :relevant_stage_links, class_name: "RelevantStage"
has_many :relevant_stages, through: :relevant_stage_links, source: :stage
+ def import_in_progress?
+ last_import_started_at.present? &&
+ (last_import_completed_at.nil? || last_import_started_at > last_import_completed_at) &&
+ last_import_error.nil? &&
+ last_import_started_at > 30.minutes.ago
+ end
+
+ def import_completed?
+ last_import_completed_at.present? &&
+ (!last_import_started_at.present? || last_import_completed_at >= last_import_started_at)
+ end
+
+ def import_failed?
+ last_import_error.present? ||
+ (last_import_started_at.present? &&
+ last_import_completed_at.nil? &&
+ last_import_started_at < 30.minutes.ago)
+ end
+
+ def import_status
+ if import_in_progress?
+ "in_progress"
+ elsif import_failed?
+ "failed"
+ elsif import_completed?
+ "completed"
+ else
+ "pending"
+ end
+ end
+
+ def import_error_summary
+ return nil unless last_import_error.present?
+ last_import_error.split("\n").first
+ end
+
+ def should_retry_import?
+ return false unless import_failed?
+ return false if retry_count >= 3
+ return false if permanent_error?
+
+ last_retry = last_import_started_at || Time.at(0)
+ backoff_time = [ 30, 60, 120 ][retry_count] # 30s, 1m, 2m backoff
+ Time.current > (last_retry + backoff_time.minutes)
+ end
+
+ def permanent_error?
+ return false unless last_import_error.present?
+
+ permanent_error_patterns = [
+ /Invalid credentials/i,
+ /Access denied/i,
+ /Not found/i,
+ /Invalid URL/i,
+ /No such file or directory/i
+ ]
+
+ permanent_error_patterns.any? { |pattern| last_import_error.match?(pattern) }
+ end
+
+ def retry_count
+ import_histories.where("created_at > ?", 24.hours.ago)
+ .where(status: "failed")
+ .count
+ end
+
def days
(starts_at.to_date..ends_at.to_date)
end
diff --git a/app/models/import_history.rb b/app/models/import_history.rb
new file mode 100644
index 0000000..3989620
--- /dev/null
+++ b/app/models/import_history.rb
@@ -0,0 +1,10 @@
+class ImportHistory < ApplicationRecord
+ belongs_to :conference
+
+ enum :status, started: 0, completed: 1, failed: 2
+
+ def duration
+ return nil unless started_at
+ (completed_at || Time.current) - started_at
+ end
+end
diff --git a/app/views/admin/conferences/edit.html.erb b/app/views/admin/conferences/edit.html.erb
index 3b7dd6b..3f45d11 100644
--- a/app/views/admin/conferences/edit.html.erb
+++ b/app/views/admin/conferences/edit.html.erb
@@ -159,9 +159,21 @@
</div>
<div class="flex justify-between pt-6">
- <%= link_to "Cancel", admin_conferences_path, class: "px-4 py-2 text-sm font-medium text-gray-700 bg-white border border-gray-300 rounded-md shadow-sm hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 dark:bg-gray-700 dark:text-gray-200 dark:border-gray-600 dark:hover:bg-gray-600" %>
+ <div class="space-x-2">
+ <%= link_to "Cancel", admin_conferences_path, class: "px-4 py-2 text-sm font-medium text-gray-700 bg-white border border-gray-300 rounded-md shadow-sm hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 dark:bg-gray-700 dark:text-gray-200 dark:border-gray-600 dark:hover:bg-gray-600" %>
+
+ <% if @conference.persisted? %>
+ <%= link_to "View Import History", import_history_admin_conference_path(@conference), class: "px-4 py-2 text-sm font-medium text-indigo-700 bg-indigo-100 border border-transparent rounded-md shadow-sm hover:bg-indigo-200 focus:outline-none focus:ring-2 focus:ring-indigo-500 focus:ring-offset-2 dark:bg-indigo-900 dark:text-indigo-200 dark:hover:bg-indigo-800" %>
+ <% end %>
+ </div>
- <%= form.submit "Update Conference", class: "inline-flex justify-center px-4 py-2 text-sm font-medium text-white bg-blue-600 border border-transparent rounded-md shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 dark:bg-blue-700 dark:hover:bg-blue-800" %>
+ <div class="space-x-2">
+ <% if @conference.persisted? %>
+ <%= link_to "Manage Relevant Stages", select_relevant_stages_admin_conference_path(@conference), class: "px-4 py-2 text-sm font-medium text-green-700 bg-green-100 border border-transparent rounded-md shadow-sm hover:bg-green-200 focus:outline-none focus:ring-2 focus:ring-green-500 focus:ring-offset-2 dark:bg-green-900 dark:text-green-200 dark:hover:bg-green-800" %>
+ <% end %>
+
+ <%= form.submit "Update Conference", class: "inline-flex justify-center px-4 py-2 text-sm font-medium text-white bg-blue-600 border border-transparent rounded-md shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 dark:bg-blue-700 dark:hover:bg-blue-800" %>
+ </div>
</div>
<% end %>
diff --git a/app/views/admin/conferences/import_error.html.erb b/app/views/admin/conferences/import_error.html.erb
new file mode 100644
index 0000000..a5e02aa
--- /dev/null
+++ b/app/views/admin/conferences/import_error.html.erb
@@ -0,0 +1,26 @@
+<div class="bg-white rounded-lg shadow-xl max-w-4xl max-h-[80vh] overflow-auto dark:bg-gray-800">
+ <div class="flex justify-between items-center p-4 border-b dark:border-gray-700">
+ <h2 class="text-xl font-bold dark:text-white">Import Error Details</h2>
+ <button data-action="close-modal" class="text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200">
+ <svg xmlns="http://www.w3.org/2000/svg" class="h-6 w-6" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+ <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
+ </svg>
+ </button>
+ </div>
+
+ <div class="p-4">
+ <div class="mb-4">
+ <p class="text-sm text-gray-500 dark:text-gray-400">
+ Error occurred during import started at <%= @conference.last_import_started_at&.strftime("%Y-%m-%d %H:%M:%S") %>
+ </p>
+ </div>
+
+ <div class="bg-gray-100 p-4 rounded overflow-auto max-h-[60vh] font-mono text-sm dark:bg-gray-900 dark:text-gray-300">
+ <pre><%= @conference.last_import_error %></pre>
+ </div>
+
+ <div class="mt-6 flex justify-end">
+ <%= button_to "Retry Import", retry_import_admin_conference_path(@conference), method: :post, class: "px-4 py-2 bg-blue-600 text-white rounded hover:bg-blue-700 dark:bg-blue-700 dark:hover:bg-blue-800" %>
+ </div>
+ </div>
+</div>
diff --git a/app/views/admin/conferences/import_history.html.erb b/app/views/admin/conferences/import_history.html.erb
new file mode 100644
index 0000000..c481d63
--- /dev/null
+++ b/app/views/admin/conferences/import_history.html.erb
@@ -0,0 +1,138 @@
+<div class="container mx-auto px-4 py-8">
+ <h1 class="text-2xl font-bold mb-6">Import History for <%= @conference.name %></h1>
+
+ <div class="mb-6">
+ <%= link_to "Back to Conference", edit_admin_conference_path(@conference), class: "text-blue-600 hover:text-blue-800 dark:text-blue-400 dark:hover:text-blue-300" %>
+ <span class="mx-2">|</span>
+ <%= button_to "Run Import Now", retry_import_admin_conference_path(@conference), method: :post, class: "inline-flex items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 dark:bg-blue-700 dark:hover:bg-blue-800" %>
+ </div>
+
+ <div class="bg-white shadow overflow-hidden sm:rounded-lg dark:bg-gray-800">
+ <table class="min-w-full divide-y divide-gray-200 dark:divide-gray-700">
+ <thead class="bg-gray-50 dark:bg-gray-700">
+ <tr>
+ <th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider dark:text-gray-300">
+ Date
+ </th>
+ <th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider dark:text-gray-300">
+ Status
+ </th>
+ <th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider dark:text-gray-300">
+ Duration
+ </th>
+ <th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider dark:text-gray-300">
+ Changes
+ </th>
+ <th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider dark:text-gray-300">
+ Actions
+ </th>
+ </tr>
+ </thead>
+ <tbody class="bg-white divide-y divide-gray-200 dark:bg-gray-800 dark:divide-gray-700">
+ <% @import_histories.each do |history| %>
+ <tr>
+ <td class="px-6 py-4 whitespace-nowrap text-sm text-gray-500 dark:text-gray-400">
+ <%= history.created_at.strftime("%Y-%m-%d %H:%M:%S") %>
+ </td>
+ <td class="px-6 py-4 whitespace-nowrap">
+ <% if history.status == 'completed' %>
+ <span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200">
+ Completed
+ </span>
+ <% elsif history.status == 'failed' %>
+ <span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200">
+ Failed
+ </span>
+ <% else %>
+ <span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200">
+ In Progress
+ </span>
+ <% end %>
+ </td>
+ <td class="px-6 py-4 whitespace-nowrap text-sm text-gray-500 dark:text-gray-400">
+ <% if history.duration %>
+ <%= number_to_human(history.duration, units: {unit: "s", thousand: "k"}) %>
+ <% else %>
+ -
+ <% end %>
+ </td>
+ <td class="px-6 py-4 whitespace-nowrap text-sm text-gray-500 dark:text-gray-400">
+ <% if history.completed? %>
+ <span class="text-xs">
+ <%= pluralize(history.stages_count, "stage") %>,
+ <%= pluralize(history.sessions_count, "session") %>,
+ <%= pluralize(history.speakers_count, "speaker") %>
+ </span>
+ <% else %>
+ -
+ <% end %>
+ </td>
+ <td class="px-6 py-4 whitespace-nowrap text-sm font-medium">
+ <% if history.error.present? %>
+ <button class="text-red-600 hover:text-red-900 dark:text-red-400 dark:hover:text-red-300"
+ data-action="view-error"
+ data-error="<%= h(history.error) %>">
+ View Error
+ </button>
+ <% end %>
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+ </table>
+ </div>
+
+ <% if @import_histories.empty? %>
+ <div class="text-center py-8 text-gray-500 dark:text-gray-400">
+ No import history found for this conference.
+ </div>
+ <% end %>
+
+ <template id="error-modal-template">
+ <div class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50">
+ <div class="bg-white rounded-lg shadow-xl max-w-4xl max-h-[80vh] overflow-auto dark:bg-gray-800">
+ <div class="flex justify-between items-center p-4 border-b dark:border-gray-700">
+ <h2 class="text-xl font-bold dark:text-white">Import Error Details</h2>
+ <button data-action="close-modal" class="text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200">
+ <svg xmlns="http://www.w3.org/2000/svg" class="h-6 w-6" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+ <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
+ </svg>
+ </button>
+ </div>
+
+ <div class="p-4">
+ <div class="bg-gray-100 p-4 rounded overflow-auto max-h-[60vh] font-mono text-sm dark:bg-gray-900 dark:text-gray-300">
+ <pre id="error-content"></pre>
+ </div>
+
+ <div class="mt-6 flex justify-end">
+ <button data-action="close-modal" class="px-4 py-2 bg-gray-200 text-gray-700 rounded hover:bg-gray-300 dark:bg-gray-700 dark:text-gray-200 dark:hover:bg-gray-600">
+ Close
+ </button>
+ </div>
+ </div>
+ </div>
+ </div>
+ </template>
+
+ <script>
+ document.addEventListener('DOMContentLoaded', function() {
+ document.querySelectorAll('[data-action="view-error"]').forEach(function(button) {
+ button.addEventListener('click', function() {
+ const errorContent = this.getAttribute('data-error');
+ const template = document.getElementById('error-modal-template');
+ const modal = template.content.cloneNode(true);
+
+ modal.querySelector('#error-content').textContent = errorContent;
+ document.body.appendChild(modal);
+
+ document.querySelectorAll('[data-action="close-modal"]').forEach(function(closeButton) {
+ closeButton.addEventListener('click', function() {
+ document.querySelector('.fixed.inset-0').remove();
+ });
+ });
+ });
+ });
+ });
+ </script>
+</div>
diff --git a/app/views/admin/conferences/import_progress.html.erb b/app/views/admin/conferences/import_progress.html.erb
new file mode 100644
index 0000000..7b9a9dd
--- /dev/null
+++ b/app/views/admin/conferences/import_progress.html.erb
@@ -0,0 +1,67 @@
+<div class="container mx-auto px-4 py-8">
+ <h1 class="text-2xl font-bold mb-6">Importing Conference Data</h1>
+
+ <div id="import-status" data-conference-slug="<%= @conference.slug %>">
+ <div class="flex items-center">
+ <svg class="animate-spin h-5 w-5 mr-3 text-blue-500" viewBox="0 0 24 24">
+ <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
+ <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
+ </svg>
+ <p>Importing data for <%= @conference.name %>. This may take a few minutes...</p>
+ <% if @conference.last_import_started_at %>
+ <p class="text-sm text-gray-500 ml-2">
+ Started at <%= @conference.last_import_started_at.strftime("%H:%M:%S") %>
+ </p>
+ <% end %>
+ </div>
+ </div>
+
+ <script>
+ function checkImportStatus() {
+ const slug = document.getElementById('import-status').dataset.conferenceSlug;
+
+ fetch(`/admin/conferences/${slug}/import_status`)
+ .then(response => response.json())
+ .then(data => {
+ if (data.status === 'completed') {
+ window.location.href = `/admin/conferences/${slug}/select_relevant_stages`;
+ } else if (data.status === 'failed') {
+ let errorHtml = '<div class="bg-red-100 border-l-4 border-red-500 text-red-700 p-4 mb-6">' +
+ '<p class="font-bold">Import failed</p>';
+
+ if (data.error_summary) {
+ errorHtml += `<p class="mt-2">${data.error_summary}</p>`;
+ }
+
+ errorHtml += '<div class="mt-4 flex space-x-4">' +
+ `<a href="/admin/conferences/${slug}/retry_import" data-method="post" class="px-4 py-2 bg-red-600 text-white rounded hover:bg-red-700">Retry Import</a>` +
+ `<a href="/admin/conferences/${slug}/import_error" class="px-4 py-2 border border-red-600 text-red-600 rounded hover:bg-red-50" data-action="view-error">View Error Details</a>` +
+ '</div></div>';
+
+ document.getElementById('import-status').innerHTML = errorHtml;
+
+ document.querySelector('[data-action="view-error"]').addEventListener('click', function(e) {
+ e.preventDefault();
+
+ fetch(this.href)
+ .then(response => response.text())
+ .then(html => {
+ const modal = document.createElement('div');
+ modal.className = 'fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50';
+ modal.innerHTML = html;
+ document.body.appendChild(modal);
+
+ modal.querySelector('[data-action="close-modal"]').addEventListener('click', function() {
+ document.body.removeChild(modal);
+ });
+ });
+ });
+ } else {
+ setTimeout(checkImportStatus, 2000);
+ }
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', checkImportStatus);
+ </script>
+</div>
diff --git a/app/views/admin/conferences/select_relevant_stages.html.erb b/app/views/admin/conferences/select_relevant_stages.html.erb
new file mode 100644
index 0000000..3eb8c3c
--- /dev/null
+++ b/app/views/admin/conferences/select_relevant_stages.html.erb
@@ -0,0 +1,93 @@
+<div class="container mx-auto px-4 py-8">
+ <h1 class="text-2xl font-bold mb-6">Select Relevant Stages for <%= @conference.name %></h1>
+
+ <% if @conference.import_in_progress? %>
+ <div class="bg-blue-100 border-l-4 border-blue-500 text-blue-700 p-4 mb-6">
+ <p>Data import is still in progress. The stage list will be available once the import completes.</p>
+ <div class="flex items-center mt-2">
+ <svg class="animate-spin h-5 w-5 mr-3 text-blue-500" viewBox="0 0 24 24">
+ <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
+ <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
+ </svg>
+ <p>Importing... Started at <%= @conference.last_import_started_at.strftime("%H:%M:%S") %></p>
+ </div>
+ </div>
+ <% elsif @conference.import_failed? %>
+ <div class="bg-red-100 border-l-4 border-red-500 text-red-700 p-4 mb-6">
+ <p class="font-bold">Data import failed</p>
+
+ <% if @conference.import_error_summary %>
+ <p class="mt-2"><%= @conference.import_error_summary %></p>
+ <% end %>
+
+ <div class="mt-4 flex space-x-4">
+ <%= button_to "Retry Import", retry_import_admin_conference_path(@conference), method: :post, class: "px-4 py-2 bg-red-600 text-white rounded hover:bg-red-700" %>
+ <%= link_to "View Error Details", import_error_admin_conference_path(@conference), class: "px-4 py-2 border border-red-600 text-red-600 rounded hover:bg-red-50", data: { action: "view-error" } %>
+ </div>
+ </div>
+
+ <script>
+ document.querySelector('[data-action="view-error"]').addEventListener('click', function(e) {
+ e.preventDefault();
+
+ fetch(this.href)
+ .then(response => response.text())
+ .then(html => {
+ const modal = document.createElement('div');
+ modal.className = 'fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50';
+ modal.innerHTML = html;
+ document.body.appendChild(modal);
+
+ modal.querySelector('[data-action="close-modal"]').addEventListener('click', function() {
+ document.body.removeChild(modal);
+ });
+ });
+ });
+ </script>
+ <% elsif @stages.empty? %>
+ <div class="bg-yellow-100 border-l-4 border-yellow-500 text-yellow-700 p-4 mb-6">
+ <p>No stages found for this conference. This could be because:</p>
+ <ul class="list-disc ml-5 mt-2">
+ <li>The import hasn't been run yet</li>
+ <li>The import completed but no stages were found</li>
+ <li>There was an issue with the import</li>
+ </ul>
+ <% if @conference.last_import_completed_at %>
+ <p class="mt-2">Last import completed at <%= @conference.last_import_completed_at.strftime("%Y-%m-%d %H:%M:%S") %></p>
+ <% end %>
+ <%= button_to "Run Import", retry_import_admin_conference_path(@conference), method: :post, class: "mt-2 bg-yellow-500 hover:bg-yellow-700 text-white font-bold py-2 px-4 rounded" %>
+ </div>
+ <% else %>
+ <div class="flex justify-between items-center mb-6">
+ <div class="text-sm text-gray-500">
+ <% if @conference.last_import_completed_at %>
+ Last import completed at <%= @conference.last_import_completed_at.strftime("%Y-%m-%d %H:%M:%S") %>
+ <% end %>
+ </div>
+
+ <%= button_to "Run Import Again", retry_import_admin_conference_path(@conference), method: :post, class: "px-4 py-2 bg-gray-200 text-gray-700 rounded hover:bg-gray-300 dark:bg-gray-700 dark:text-gray-200 dark:hover:bg-gray-600" %>
+ </div>
+
+ <%= form_with(model: [:admin, @conference], url: update_relevant_stages_admin_conference_path(@conference), method: :patch, class: "space-y-6") do |form| %>
+ <div class="bg-white shadow overflow-hidden sm:rounded-md dark:bg-gray-800">
+ <ul class="divide-y divide-gray-200 dark:divide-gray-700">
+ <% @stages.each do |stage| %>
+ <li class="px-4 py-4 sm:px-6">
+ <div class="flex items-center">
+ <%= check_box_tag "conference[relevant_stage_ids][]", stage.id, @conference.relevant_stages.include?(stage), id: "stage_#{stage.id}", class: "h-4 w-4 text-blue-600 focus:ring-blue-500 border-gray-300 rounded" %>
+ <label for="stage_<%= stage.id %>" class="ml-3 block text-sm font-medium text-gray-700 dark:text-gray-300">
+ <%= stage.name %>
+ </label>
+ </div>
+ </li>
+ <% end %>
+ </ul>
+ </div>
+
+ <div class="flex justify-between pt-6">
+ <%= link_to "Back to Conferences", admin_conferences_path, class: "px-4 py-2 text-sm font-medium text-gray-700 bg-white border border-gray-300 rounded-md shadow-sm hover:bg-gray-50 dark:bg-gray-700 dark:text-gray-200 dark:border-gray-600 dark:hover:bg-gray-600" %>
+ <%= form.submit "Save Relevant Stages", class: "inline-flex justify-center px-4 py-2 text-sm font-medium text-white bg-blue-600 border border-transparent rounded-md shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 dark:bg-blue-700 dark:hover:bg-blue-800" %>
+ </div>
+ <% end %>
+ <% end %>
+</div>
diff --git a/config/cronotab.rb b/config/cronotab.rb
index 6185317..ee7474b 100644
--- a/config/cronotab.rb
+++ b/config/cronotab.rb
@@ -14,6 +14,13 @@
# Crono.perform(TestJob).every 2.days, at: '15:30'
#
-Crono.perform(FetchConferenceDataJob, "38c3").every 5.minutes
-Crono.perform(FetchConferenceDataJob, "38c3-more").every 5.minutes
+# Dynamic conference imports
+Conference.where.not(import_job_class: nil).pluck(:slug).each do |slug|
+ Crono.perform(ConferenceImportJob, slug).every 5.minutes
+end
+
+# Uncomment to manually configure specific conferences
+# Crono.perform(FetchConferenceDataJob, "38c3").every 5.minutes
+# Crono.perform(FetchConferenceDataJob, "38c3-more").every 5.minutes
+
# Crono.perform(TelegramNotifyUpcomingJob, { offset: 15.minutes.to_i, interval: 1.minute.to_i }).every 1.minute
diff --git a/config/routes.rb b/config/routes.rb
index 561cc66..9b1c73d 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,7 +1,17 @@
Rails.application.routes.draw do
namespace :admin do
resources :roles, only: [ :index, :edit, :update ]
- resources :conferences, param: :slug
+ resources :conferences, param: :slug do
+ member do
+ get :import_progress
+ get :import_status
+ get :import_error
+ get :import_history
+ post :retry_import
+ get :select_relevant_stages
+ patch :update_relevant_stages
+ end
+ end
end
devise_for :users
mount Crono::Engine, at: "/crono"
diff --git a/db/migrate/20250304204757_add_import_timestamps_and_error_to_conferences.rb b/db/migrate/20250304204757_add_import_timestamps_and_error_to_conferences.rb
new file mode 100644
index 0000000..fe06d16
--- /dev/null
+++ b/db/migrate/20250304204757_add_import_timestamps_and_error_to_conferences.rb
@@ -0,0 +1,7 @@
+class AddImportTimestampsAndErrorToConferences < ActiveRecord::Migration[8.0]
+ def change
+ add_column :conferences, :last_import_started_at, :datetime
+ add_column :conferences, :last_import_completed_at, :datetime
+ add_column :conferences, :last_import_error, :text
+ end
+end
diff --git a/db/migrate/20250304204807_create_import_histories.rb b/db/migrate/20250304204807_create_import_histories.rb
new file mode 100644
index 0000000..e5a2793
--- /dev/null
+++ b/db/migrate/20250304204807_create_import_histories.rb
@@ -0,0 +1,16 @@
+class CreateImportHistories < ActiveRecord::Migration[8.0]
+ def change
+ create_table :import_histories do |t|
+ t.references :conference, null: false, foreign_key: true
+ t.datetime :started_at
+ t.datetime :completed_at
+ t.string :status
+ t.text :error
+ t.integer :stages_count
+ t.integer :sessions_count
+ t.integer :speakers_count
+
+ t.timestamps
+ end
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index d7b3f45..404bf0e 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[8.0].define(version: 2025_01_02_175732) do
+ActiveRecord::Schema[8.0].define(version: 2025_03_04_204807) do
create_table "assignments", force: :cascade do |t|
t.integer "user_id", null: false
t.integer "session_id", null: false
@@ -44,6 +44,9 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_02_175732) do
t.string "import_job_class"
t.string "time_zone"
t.boolean "more_languages", default: false, null: false
+ t.datetime "last_import_started_at"
+ t.datetime "last_import_completed_at"
+ t.text "last_import_error"
end
create_table "crono_jobs", force: :cascade do |t|
@@ -76,6 +79,20 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_02_175732) do
t.index ["session_id"], name: "index_filedrop_files_on_session_id"
end
+ create_table "import_histories", force: :cascade do |t|
+ t.integer "conference_id", null: false
+ t.datetime "started_at"
+ t.datetime "completed_at"
+ t.string "status"
+ t.text "error"
+ t.integer "stages_count"
+ t.integer "sessions_count"
+ t.integer "speakers_count"
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["conference_id"], name: "index_import_histories_on_conference_id"
+ end
+
create_table "model_versions", force: :cascade do |t|
t.string "model"
t.text "changed_data"
@@ -103,6 +120,13 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_02_175732) do
t.datetime "updated_at", null: false
end
+ create_table "permissions", force: :cascade do |t|
+ t.string "name"
+ t.text "description"
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ end
+
create_table "relevant_stages", force: :cascade do |t|
t.integer "conference_id", null: false
t.integer "stage_id", null: false
@@ -132,6 +156,22 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_02_175732) do
t.index ["conference_id"], name: "index_revisions_on_conference_id"
end
+ create_table "role_permissions", force: :cascade do |t|
+ t.integer "role_id", null: false
+ t.integer "permission_id", null: false
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["permission_id"], name: "index_role_permissions_on_permission_id"
+ t.index ["role_id"], name: "index_role_permissions_on_role_id"
+ end
+
+ create_table "roles", force: :cascade do |t|
+ t.string "name"
+ t.text "description"
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ end
+
create_table "session_speakers", force: :cascade do |t|
t.integer "session_id", null: false
t.integer "speaker_id", null: false
@@ -312,6 +352,15 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_02_175732) do
t.index ["ref_id", "conference_id"], name: "index_stages_on_ref_id_and_conference_id", unique: true
end
+ create_table "user_roles", force: :cascade do |t|
+ t.integer "user_id", null: false
+ t.integer "role_id", null: false
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["role_id"], name: "index_user_roles_on_role_id"
+ t.index ["user_id"], name: "index_user_roles_on_user_id"
+ end
+
create_table "users", force: :cascade do |t|
t.string "name"
t.string "email"
@@ -321,11 +370,11 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_02_175732) do
t.string "telegram_username"
t.string "encrypted_password", default: "", null: false
t.datetime "remember_created_at"
- t.boolean "shiftcoordinator", default: false, null: false
t.string "invitation_token"
t.string "languages_from"
t.string "languages_to"
t.integer "darkmode", default: 0, null: false
+ t.boolean "migrated_to_rbac", default: false
end
add_foreign_key "assignments", "sessions"
@@ -334,10 +383,13 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_02_175732) do
add_foreign_key "candidates", "users"
add_foreign_key "filedrop_comments", "sessions"
add_foreign_key "filedrop_files", "sessions"
+ add_foreign_key "import_histories", "conferences"
add_foreign_key "relevant_stages", "conferences"
add_foreign_key "relevant_stages", "stages"
add_foreign_key "revision_sets", "conferences"
add_foreign_key "revisions", "conferences"
+ add_foreign_key "role_permissions", "permissions"
+ add_foreign_key "role_permissions", "roles"
add_foreign_key "session_speakers", "sessions"
add_foreign_key "session_speakers", "speakers"
add_foreign_key "sessions", "conferences"
@@ -350,4 +402,6 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_02_175732) do
add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
add_foreign_key "speakers", "conferences"
add_foreign_key "stages", "conferences"
+ add_foreign_key "user_roles", "roles"
+ add_foreign_key "user_roles", "users"
end
diff --git a/test/fixtures/import_histories.yml b/test/fixtures/import_histories.yml
new file mode 100644
index 0000000..b10b470
--- /dev/null
+++ b/test/fixtures/import_histories.yml
@@ -0,0 +1,21 @@
+# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+one:
+ conference: one
+ started_at: 2025-03-04 21:48:07
+ completed_at: 2025-03-04 21:48:07
+ status: MyString
+ error: MyText
+ stages_count: 1
+ sessions_count: 1
+ speakers_count: 1
+
+two:
+ conference: two
+ started_at: 2025-03-04 21:48:07
+ completed_at: 2025-03-04 21:48:07
+ status: MyString
+ error: MyText
+ stages_count: 1
+ sessions_count: 1
+ speakers_count: 1
diff --git a/test/models/import_history_test.rb b/test/models/import_history_test.rb
new file mode 100644
index 0000000..6589a3b
--- /dev/null
+++ b/test/models/import_history_test.rb
@@ -0,0 +1,7 @@
+require "test_helper"
+
+class ImportHistoryTest < ActiveSupport::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
--
GitLab