diff --git a/app/jobs/pretalx/import_job.rb b/app/jobs/pretalx/import_job.rb
index 234170bd515db70e0439dfb86bee23b8ae91976a..268289bfaddf98d906a965f39ea5c235f5bd7e31 100644
--- a/app/jobs/pretalx/import_job.rb
+++ b/app/jobs/pretalx/import_job.rb
@@ -8,7 +8,7 @@ module Pretalx
queue_as :default
include ActionView::Helpers
- def import_schedule(conference, url)
+ def import_schedule(conference, url, filedrop_config)
response = HTTParty.get(url)
response.success? or return Rails.logger.error "Failed to fetch schedule from #{url}"
@@ -17,6 +17,8 @@ module Pretalx
schedule.dig('schedule', 'conference', 'rooms') &&
schedule.dig('schedule', 'conference', 'days')
+ filedrop_index = fetch_filedrop_index(filedrop_config)
+
# We keep a local hash of the stages, because the sessions reference stages by name instead of id
stages = {}
schedule['schedule']['conference']['rooms'].each do |stage_data|
@@ -60,6 +62,7 @@ module Pretalx
end
end
session.recorded = !session_data.fetch('do_not_record', false)
+ update_filedrop_data(session, filedrop_config) if filedrop_index[session.ref_id]
session.save!
end
end
@@ -74,8 +77,98 @@ module Pretalx
def perform(conference_slug, *args)
conference = Conference.find_by(slug: conference_slug)
- import_schedule(conference, conference.data['schedule_url'])
+ import_schedule(conference, conference.data['schedule_url'], conference.data['filedrop'])
RevisionSet.create!(conference:)
end
+
+ private
+
+ def fetch_filedrop_index(filedrop_config)
+ if !filedrop_config || !filedrop_config['url']
+ return {}
+ end
+
+ begin
+ response = HTTParty.get(
+ filedrop_config['url'] + "/",
+ basic_auth: {
+ username: fetch_credential("filedrop_user"),
+ password: fetch_credential("filedrop_password") },
+ headers: { 'Accept' => 'application/json' },
+ timeout: 5
+ )
+ data = JSON.parse(response.body)
+ rescue => e
+ Rails.logger.warn("Filedrop response for #{session.ref_id} failed: #{e.message}")
+ return {}
+ end
+ if !data["talks"].is_a?(Array)
+ Rails.logger.warn("Filedrop index was incomplete")
+ return {}
+ end
+
+ return data["talks"].each_with_object({}) do |item, hash|
+ hash[item["id"]] = item
+ end
+ end
+
+ def update_filedrop_data(session, filedrop_config)
+ if !filedrop_config || !filedrop_config['url']
+ return {}
+ end
+
+ response = HTTParty.get(
+ filedrop_config['url'] + "/talks/" + session.ref_id,
+ basic_auth: {
+ username: fetch_credential("filedrop_user"),
+ password: fetch_credential("filedrop_password") },
+ headers: { 'Accept' => 'application/json' }
+ )
+
+ begin
+ data = JSON.parse(response.body)
+ rescue => e
+ Rails.logger.warn("Filedrop response could not be parsed: #{e.message}")
+ return {}
+ end
+ if !data["files"].is_a?(Array) || !data["comments"].is_a?(Array)
+ Rails.logger.warn("Filedrop info for #{session.ref_id} was incomplete")
+ return {}
+ end
+
+ existing_comments = session.filedrop_comments.pluck(:body)
+ new_comments = data["comments"]&.pluck("body") || []
+
+ # Remove comments not in the JSON file
+ (existing_comments - new_comments).each do |body|
+ session.filedrop_comments.where(body: body).destroy_all
+ end
+
+ # Add or update comments
+ data["comments"]&.each do |comment_data|
+ session.filedrop_comments.find_or_initialize_by(body: comment_data['body']).tap do |comment|
+ comment.orig_created = comment_data['created']
+ comment.save!
+ end
+ end
+
+ existing_files = session.filedrop_files.pluck(:name, :checksum)
+ new_files = data['filedrop_files']&.pluck('name', 'hash') || []
+
+ # Remove files not in the JSON file
+ (existing_files - new_files).each do |name, checksum|
+ session.filedrop_files.where(name: name, checksum: checksum).destroy_all
+ end
+
+ # Add or update files
+ data['files']&.each do |file_data|
+ session.filedrop_files.find_or_initialize_by(name: file_data['name'], checksum: file_data['meta']['hash']).tap do |file|
+ file.size = file_data['meta']['size']
+ file.orig_created = file_data['meta']['created']
+ file.download(filedrop_config['url'] + file_data['url']) unless filedrop_config.fetch('skip_downloads', false)
+ file.save
+ end
+ end
+ end
end
end
diff --git a/app/models/filedrop_comment.rb b/app/models/filedrop_comment.rb
new file mode 100644
index 0000000000000000000000000000000000000000..dbb244b6b74052b8b40439b2968b41a7975a8d30
--- /dev/null
+++ b/app/models/filedrop_comment.rb
@@ -0,0 +1,3 @@
+class FiledropComment < ApplicationRecord
+ belongs_to :session
+end
diff --git a/app/models/filedrop_file.rb b/app/models/filedrop_file.rb
new file mode 100644
index 0000000000000000000000000000000000000000..39f72ba6246ae5507bdfd1de72e11154488d1803
--- /dev/null
+++ b/app/models/filedrop_file.rb
@@ -0,0 +1,41 @@
+class FiledropFile < ApplicationRecord
+ belongs_to :session
+
+ def sanitize_filename(filename)
+ filename.gsub(/[^\w\s.-]/, '_')
+ end
+
+ def safe_download_path(download_dir, filename)
+ sanitized_filename = sanitize_filename(filename)
+ output_path = File.join(download_dir, sanitized_filename)
+ if File.expand_path(output_path).start_with?(File.expand_path(download_dir))
+ output_path
+ else
+ raise "Invalid filename, potential directory traversal detected!"
+ end
+ end
+
+ def download(url)
+ # TBD: only download if orig_created is newer than actual file change time
+ response = HTTParty.get(url)
+ if response.success?
+ File.open(local_path, 'wb') do |file|
+ file.write(response.body)
+ end
+ Rails.logger.debug("File downloaded successfully and saved as #{local_path}.")
+ else
+ Rails.logger.warn("Failed to download file #{name} for #{session.ref_id}: #{response.code} #{response.message}")
+ end
+ end
+
+ def local_path
+ dir = File.join(
+ ActiveStorage::Blob.service.root,
+ "filedrop",
+ session.conference.slug,
+ session.ref_id
+ )
+ FileUtils.mkdir_p(dir)
+ return File.join(dir, name)
+ end
+end
diff --git a/app/models/session.rb b/app/models/session.rb
index 28e5bbf5ef4465881b47f0cac2deb180ef976ebd..86ca06a1537296757777eb0ec0271fb27623b694 100644
--- a/app/models/session.rb
+++ b/app/models/session.rb
@@ -6,6 +6,8 @@ class Session < ApplicationRecord
has_many :candidates
has_many :session_speakers, dependent: :destroy
has_many :speakers, through: :session_speakers
+ has_many :filedrop_comments, dependent: :destroy
+ has_many :filedrop_files, dependent: :destroy
scope :scheduled, -> { where(status: 'scheduled') }
scope :future, -> { where(starts_at: Time.now..) }
diff --git a/app/models/user.rb b/app/models/user.rb
index b96957d3c7f6b789bffaf58d63b804ce43d27bbe..aa06e39cb5cd512b3601619c98cc73fb57c99336 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -25,7 +25,7 @@ class User < ApplicationRecord
if (login = conditions.delete(:name))
where(conditions.to_h).where(["lower(name) = :value", { value: login.downcase }]).first
else
- logger.warn("Authentication did not query :name as expected, login will only work with exact case!")
+ Rails.logger.warn("Authentication did not query :name as expected, login will only work with exact case!")
where(conditions.to_h).first
end
end
diff --git a/config/credentials.yml.enc b/config/credentials.yml.enc
index 7783f40d7b9aa875142fb9d4286784381d000513..229de6f89acd276d8c3693065c99d60583efe8bb 100644
--- a/config/credentials.yml.enc
+++ b/config/credentials.yml.enc
@@ -1 +1 @@
-mk0Oxq2UirmjsN7tckn4aXG2nU0JQQpa9AhwJk4Brq+9F7r4991T5RF8r5y54PEx8EgfOhN6eLHW6j6rQv6pNn86IUT/Awoc1771fWX1k/RRC3V1bOKNdfs+z5XNMSVH3ElpNs77cHz2i0ASSzerooNodAlnTWh5IJY2wqmg/3o84ndPXk61JuC96YNdJKOyZTsYNGlhLWCdwM0R+ehyiI2V92jULwuv805ml7kv58DYjAuJ0Zl5hO0gDc8+xe49VHXZahZv8lXriouc2E57IK4+YkByu8alrfrSlnqTvcZQ5/PwdeSG2hcyRH7PBCJM/nRIkWMiMqTqANlYPhn2PYSa5Iuei5nHVD1eo8fwm90KPAN2eJ20o4mrLb3GgkokCyDvWTi4Z0FL5RvJx/AmBEeiSVvR/9/XD/FaCjjJTwF36I4Q11JZbB81rkmVqPZTdW8thatGDQq+f7pOJHtXTFFbDWvH6rj6MpEgHw7EBpdKO2ypzlCnMw==--Do4Eq2X5bdeAXmaK--y66xcHPgudqA58YaQNaZhQ==
\ No newline at end of file
+yyticap6zoGwsCHAopvKcSo3S00K8NrYsUdsKi4DT70yAL0/4AfxxhgTGX1y6sxKryjo93305AXDPr0r5SaSgaAIPMhLblHLamiYlVVhvLB7r+LrGm3gkSDRKeJd6xKXMC9FxjzrodSxxAmG1yz+c3geOpCfSTRH1znkb75Dc11NRJ7w48wNRjBbYKmvmwnaWsyW5QJFcKiGU7QzFHiyyenGujRTO/gc3nm/NQShd1dY9Y3idpg1PA68iaaVQrx5RrNq+KYtrEBYTMscJBEGxs8mJDMRYi6TsCuodl+m5agg0vAJiUsc/jPHS4Y5DiaaJKLF9J6DxZIZR+DobBybDvALNnYHUFWH1QcuyqfK8rdVqbtS40QbmVXwKBqMOEGs6mAdoAIRp1C2T1ZkcHPL2oHwUFuxKWI7gDeQ0lF+18Aixa+v4BB/uKfCKtPjyzREA4LqUWlNstlOfuRP9+AzVdHKWfDeotFaU8aEb/It0ktKcrEuDGcVAtpO9eWtvcoEZ8BlWaPGbVoTaxCJWDRbKe0JPz9JFPs0YEaT6nt5LaqWYI8yByK7M2thoK7DIoZ93zvqyTHM2IJjeUDWE6f+G0/aib48K0myT3g3NPvHhCXi3b879Q231oDQVDlbnX8piSoKBu/mIQ==--fOeHlW2YD8T8xWB0--pbQRclG873vKQ79QBDa07g==
\ No newline at end of file
diff --git a/db/migrate/20241225223325_create_filedrop_comments.rb b/db/migrate/20241225223325_create_filedrop_comments.rb
new file mode 100644
index 0000000000000000000000000000000000000000..e75a7976b8d5bb47e184dd9dd9a98d39032783e3
--- /dev/null
+++ b/db/migrate/20241225223325_create_filedrop_comments.rb
@@ -0,0 +1,15 @@
+class CreateFiledropComments < ActiveRecord::Migration[7.1]
+ def change
+ create_table :filedrop_comments do |t|
+ t.text :body
+ t.datetime :orig_created
+ t.references :session, null: false, foreign_key: true
+
+ t.timestamps
+ end
+ end
+
+ def down
+ drop_table :filedrop_comments
+ end
+end
diff --git a/db/migrate/20241225223515_create_filedrop_files.rb b/db/migrate/20241225223515_create_filedrop_files.rb
new file mode 100644
index 0000000000000000000000000000000000000000..becfb98d060cf24af27adee55d56c4a394657936
--- /dev/null
+++ b/db/migrate/20241225223515_create_filedrop_files.rb
@@ -0,0 +1,17 @@
+class CreateFiledropFiles < ActiveRecord::Migration[7.1]
+ def change
+ create_table :filedrop_files do |t|
+ t.string :name
+ t.integer :size
+ t.string :checksum
+ t.datetime :orig_created
+ t.references :session, null: false, foreign_key: true
+
+ t.timestamps
+ end
+ end
+
+ def down
+ drop_table :filedrop_files
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index fc0392c661bc77d2876fa992b58e410e7ea0da21..f2d669d7e780937eb61fbd77acab2e2e0d7a28f0 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.1].define(version: 2024_12_24_002828) do
+ActiveRecord::Schema[7.1].define(version: 2024_12_25_223515) do
create_table "assignments", force: :cascade do |t|
t.integer "user_id", null: false
t.integer "session_id", null: false
@@ -55,6 +55,26 @@ ActiveRecord::Schema[7.1].define(version: 2024_12_24_002828) do
t.index ["job_id"], name: "index_crono_jobs_on_job_id", unique: true
end
+ create_table "filedrop_comments", force: :cascade do |t|
+ t.text "body"
+ t.datetime "orig_created"
+ t.integer "session_id", null: false
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["session_id"], name: "index_filedrop_comments_on_session_id"
+ end
+
+ create_table "filedrop_files", force: :cascade do |t|
+ t.string "name"
+ t.integer "size"
+ t.string "checksum"
+ t.datetime "orig_created"
+ t.integer "session_id", null: false
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["session_id"], name: "index_filedrop_files_on_session_id"
+ end
+
create_table "model_versions", force: :cascade do |t|
t.string "model"
t.text "changed_data"
@@ -290,6 +310,8 @@ ActiveRecord::Schema[7.1].define(version: 2024_12_24_002828) do
add_foreign_key "assignments", "users"
add_foreign_key "candidates", "sessions"
add_foreign_key "candidates", "users"
+ add_foreign_key "filedrop_comments", "sessions"
+ add_foreign_key "filedrop_files", "sessions"
add_foreign_key "relevant_stages", "conferences"
add_foreign_key "relevant_stages", "stages"
add_foreign_key "revision_sets", "conferences"
diff --git a/db/seeds.rb b/db/seeds.rb
index 51926174fc983fa3b41132f29d5560192d78bae0..4b58a58f071063cb0a7325e41687950fb5247808 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -49,26 +49,34 @@
# c.save!
# end
-Conference.find_or_create_by!(slug: "38c3") do |c|
+Conference.find_or_create_by!(slug: "38c3").tap do |c|
c.name = "38th Chaos Communication Congress (de-en)"
c.time_zone = "Berlin"
c.starts_at = DateTime.parse("27 December 2024 10:30 CET")
c.ends_at = DateTime.parse("30 December 2024 19:00 CET")
c.data = {
- "schedule_url" => "https://api.events.ccc.de/congress/2024/assembly/6840c453-af5c-413c-8127-adcbdcd98e9e/schedule.json"
+ "schedule_url" => "https://api.events.ccc.de/congress/2024/assembly/6840c453-af5c-413c-8127-adcbdcd98e9e/schedule.json",
+ "filedrop" => {
+ "url" => "https://speakers.c3lingo.org"
+ }
+
}
c.import_job_class = "pretalx"
c.location = "Congress Center Hamburg"
c.save!
end
-Conference.find_or_create_by!(slug: "38c3-more") do |c|
+Conference.find_or_create_by!(slug: "38c3-more").tap do |c|
c.name = "38th Chaos Communication Congress (more languages)"
c.time_zone = "Berlin"
c.starts_at = DateTime.parse("27 December 2024 10:30 CET")
c.ends_at = DateTime.parse("30 December 2024 19:00 CET")
c.data = {
- "schedule_url" => "https://api.events.ccc.de/congress/2024/assembly/6840c453-af5c-413c-8127-adcbdcd98e9e/schedule.json"
+ "schedule_url" => "https://api.events.ccc.de/congress/2024/assembly/6840c453-af5c-413c-8127-adcbdcd98e9e/schedule.json",
+ "filedrop" => {
+ "url" => "https://speakers.c3lingo.org",
+ "skip_downloads" => true
+ }
}
c.import_job_class = "pretalx"
c.location = "Congress Center Hamburg"
diff --git a/test/fixtures/filedrop_comments.yml b/test/fixtures/filedrop_comments.yml
new file mode 100644
index 0000000000000000000000000000000000000000..14211cbbda587f625d9457b53957adac00bbc873
--- /dev/null
+++ b/test/fixtures/filedrop_comments.yml
@@ -0,0 +1,9 @@
+# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+one:
+ body: MyText
+ session: one
+
+two:
+ body: MyText
+ session: two
diff --git a/test/fixtures/filedrop_files.yml b/test/fixtures/filedrop_files.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2366dbd19b1015172fe6fe86470f17ac4602b1f8
--- /dev/null
+++ b/test/fixtures/filedrop_files.yml
@@ -0,0 +1,13 @@
+# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+one:
+ name: MyString
+ size: 1
+ checksum: MyString
+ session: one
+
+two:
+ name: MyString
+ size: 1
+ checksum: MyString
+ session: two
diff --git a/test/models/filedrop_comment_test.rb b/test/models/filedrop_comment_test.rb
new file mode 100644
index 0000000000000000000000000000000000000000..d3d6e986ba86fc5e90938d5c9d082168af5c629e
--- /dev/null
+++ b/test/models/filedrop_comment_test.rb
@@ -0,0 +1,7 @@
+require "test_helper"
+
+class FiledropCommentTest < ActiveSupport::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
diff --git a/test/models/filedrop_file_test.rb b/test/models/filedrop_file_test.rb
new file mode 100644
index 0000000000000000000000000000000000000000..3179bf265de23f529ab8e9a28fb134373a0a7b2c
--- /dev/null
+++ b/test/models/filedrop_file_test.rb
@@ -0,0 +1,7 @@
+require "test_helper"
+
+class FiledropFileTest < ActiveSupport::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end