From 96c1e7132971877fba308c51cd42306f0b1bf166 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 19 Nov 2020 17:48:13 +0100
Subject: [PATCH] Add import/export feature for bookmarks (#14956)
* Add ability to export bookmarks
* Add support for importing bookmarks
* Add bookmark import tests
* Add bookmarks export test
---
.../settings/exports/bookmarks_controller.rb | 19 ++++++++
app/models/export.rb | 12 +++++
app/models/import.rb | 2 +-
app/services/import_service.rb | 45 +++++++++++++++++++
app/views/settings/exports/show.html.haml | 4 ++
config/locales/en.yml | 2 +
config/routes.rb | 1 +
.../exports/bookmarks_controller_specs.rb | 17 +++++++
spec/fixtures/files/bookmark-imports.txt | 4 ++
spec/services/import_service_spec.rb | 42 +++++++++++++++++
10 files changed, 147 insertions(+), 1 deletion(-)
create mode 100644 app/controllers/settings/exports/bookmarks_controller.rb
create mode 100644 spec/controllers/settings/exports/bookmarks_controller_specs.rb
create mode 100644 spec/fixtures/files/bookmark-imports.txt
diff --git a/app/controllers/settings/exports/bookmarks_controller.rb b/app/controllers/settings/exports/bookmarks_controller.rb
new file mode 100644
index 0000000000..c12e2f147a
--- /dev/null
+++ b/app/controllers/settings/exports/bookmarks_controller.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Settings
+ module Exports
+ class BookmarksController < BaseController
+ include ExportControllerConcern
+
+ def index
+ send_export_file
+ end
+
+ private
+
+ def export_data
+ @export.to_bookmarks_csv
+ end
+ end
+ end
+end
diff --git a/app/models/export.rb b/app/models/export.rb
index cab01f11ad..5216eed5ea 100644
--- a/app/models/export.rb
+++ b/app/models/export.rb
@@ -9,6 +9,14 @@ class Export
@account = account
end
+ def to_bookmarks_csv
+ CSV.generate do |csv|
+ account.bookmarks.includes(:status).reorder(id: :desc).each do |bookmark|
+ csv << [ActivityPub::TagManager.instance.uri_for(bookmark.status)]
+ end
+ end
+ end
+
def to_blocked_accounts_csv
to_csv account.blocking.select(:username, :domain)
end
@@ -55,6 +63,10 @@ class Export
account.statuses_count
end
+ def total_bookmarks
+ account.bookmarks.count
+ end
+
def total_follows
account.following_count
end
diff --git a/app/models/import.rb b/app/models/import.rb
index c78a04d073..7024532895 100644
--- a/app/models/import.rb
+++ b/app/models/import.rb
@@ -24,7 +24,7 @@ class Import < ApplicationRecord
belongs_to :account
- enum type: [:following, :blocking, :muting, :domain_blocking]
+ enum type: [:following, :blocking, :muting, :domain_blocking, :bookmarks]
validates :type, presence: true
diff --git a/app/services/import_service.rb b/app/services/import_service.rb
index 7e55452de7..288e47f1ea 100644
--- a/app/services/import_service.rb
+++ b/app/services/import_service.rb
@@ -18,6 +18,8 @@ class ImportService < BaseService
import_mutes!
when 'domain_blocking'
import_domain_blocks!
+ when 'bookmarks'
+ import_bookmarks!
end
end
@@ -88,6 +90,39 @@ class ImportService < BaseService
end
end
+ def import_bookmarks!
+ parse_import_data!(['#uri'])
+ items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip }
+
+ if @import.overwrite?
+ presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true }
+
+ @account.bookmarks.find_each do |bookmark|
+ if presence_hash[bookmark.status.uri]
+ items.delete(bookmark.status.uri)
+ else
+ bookmark.destroy!
+ end
+ end
+ end
+
+ statuses = items.map do |uri|
+ status = ActivityPub::TagManager.instance.uri_to_resource(uri, Status)
+ next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri)
+
+ status || ActivityPub::FetchRemoteStatusService.new.call(uri)
+ end.compact
+
+ account_ids = statuses.map(&:account_id)
+ preloaded_relations = relations_map_for_account(@account, account_ids)
+
+ statuses.keep_if { |status| StatusPolicy.new(@account, status, preloaded_relations).show? }
+
+ statuses.each do |status|
+ @account.bookmarks.find_or_create_by!(account: @account, status: status)
+ end
+ end
+
def parse_import_data!(default_headers)
data = CSV.parse(import_data, headers: true)
data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ')
@@ -101,4 +136,14 @@ class ImportService < BaseService
def follow_limit
FollowLimitValidator.limit_for_account(@account)
end
+
+ def relations_map_for_account(account, account_ids)
+ {
+ blocking: {},
+ blocked_by: Account.blocked_by_map(account_ids, account.id),
+ muting: {},
+ following: Account.following_map(account_ids, account.id),
+ domain_blocking_by_domain: {},
+ }
+ end
end
diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml
index 0bb80e9372..18b52c0c2c 100644
--- a/app/views/settings/exports/show.html.haml
+++ b/app/views/settings/exports/show.html.haml
@@ -36,6 +36,10 @@
%th= t('exports.domain_blocks')
%td= number_with_delimiter @export.total_domain_blocks
%td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv)
+ %tr
+ %th= t('exports.bookmarks')
+ %td= number_with_delimiter @export.total_bookmarks
+ %td= table_link_to 'download', t('bookmarks.csv'), settings_exports_bookmarks_path(format: :csv)
%hr.spacer/
diff --git a/config/locales/en.yml b/config/locales/en.yml
index bec0990827..263ffcdc77 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -842,6 +842,7 @@ en:
request: Request your archive
size: Size
blocks: You block
+ bookmarks: Bookmarks
csv: CSV
domain_blocks: Domain blocks
lists: Lists
@@ -918,6 +919,7 @@ en:
success: Your data was successfully uploaded and will now be processed in due time
types:
blocking: Blocking list
+ bookmarks: Bookmarks
domain_blocking: Domain blocking list
following: Following list
muting: Muting list
diff --git a/config/routes.rb b/config/routes.rb
index 54c76799ca..a534b433e0 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -125,6 +125,7 @@ Rails.application.routes.draw do
resources :mutes, only: :index, controller: :muted_accounts
resources :lists, only: :index, controller: :lists
resources :domain_blocks, only: :index, controller: :blocked_domains
+ resources :bookmarks, only: :index, controller: :bookmarks
end
resources :two_factor_authentication_methods, only: [:index] do
diff --git a/spec/controllers/settings/exports/bookmarks_controller_specs.rb b/spec/controllers/settings/exports/bookmarks_controller_specs.rb
new file mode 100644
index 0000000000..85761577bd
--- /dev/null
+++ b/spec/controllers/settings/exports/bookmarks_controller_specs.rb
@@ -0,0 +1,17 @@
+require 'rails_helper'
+
+describe Settings::Exports::BookmarksController do
+ render_views
+
+ describe 'GET #index' do
+ it 'returns a csv of the bookmarked toots' do
+ user = Fabricate(:user)
+ user.account.bookmarks.create!(status: Fabricate(:status, uri: 'https://foo.bar/statuses/1312'))
+
+ sign_in user, scope: :user
+ get :index, format: :csv
+
+ expect(response.body).to eq "https://foo.bar/statuses/1312\n"
+ end
+ end
+end
diff --git a/spec/fixtures/files/bookmark-imports.txt b/spec/fixtures/files/bookmark-imports.txt
new file mode 100644
index 0000000000..7cc8901a0a
--- /dev/null
+++ b/spec/fixtures/files/bookmark-imports.txt
@@ -0,0 +1,4 @@
+https://example.com/statuses/1312
+https://local.com/users/foo/statuses/42
+https://unknown-remote.com/users/bar/statuses/1
+https://example.com/statuses/direct
diff --git a/spec/services/import_service_spec.rb b/spec/services/import_service_spec.rb
index b1909d4fd1..764225aa72 100644
--- a/spec/services/import_service_spec.rb
+++ b/spec/services/import_service_spec.rb
@@ -1,6 +1,8 @@
require 'rails_helper'
RSpec.describe ImportService, type: :service do
+ include RoutingHelper
+
let!(:account) { Fabricate(:account, locked: false) }
let!(:bob) { Fabricate(:account, username: 'bob', locked: false) }
let!(:eve) { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') }
@@ -169,4 +171,44 @@ RSpec.describe ImportService, type: :service do
end
end
end
+
+ context 'import bookmarks' do
+ subject { ImportService.new }
+
+ let(:csv) { attachment_fixture('bookmark-imports.txt') }
+
+ around(:each) do |example|
+ local_before = Rails.configuration.x.local_domain
+ web_before = Rails.configuration.x.web_domain
+ Rails.configuration.x.local_domain = 'local.com'
+ Rails.configuration.x.web_domain = 'local.com'
+ example.run
+ Rails.configuration.x.web_domain = web_before
+ Rails.configuration.x.local_domain = local_before
+ end
+
+ let(:local_account) { Fabricate(:account, username: 'foo', domain: '') }
+ let!(:remote_status) { Fabricate(:status, uri: 'https://example.com/statuses/1312') }
+ let!(:direct_status) { Fabricate(:status, uri: 'https://example.com/statuses/direct', visibility: :direct) }
+
+ before do
+ service = double
+ allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service)
+ allow(service).to receive(:call).with('https://unknown-remote.com/users/bar/statuses/1') do
+ Fabricate(:status, uri: 'https://unknown-remote.com/users/bar/statuses/1')
+ end
+ end
+
+ describe 'when no bookmarks are set' do
+ let(:import) { Import.create(account: account, type: 'bookmarks', data: csv) }
+ it 'adds the toots the user has access to to bookmarks' do
+ local_status = Fabricate(:status, account: local_account, uri: 'https://local.com/users/foo/statuses/42', id: 42, local: true)
+ subject.call(import)
+ expect(account.bookmarks.map(&:status).map(&:id)).to include(local_status.id)
+ expect(account.bookmarks.map(&:status).map(&:id)).to include(remote_status.id)
+ expect(account.bookmarks.map(&:status).map(&:id)).not_to include(direct_status.id)
+ expect(account.bookmarks.count).to eq 3
+ end
+ end
+ end
end
--
GitLab