Lists (#5703)

* Add structure for lists

* Add list timeline streaming API

* Add list APIs, bind list-account relation to follow relation

* Add API for adding/removing accounts from lists

* Add pagination to lists API

* Add pagination to list accounts API

* Adjust scopes for new APIs

- Creating and modifying lists merely requires "write" scope
- Fetching information about lists merely requires "read" scope

* Add test for wrong user context on list timeline

* Clean up tests

app/controllers/api/v1/lists/accounts_controller.rb

+# frozen_string_literal: true
+
+class Api::V1::Lists::AccountsController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read },    only: [:show]
+  before_action -> { doorkeeper_authorize! :write }, except: [:show]
+
+  before_action :require_user!
+  before_action :set_list
+
+  after_action :insert_pagination_headers, only: :show
+
+  def show
+    @accounts = @list.accounts.paginate_by_max_id(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:max_id], params[:since_id])
+    render json: @accounts, each_serializer: REST::AccountSerializer
+  end
+
+  def create
+    ApplicationRecord.transaction do
+      list_accounts.each do |account|
+        @list.accounts << account
+      end
+    end
+
+    render_empty
+  end
+
+  def destroy
+    ListAccount.where(list: @list, account_id: account_ids).destroy_all
+    render_empty
+  end
+
+  private
+
+  def set_list
+    @list = List.where(account: current_account).find(params[:list_id])
+  end
+
+  def list_accounts
+    Account.find(account_ids)
+  end
+
+  def account_ids
+    Array(resource_params[:account_ids])
+  end
+
+  def resource_params
+    params.permit(account_ids: [])
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    if records_continue?
+      api_v1_list_accounts_url pagination_params(max_id: pagination_max_id)
+    end
+  end
+
+  def prev_path
+    unless @accounts.empty?
+      api_v1_list_accounts_url pagination_params(since_id: pagination_since_id)
+    end
+  end
+
+  def pagination_max_id
+    @accounts.last.id
+  end
+
+  def pagination_since_id
+    @accounts.first.id
+  end
+
+  def records_continue?
+    @accounts.size == limit_param(DEFAULT_ACCOUNTS_LIMIT)
+  end
+
+  def pagination_params(core_params)
+    params.permit(:limit).merge(core_params)
+  end
+end

app/controllers/api/v1/lists_controller.rb

+# frozen_string_literal: true
+
+class Api::V1::ListsController < Api::BaseController
+  LISTS_LIMIT = 50
+
+  before_action -> { doorkeeper_authorize! :read },    only: [:index, :show]
+  before_action -> { doorkeeper_authorize! :write }, except: [:index, :show]
+
+  before_action :require_user!
+  before_action :set_list, except: [:index, :create]
+
+  after_action :insert_pagination_headers, only: :index
+
+  def index
+    @lists = List.where(account: current_account).paginate_by_max_id(limit_param(LISTS_LIMIT), params[:max_id], params[:since_id])
+    render json: @lists, each_serializer: REST::ListSerializer
+  end
+
+  def show
+    render json: @list, serializer: REST::ListSerializer
+  end
+
+  def create
+    @list = List.create!(list_params.merge(account: current_account))
+    render json: @list, serializer: REST::ListSerializer
+  end
+
+  def update
+    @list.update!(list_params)
+    render json: @list, serializer: REST::ListSerializer
+  end
+
+  def destroy
+    @list.destroy!
+    render_empty
+  end
+
+  private
+
+  def set_list
+    @list = List.where(account: current_account).find(params[:id])
+  end
+
+  def list_params
+    params.permit(:title)
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    if records_continue?
+      api_v1_lists_url pagination_params(max_id: pagination_max_id)
+    end
+  end
+
+  def prev_path
+    unless @lists.empty?
+      api_v1_lists_url pagination_params(since_id: pagination_since_id)
+    end
+  end
+
+  def pagination_max_id
+    @lists.last.id
+  end
+
+  def pagination_since_id
+    @lists.first.id
+  end
+
+  def records_continue?
+    @lists.size == limit_param(LISTS_LIMIT)
+  end
+
+  def pagination_params(core_params)
+    params.permit(:limit).merge(core_params)
+  end
+end

app/controllers/api/v1/timelines/home_controller.rb

@@ -31,7 +31,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController
   end
 
   def account_home_feed
-    Feed.new(:home, current_account)
+    HomeFeed.new(current_account)
   end
 
   def insert_pagination_headers

app/controllers/api/v1/timelines/list_controller.rb

+# frozen_string_literal: true
+
+class Api::V1::Timelines::ListController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read }
+  before_action :require_user!
+  before_action :set_list
+  before_action :set_statuses
+
+  after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
+
+  def show
+    render json: @statuses,
+           each_serializer: REST::StatusSerializer,
+           relationships: StatusRelationshipsPresenter.new(@statuses, current_user.account_id)
+  end
+
+  private
+
+  def set_list
+    @list = List.where(account: current_account).find(params[:id])
+  end
+
+  def set_statuses
+    @statuses = cached_list_statuses
+  end
+
+  def cached_list_statuses
+    cache_collection list_statuses, Status
+  end
+
+  def list_statuses
+    list_feed.get(
+      limit_param(DEFAULT_STATUSES_LIMIT),
+      params[:max_id],
+      params[:since_id]
+    )
+  end
+
+  def list_feed
+    ListFeed.new(@list)
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def pagination_params(core_params)
+    params.permit(:limit).merge(core_params)
+  end
+
+  def next_path
+    api_v1_timelines_list_url params[:id], pagination_params(max_id: pagination_max_id)
+  end
+
+  def prev_path
+    api_v1_timelines_list_url params[:id], pagination_params(since_id: pagination_since_id)
+  end
+
+  def pagination_max_id
+    @statuses.last.id
+  end
+
+  def pagination_since_id
+    @statuses.first.id
+  end
+end

app/lib/feed_manager.rb

@@ -26,34 +26,42 @@ class FeedManager
     end
   end
 
-  def push(timeline_type, account, status)
-    return false unless add_to_feed(timeline_type, account, status)
-
-    trim(timeline_type, account.id)
-
-    PushUpdateWorker.perform_async(account.id, status.id) if push_update_required?(timeline_type, account.id)
-
+  def push_to_home(account, status)
+    return false unless add_to_feed(:home, account.id, status)
+    trim(:home, account.id)
+    PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}")
     true
   end
 
-  def unpush(timeline_type, account, status)
-    return false unless remove_from_feed(timeline_type, account, status)
+  def unpush_from_home(account, status)
+    return false unless remove_from_feed(:home, account.id, status)
+    Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
+    true
+  end
 
-    payload = Oj.dump(event: :delete, payload: status.id.to_s)
-    Redis.current.publish("timeline:#{account.id}", payload)
+  def push_to_list(list, status)
+    return false unless add_to_feed(:list, list.id, status)
+    trim(:list, list.id)
+    PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}")
+    true
+  end
 
+  def unpush_from_list(list, status)
+    return false unless remove_from_feed(:list, list.id, status)
+    Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
     true
   end
 
   def trim(type, account_id)
     timeline_key = key(type, account_id)
-    reblog_key = key(type, account_id, 'reblogs')
+    reblog_key   = key(type, account_id, 'reblogs')
+
     # Remove any items past the MAX_ITEMS'th entry in our feed
     redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s)
 
     # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop
     # tracking anything after it for deduplication purposes.
-    falloff_rank = FeedManager::REBLOG_FALLOFF - 1
+    falloff_rank  = FeedManager::REBLOG_FALLOFF - 1
     falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true)
     falloff_score = falloff_range&.first&.last&.to_i || 0
 
@@ -69,10 +77,6 @@ class FeedManager
     end
   end
 
-  def push_update_required?(timeline_type, account_id)
-    timeline_type != :home || redis.get("subscribed:timeline:#{account_id}").present?
-  end
-
   def merge_into_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
     query        = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
@@ -84,28 +88,28 @@ class FeedManager
 
     query.each do |status|
       next if status.direct_visibility? || filter?(:home, status, into_account)
-      add_to_feed(:home, into_account, status)
+      add_to_feed(:home, into_account.id, status)
     end
 
     trim(:home, into_account.id)
   end
 
   def unmerge_from_timeline(from_account, into_account)
-    timeline_key = key(:home, into_account.id)
+    timeline_key      = key(:home, into_account.id)
     oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
 
     from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status|
-      remove_from_feed(:home, into_account, status)
+      remove_from_feed(:home, into_account.id, status)
     end
   end
 
   def clear_from_timeline(account, target_account)
-    timeline_key = key(:home, account.id)
+    timeline_key        = key(:home, account.id)
     timeline_status_ids = redis.zrange(timeline_key, 0, -1)
-    target_statuses = Status.where(id: timeline_status_ids, account: target_account)
+    target_statuses     = Status.where(id: timeline_status_ids, account: target_account)
 
     target_statuses.each do |status|
-      unpush(:home, account, status)
+      unpush_from_home(account, status)
     end
   end
 
@@ -122,7 +126,7 @@ class FeedManager
 
       statuses.each do |status|
         next if filter_from_home?(status, account)
-        added += 1 if add_to_feed(:home, account, status)
+        added += 1 if add_to_feed(:home, account.id, status)
       end
 
       break unless added.zero?
@@ -137,6 +141,10 @@ class FeedManager
     Redis.current
   end
 
+  def push_update_required?(timeline_id)
+    redis.exists("subscribed:#{timeline_id}")
+  end
+
   def filter_from_home?(status, receiver_id)
     return false if receiver_id == status.account_id
     return true  if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
@@ -182,9 +190,9 @@ class FeedManager
   # added, and false if it was not added to the feed. Note that this is
   # an internal helper: callers must call trim or push updates if
   # either action is appropriate.
-  def add_to_feed(timeline_type, account, status)
-    timeline_key = key(timeline_type, account.id)
-    reblog_key   = key(timeline_type, account.id, 'reblogs')
+  def add_to_feed(timeline_type, account_id, status)
+    timeline_key = key(timeline_type, account_id)
+    reblog_key   = key(timeline_type, account_id, 'reblogs')
 
     if status.reblog?
       # If the original status or a reblog of it is within
@@ -195,6 +203,7 @@ class FeedManager
       return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF
 
       reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id)
+
       if reblog_rank.nil?
         # This is not something we've already seen reblogged, so we
         # can just add it to the feed (and note that we're
@@ -205,7 +214,7 @@ class FeedManager
         # Another reblog of the same status was already in the
         # REBLOG_FALLOFF most recent statuses, so we note that this
         # is an "extra" reblog, by storing it in reblog_set_key.
-        reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}")
+        reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}")
         redis.sadd(reblog_set_key, status.id)
         return false
       end
@@ -220,8 +229,8 @@ class FeedManager
   # with reblogs, and returning true if a status was removed. As with
   # `add_to_feed`, this does not trigger push updates, so callers must
   # do so if appropriate.
-  def remove_from_feed(timeline_type, account, status)
-    timeline_key = key(timeline_type, account.id)
+  def remove_from_feed(timeline_type, account_id, status)
+    timeline_key = key(timeline_type, account_id)
 
     if status.reblog?
       # 1. If the reblogging status is not in the feed, stop.
@@ -229,7 +238,7 @@ class FeedManager
       return false if status_rank.nil?
 
       # 2. Remove reblog from set of this status's reblogs.
-      reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}")
+      reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}")
 
       redis.srem(reblog_set_key, status.id)
       # 3. Re-insert another reblog or original into the feed if one
@@ -244,7 +253,7 @@ class FeedManager
       # (outside conditional)
     else
       # If the original is getting deleted, no use for reblog references
-      redis.del(key(timeline_type, account.id, "reblogs:#{status.id}"))
+      redis.del(key(timeline_type, account_id, "reblogs:#{status.id}"))
     end
 
     redis.zrem(timeline_key, status.id)

app/models/account.rb

@@ -3,7 +3,7 @@
 #
 # Table name: accounts
 #
-#  id                      :bigint           not null, primary key
+#  id                      :integer          not null, primary key
 #  username                :string           default(""), not null
 #  domain                  :string
 #  secret                  :string           default(""), not null
@@ -53,6 +53,7 @@ class Account < ApplicationRecord
   include AccountInteractions
   include Attachmentable
   include Remotable
+  include Paginable
 
   enum protocol: [:ostatus, :activitypub]
 
@@ -95,6 +96,10 @@ class Account < ApplicationRecord
   has_many :account_moderation_notes, dependent: :destroy
   has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy
 
+  # Lists
+  has_many :list_accounts, inverse_of: :account, dependent: :destroy
+  has_many :lists, through: :list_accounts
+
   scope :remote, -> { where.not(domain: nil) }
   scope :local, -> { where(domain: nil) }
   scope :without_followers, -> { where(followers_count: 0) }

app/models/account_domain_block.rb

@@ -3,11 +3,11 @@
 #
 # Table name: account_domain_blocks
 #
+#  id         :integer          not null, primary key
 #  domain     :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
-#  account_id :bigint
-#  id         :bigint           not null, primary key
+#  account_id :integer
 #
 
 class AccountDomainBlock < ApplicationRecord

app/models/account_moderation_note.rb

@@ -3,10 +3,10 @@
 #
 # Table name: account_moderation_notes
 #
-#  id                :bigint           not null, primary key
+#  id                :integer          not null, primary key
 #  content           :text             not null
-#  account_id        :bigint           not null
-#  target_account_id :bigint           not null
+#  account_id        :integer          not null
+#  target_account_id :integer          not null
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
 #

app/models/block.rb

@@ -3,11 +3,11 @@
 #
 # Table name: blocks
 #
+#  id                :integer          not null, primary key
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
-#  account_id        :bigint           not null
-#  id                :bigint           not null, primary key
-#  target_account_id :bigint           not null
+#  account_id        :integer          not null
+#  target_account_id :integer          not null
 #
 
 class Block < ApplicationRecord

app/models/conversation.rb

@@ -3,7 +3,7 @@
 #
 # Table name: conversations
 #
-#  id         :bigint           not null, primary key
+#  id         :integer          not null, primary key
 #  uri        :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null

app/models/conversation_mute.rb

@@ -3,9 +3,9 @@
 #
 # Table name: conversation_mutes
 #
-#  conversation_id :bigint           not null
-#  account_id      :bigint           not null
-#  id              :bigint           not null, primary key
+#  id              :integer          not null, primary key
+#  conversation_id :integer          not null
+#  account_id      :integer          not null
 #
 
 class ConversationMute < ApplicationRecord

app/models/custom_emoji.rb

@@ -3,7 +3,7 @@
 #
 # Table name: custom_emojis
 #
-#  id                 :bigint           not null, primary key
+#  id                 :integer          not null, primary key
 #  shortcode          :string           default(""), not null
 #  domain             :string
 #  image_file_name    :string

app/models/domain_block.rb

@@ -3,12 +3,12 @@
 #
 # Table name: domain_blocks
 #
+#  id           :integer          not null, primary key
 #  domain       :string           default(""), not null
 #  created_at   :datetime         not null
 #  updated_at   :datetime         not null
 #  severity     :integer          default("silence")
 #  reject_media :boolean          default(FALSE), not null
-#  id           :bigint           not null, primary key
 #
 
 class DomainBlock < ApplicationRecord

app/models/email_domain_block.rb

@@ -3,7 +3,7 @@
 #
 # Table name: email_domain_blocks
 #
-#  id         :bigint           not null, primary key
+#  id         :integer          not null, primary key
 #  domain     :string           default(""), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null

app/models/favourite.rb

@@ -3,11 +3,11 @@
 #
 # Table name: favourites
 #
+#  id         :integer          not null, primary key
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
-#  account_id :bigint           not null
-#  id         :bigint           not null, primary key
-#  status_id  :bigint           not null
+#  account_id :integer          not null
+#  status_id  :integer          not null
 #
 
 class Favourite < ApplicationRecord

app/models/feed.rb

 # frozen_string_literal: true
 
 class Feed
-  def initialize(type, account)
-    @type    = type
-    @account = account
+  def initialize(type, id)
+    @type = type
+    @id   = id
   end
 
   def get(limit, max_id = nil, since_id = nil)
-    if redis.exists("account:#{@account.id}:regeneration")
-      from_database(limit, max_id, since_id)
-    else
-      from_redis(limit, max_id, since_id)
-    end
+    from_redis(limit, max_id, since_id)
   end
 
-  private
+  protected
 
   def from_redis(limit, max_id, since_id)
     max_id     = '+inf' if max_id.blank?
     since_id   = '-inf' if since_id.blank?
     unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i)
-    Status.where(id: unhydrated).cache_ids
-  end
 
-  def from_database(limit, max_id, since_id)
-    Status.as_home_timeline(@account)
-          .paginate_by_max_id(limit, max_id, since_id)
-          .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
+    Status.where(id: unhydrated).cache_ids
   end
 
   def key
-    FeedManager.instance.key(@type, @account.id)
+    FeedManager.instance.key(@type, @id)
   end
 
   def redis

app/models/follow.rb

@@ -3,11 +3,11 @@
 #
 # Table name: follows
 #
+#  id                :integer          not null, primary key
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
-#  account_id        :bigint           not null
-#  id                :bigint           not null, primary key
-#  target_account_id :bigint           not null
+#  account_id        :integer          not null
+#  target_account_id :integer          not null
 #
 
 class Follow < ApplicationRecord

app/models/follow_request.rb

@@ -3,11 +3,11 @@
 #
 # Table name: follow_requests
 #
+#  id                :integer          not null, primary key
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
-#  account_id        :bigint           not null
-#  id                :bigint           not null, primary key
-#  target_account_id :bigint           not null
+#  account_id        :integer          not null
+#  target_account_id :integer          not null
 #
 
 class FollowRequest < ApplicationRecord

app/models/home_feed.rb

+# frozen_string_literal: true
+
+class HomeFeed < Feed
+  def initialize(account)
+    @type    = :home
+    @id      = account.id
+    @account = account
+  end
+
+  def get(limit, max_id = nil, since_id = nil)
+    if redis.exists("account:#{@account.id}:regeneration")
+      from_database(limit, max_id, since_id)
+    else
+      super
+    end
+  end
+
+  private
+
+  def from_database(limit, max_id, since_id)
+    Status.as_home_timeline(@account)
+          .paginate_by_max_id(limit, max_id, since_id)
+          .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
+  end
+end

app/models/import.rb

@@ -3,6 +3,7 @@
 #
 # Table name: imports
 #
+#  id                :integer          not null, primary key
 #  type              :integer          not null
 #  approved          :boolean          default(FALSE), not null
 #  created_at        :datetime         not null
@@ -11,8 +12,7 @@
 #  data_content_type :string
 #  data_file_size    :integer
 #  data_updated_at   :datetime
-#  account_id        :bigint           not null
-#  id                :bigint           not null, primary key
+#  account_id        :integer          not null
 #
 
 class Import < ApplicationRecord