Slightly reduce RAM usage (#7301)

* No need to re-require sidekiq plugins, they are required via Gemfile

* Add derailed_benchmarks tool, no need to require TTY gems in Gemfile

* Replace ruby-oembed with FetchOEmbedService

Reduce startup by 45382 allocated objects

* Remove preloaded JSON-LD in favour of caching HTTP responses

Reduce boot RAM by about 6 MiB

* Fix tests

* Fix test suite by stubbing out JSON-LD contexts

Gemfile

@@ -54,7 +54,7 @@ gem 'httplog', '~> 1.0'
 gem 'idn-ruby', require: 'idn'
 gem 'kaminari', '~> 1.1'
 gem 'link_header', '~> 0.0'
-gem 'mime-types', '~> 3.1'
+gem 'mime-types', '~> 3.1', require: 'mime/types/columnar'
 gem 'nokogiri', '~> 1.8'
 gem 'nsa', '~> 0.2'
 gem 'oj', '~> 3.5'
@@ -70,7 +70,6 @@ gem 'rails-settings-cached', '~> 0.6'
 gem 'redis', '~> 4.0', require: ['redis', 'redis/connection/hiredis']
 gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock'
 gem 'rqrcode', '~> 0.10'
-gem 'ruby-oembed', '~> 0.12', require: 'oembed'
 gem 'ruby-progressbar', '~> 1.4'
 gem 'sanitize', '~> 4.6'
 gem 'sidekiq', '~> 5.1'
@@ -82,14 +81,14 @@ gem 'simple_form', '~> 4.0'
 gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie'
 gem 'stoplight', '~> 2.1.3'
 gem 'strong_migrations', '~> 0.2'
-gem 'tty-command', '~> 0.8'
-gem 'tty-prompt', '~> 0.16'
+gem 'tty-command', '~> 0.8', require: false
+gem 'tty-prompt', '~> 0.16', require: false
 gem 'twitter-text', '~> 1.14'
 gem 'tzinfo-data', '~> 1.2018'
 gem 'webpacker', '~> 3.4'
 gem 'webpush'
 
-gem 'json-ld-preloaded', '~> 2.2'
+gem 'json-ld', '~> 2.2'
 gem 'rdf-normalize', '~> 0.3'
 
 group :development, :test do
@@ -135,6 +134,9 @@ group :development do
   gem 'capistrano-rails', '~> 1.3'
   gem 'capistrano-rbenv', '~> 2.1'
   gem 'capistrano-yarn', '~> 2.0'
+
+  gem 'derailed_benchmarks'
+  gem 'stackprof'
 end
 
 group :production do

Gemfile.lock

@@ -75,6 +75,7 @@ GEM
       aws-sigv4 (~> 1.0)
     aws-sigv4 (1.0.2)
     bcrypt (3.1.11)
+    benchmark-ips (2.7.2)
     better_errors (2.4.0)
       coderay (>= 1.0.0)
       erubi (>= 1.0.0)
@@ -138,6 +139,14 @@ GEM
     css_parser (1.6.0)
       addressable
     debug_inspector (0.0.3)
+    derailed_benchmarks (1.3.4)
+      benchmark-ips (~> 2)
+      get_process_mem (~> 0)
+      heapy (~> 0)
+      memory_profiler (~> 0)
+      rack (>= 1)
+      rake (> 10, < 13)
+      thor (~> 0.19)
     devise (4.4.3)
       bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
@@ -206,6 +215,7 @@ GEM
     fuubar (2.3.1)
       rspec-core (~> 3.0)
       ruby-progressbar (~> 1.4)
+    get_process_mem (0.2.1)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
     goldfinger (2.1.0)
@@ -226,6 +236,7 @@ GEM
       concurrent-ruby (~> 1.0)
     hashdiff (0.3.7)
     hashie (3.5.7)
+    heapy (0.1.3)
     highline (1.7.10)
     hiredis (0.6.1)
     hitimes (1.2.6)
@@ -264,10 +275,6 @@ GEM
     json-ld (2.2.1)
       multi_json (~> 1.12)
       rdf (>= 2.2.8, < 4.0)
-    json-ld-preloaded (2.2.3)
-      json-ld (>= 2.2, < 4.0)
-      multi_json (~> 1.12)
-      rdf (>= 2.2, < 4.0)
     jsonapi-renderer (0.2.0)
     jwt (2.1.0)
     kaminari (1.1.1)
@@ -502,7 +509,6 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-oembed (0.12.0)
     ruby-progressbar (1.9.0)
     ruby-saml (1.7.2)
       nokogiri (>= 1.5.10)
@@ -557,6 +563,7 @@ GEM
     sshkit (1.16.0)
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
+    stackprof (0.2.11)
     statsd-ruby (1.2.1)
     stoplight (2.1.3)
     streamio-ffmpeg (3.0.2)
@@ -645,6 +652,7 @@ DEPENDENCIES
   chewy (~> 5.0)
   cld3 (~> 3.2.0)
   climate_control (~> 0.2)
+  derailed_benchmarks
   devise (~> 4.4)
   devise-two-factor (~> 3.0)
   devise_pam_authenticatable2 (~> 9.1)
@@ -668,7 +676,7 @@ DEPENDENCIES
   i18n-tasks (~> 0.9)
   idn-ruby
   iso-639
-  json-ld-preloaded (~> 2.2)
+  json-ld (~> 2.2)
   kaminari (~> 1.1)
   letter_opener (~> 1.4)
   letter_opener_web (~> 1.3)
@@ -714,7 +722,6 @@ DEPENDENCIES
   rspec-retry (~> 0.5)
   rspec-sidekiq (~> 3.0)
   rubocop (~> 0.55)
-  ruby-oembed (~> 0.12)
   ruby-progressbar (~> 1.4)
   sanitize (~> 4.6)
   scss_lint (~> 0.57)
@@ -726,6 +733,7 @@ DEPENDENCIES
   simple_form (~> 4.0)
   simplecov (~> 0.16)
   sprockets-rails (~> 3.2)
+  stackprof
   stoplight (~> 2.1.3)
   streamio-ffmpeg (~> 3.0)
   strong_migrations (~> 0.2)

app/controllers/api/web/embeds_controller.rb

@@ -9,9 +9,12 @@ class Api::Web::EmbedsController < Api::Web::BaseController
     status = StatusFinder.new(params[:url]).status
     render json: status, serializer: OEmbedSerializer, width: 400
   rescue ActiveRecord::RecordNotFound
-    oembed = OEmbed::Providers.get(params[:url])
-    render json: Oj.dump(oembed.fields)
-  rescue OEmbed::NotFound
-    render json: {}, status: :not_found
+    oembed = FetchOEmbedService.new.call(params[:url])
+
+    if oembed
+      render json: oembed
+    else
+      render json: {}, status: :not_found
+    end
   end
 end

app/controllers/settings/follower_domains_controller.rb

 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class Settings::FollowerDomainsController < ApplicationController
   layout 'admin'
 

app/helpers/jsonld_helper.rb

@@ -48,7 +48,7 @@ module JsonLdHelper
   end
 
   def canonicalize(json)
-    graph = RDF::Graph.new << JSON::LD::API.toRdf(json)
+    graph = RDF::Graph.new << JSON::LD::API.toRdf(json, documentLoader: method(:load_jsonld_context))
     graph.dump(:normalize)
   end
 
@@ -90,4 +90,19 @@ module JsonLdHelper
     request.add_headers('Accept' => 'application/activity+json, application/ld+json')
     request
   end
+
+  def load_jsonld_context(url, _options = {}, &_block)
+    json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do
+      request = Request.new(:get, url)
+      request.add_headers('Accept' => 'application/ld+json')
+
+      request.perform do |res|
+        raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json'
+        res.body_with_limit
+      end
+    end
+
+    doc = JSON::LD::API::RemoteDocument.new(url, json)
+    block_given? ? yield(doc) : doc
+  end
 end

app/lib/provider_discovery.rb

-# frozen_string_literal: true
-
-class ProviderDiscovery < OEmbed::ProviderDiscovery
-  class << self
-    def get(url, **options)
-      provider = discover_provider(url, options)
-
-      options.delete(:html)
-
-      provider.get(url, options)
-    end
-
-    def discover_provider(url, **options)
-      format = options[:format]
-
-      html = if options[:html]
-               Nokogiri::HTML(options[:html])
-             else
-               Request.new(:get, url).perform do |res|
-                 raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html'
-                 Nokogiri::HTML(res.body_with_limit)
-               end
-             end
-
-      if format.nil? || format == :json
-        provider_endpoint ||= html.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value
-        format ||= :json if provider_endpoint
-      end
-
-      if format.nil? || format == :xml
-        provider_endpoint ||= html.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value
-        format ||= :xml if provider_endpoint
-      end
-
-      raise OEmbed::NotFound, url if provider_endpoint.nil?
-      begin
-        provider_endpoint = Addressable::URI.parse(provider_endpoint)
-        provider_endpoint.query = nil
-        provider_endpoint = provider_endpoint.to_s
-      rescue Addressable::URI::InvalidURIError
-        raise OEmbed::NotFound, url
-      end
-
-      OEmbed::Provider.new(provider_endpoint, format)
-    end
-  end
-end

app/services/fan_out_on_write_service.rb

 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class FanOutOnWriteService < BaseService
   # Push a status into home and mentions feeds
   # @param [Status] status

app/services/fetch_link_card_service.rb

@@ -85,42 +85,40 @@ class FetchLinkCardService < BaseService
   end
 
   def attempt_oembed
-    embed = OEmbed::Providers.get(@url, html: @html)
+    embed = FetchOEmbedService.new.call(@url, html: @html)
 
-    return false unless embed.respond_to?(:type)
+    return false if embed.nil?
 
-    @card.type          = embed.type
-    @card.title         = embed.respond_to?(:title)         ? embed.title         : ''
-    @card.author_name   = embed.respond_to?(:author_name)   ? embed.author_name   : ''
-    @card.author_url    = embed.respond_to?(:author_url)    ? embed.author_url    : ''
-    @card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : ''
-    @card.provider_url  = embed.respond_to?(:provider_url)  ? embed.provider_url  : ''
+    @card.type          = embed[:type]
+    @card.title         = embed[:title]         || ''
+    @card.author_name   = embed[:author_name]   || ''
+    @card.author_url    = embed[:author_url]    || ''
+    @card.provider_name = embed[:provider_name] || ''
+    @card.provider_url  = embed[:provider_url]  || ''
     @card.width         = 0
     @card.height        = 0
 
     case @card.type
     when 'link'
-      @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
+      @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
     when 'photo'
-      return false unless embed.respond_to?(:url)
+      return false if embed[:url].blank?
 
-      @card.embed_url        = embed.url
-      @card.image_remote_url = embed.url
-      @card.width            = embed.width.presence  || 0
-      @card.height           = embed.height.presence || 0
+      @card.embed_url        = embed[:url]
+      @card.image_remote_url = embed[:url]
+      @card.width            = embed[:width].presence  || 0
+      @card.height           = embed[:height].presence || 0
     when 'video'
-      @card.width            = embed.width.presence  || 0
-      @card.height           = embed.height.presence || 0
-      @card.html             = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED)
-      @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
+      @card.width            = embed[:width].presence  || 0
+      @card.height           = embed[:height].presence || 0
+      @card.html             = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED)
+      @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
     when 'rich'
       # Most providers rely on <script> tags, which is a no-no
       return false
     end
 
     @card.save_with_optional_image!
-  rescue OEmbed::NotFound
-    false
   end
 
   def attempt_opengraph

app/services/fetch_oembed_service.rb

+# frozen_string_literal: true
+
+class FetchOEmbedService
+  attr_reader :url, :options, :format, :endpoint_url
+
+  def call(url, options = {})
+    @url     = url
+    @options = options
+
+    discover_endpoint!
+    fetch!
+  end
+
+  private
+
+  def discover_endpoint!
+    return if html.nil?
+
+    @format = @options[:format]
+    page    = Nokogiri::HTML(html)
+
+    if @format.nil? || @format == :json
+      @endpoint_url ||= page.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value
+      @format       ||= :json if @endpoint_url
+    end
+
+    if @format.nil? || @format == :xml
+      @endpoint_url ||= page.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value
+      @format       ||= :xml if @endpoint_url
+    end
+
+    return if @endpoint_url.blank?
+
+    @endpoint_url = Addressable::URI.parse(@endpoint_url).to_s
+  rescue Addressable::URI::InvalidURIError
+    @endpoint_url = nil
+  end
+
+  def fetch!
+    return if @endpoint_url.blank?
+
+    body = Request.new(:get, @endpoint_url).perform do |res|
+      res.code != 200 ? nil : res.body_with_limit
+    end
+
+    validate(parse_for_format(body)) unless body.nil?
+  rescue Oj::ParseError, Ox::ParseError
+    nil
+  end
+
+  def parse_for_format(body)
+    case @format
+    when :json
+      Oj.load(body, mode: :strict)&.with_indifferent_access
+    when :xml
+      Ox.load(body, mode: :hash_no_attrs)&.with_indifferent_access&.dig(:oembed)
+    end
+  end
+
+  def validate(oembed)
+    oembed if oembed[:version] == '1.0' && oembed[:type].present?
+  end
+
+  def html
+    return @html if defined?(@html)
+
+    @html = @options[:html] || Request.new(:get, @url).perform do |res|
+      res.code != 200 || res.mime_type != 'text/html' ? nil : res.body_with_limit
+    end
+  end
+end

app/workers/scheduler/backup_cleanup_scheduler.rb

 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::BackupCleanupScheduler
   include Sidekiq::Worker

app/workers/scheduler/doorkeeper_cleanup_scheduler.rb

 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::DoorkeeperCleanupScheduler
   include Sidekiq::Worker

app/workers/scheduler/email_scheduler.rb

 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::EmailScheduler
   include Sidekiq::Worker

app/workers/scheduler/feed_cleanup_scheduler.rb

 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::FeedCleanupScheduler
   include Sidekiq::Worker

app/workers/scheduler/ip_cleanup_scheduler.rb

 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::IpCleanupScheduler
   include Sidekiq::Worker

app/workers/scheduler/media_cleanup_scheduler.rb

 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::MediaCleanupScheduler
   include Sidekiq::Worker

app/workers/scheduler/subscriptions_cleanup_scheduler.rb

 # frozen_string_literal: true
 
-require 'sidekiq-scheduler'
-
 class Scheduler::SubscriptionsCleanupScheduler
   include Sidekiq::Worker
 

app/workers/scheduler/subscriptions_scheduler.rb

 # frozen_string_literal: true
 
-require 'sidekiq-scheduler'
-require 'sidekiq-bulk'
-
 class Scheduler::SubscriptionsScheduler
   include Sidekiq::Worker
 

app/workers/scheduler/user_cleanup_scheduler.rb

 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::UserCleanupScheduler
   include Sidekiq::Worker

app/workers/soft_block_domain_followers_worker.rb

 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class SoftBlockDomainFollowersWorker
   include Sidekiq::Worker
 

config/initializers/json_ld.rb

-# frozen_string_literal: true
-
-require_relative '../../lib/json_ld/identity'
-require_relative '../../lib/json_ld/security'
-require_relative '../../lib/json_ld/activitystreams'