diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index badb2672062bdc72d99c32f719811e76dbbfde75..a45681078ee2efca935906d0a004dc38369103e1 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -15,11 +15,11 @@ class ActivityPub::ProcessAccountService < BaseService
     @account     = Account.find_by(uri: @uri)
     @collections = {}
 
+    old_public_key = @account&.public_key
     create_account  if @account.nil?
     upgrade_account if @account.ostatus?
-    old_public_key = @account.public_key
     update_account
-    RefollowWorker.perform_async(@account.id) if old_public_key != @account.public_key
+    RefollowWorker.perform_async(@account.id) if !old_public_key.nil? && old_public_key != @account.public_key
 
     @account
   rescue Oj::ParseError
diff --git a/app/services/resolve_remote_account_service.rb b/app/services/resolve_remote_account_service.rb
index 75360150151e1e5af92c2de4c343c1ef5c19ef00..7031c98f5ee4875fbbe6da5a0020ff32d0cdd308 100644
--- a/app/services/resolve_remote_account_service.rb
+++ b/app/services/resolve_remote_account_service.rb
@@ -85,10 +85,8 @@ class ResolveRemoteAccountService < BaseService
 
   def handle_ostatus
     create_account if @account.nil?
-    old_public_key = @account.public_key
     update_account
     update_account_profile if update_profile?
-    RefollowWorker.perform_async(@account.id) if old_public_key != @account.public_key
   end
 
   def update_profile?
diff --git a/app/workers/refollow_worker.rb b/app/workers/refollow_worker.rb
index 9c42d42712897336d0fd4cd1b589b343cd3e7205..66bcd27c3e40c4816c262cb763fd19c710de744e 100644
--- a/app/workers/refollow_worker.rb
+++ b/app/workers/refollow_worker.rb
@@ -7,6 +7,7 @@ class RefollowWorker
 
   def perform(target_account_id)
     target_account = Account.find(target_account_id)
+    return unless target_account.protocol == :activitypub
 
     target_account.followers.where(domain: nil).find_each do |follower|
       # Locally unfollow remote account