2020-11-19 11:37:49 -05:00
# frozen_string_literal: true
2023-05-24 05:55:40 -04:00
require_relative 'base'
2020-11-19 11:37:49 -05:00
2023-05-23 10:08:26 -04:00
module Mastodon::CLI
2023-05-24 05:55:40 -04:00
class Maintenance < Base
2023-02-17 21:05:57 -05:00
MIN_SUPPORTED_VERSION = 2019_10_01_213028
2023-09-26 05:11:52 -04:00
MAX_SUPPORTED_VERSION = 2023_09_07_150100
2020-11-19 11:37:49 -05:00
# Stubs to enjoy ActiveRecord queries while not depending on a particular
# version of the code/database
class Status < ApplicationRecord ; end
class StatusPin < ApplicationRecord ; end
class Poll < ApplicationRecord ; end
class Report < ApplicationRecord ; end
class Tombstone < ApplicationRecord ; end
class Favourite < ApplicationRecord ; end
class Follow < ApplicationRecord ; end
class FollowRequest < ApplicationRecord ; end
class Block < ApplicationRecord ; end
class Mute < ApplicationRecord ; end
class AccountIdentityProof < ApplicationRecord ; end
class AccountModerationNote < ApplicationRecord ; end
class AccountPin < ApplicationRecord ; end
class ListAccount < ApplicationRecord ; end
class PollVote < ApplicationRecord ; end
class Mention < ApplicationRecord ; end
class AccountDomainBlock < ApplicationRecord ; end
class AnnouncementReaction < ApplicationRecord ; end
class FeaturedTag < ApplicationRecord ; end
class CustomEmoji < ApplicationRecord ; end
class CustomEmojiCategory < ApplicationRecord ; end
class Bookmark < ApplicationRecord ; end
class WebauthnCredential < ApplicationRecord ; end
2021-05-12 17:19:44 -04:00
class FollowRecommendationSuppression < ApplicationRecord ; end
class CanonicalEmailBlock < ApplicationRecord ; end
2022-03-12 02:33:11 -05:00
class Appeal < ApplicationRecord ; end
2022-11-13 15:01:38 -05:00
class Webhook < ApplicationRecord ; end
2023-09-26 05:11:52 -04:00
class BulkImport < ApplicationRecord ; end
class SoftwareUpdate < ApplicationRecord ; end
2020-11-19 11:37:49 -05:00
class PreviewCard < ApplicationRecord
self . inheritance_column = false
end
class MediaAttachment < ApplicationRecord
self . inheritance_column = nil
end
class AccountStat < ApplicationRecord
belongs_to :account , inverse_of : :account_stat
end
2020-12-03 20:23:51 -05:00
# Dummy class, to make migration possible across version changes
2020-11-19 11:37:49 -05:00
class Account < ApplicationRecord
has_one :user , inverse_of : :account
has_one :account_stat , inverse_of : :account
scope :local , - > { where ( domain : nil ) }
def local?
domain . nil?
end
def acct
local? ? username : " #{ username } @ #{ domain } "
end
2020-12-03 20:23:51 -05:00
2023-12-01 06:00:41 -05:00
# This is a duplicate of the Account::Merging concern because we need it
# to be independent from code version.
2020-12-03 20:23:51 -05:00
def merge_with! ( other_account )
# Since it's the same remote resource, the remote resource likely
# already believes we are following/blocking, so it's safe to
# re-attribute the relationships too. However, during the presence
# of the index bug users could have *also* followed the reference
# account already, therefore mass update will not work and we need
# to check for (and skip past) uniqueness errors
owned_classes = [
Status , StatusPin , MediaAttachment , Poll , Report , Tombstone , Favourite ,
2022-01-23 09:46:30 -05:00
Follow , FollowRequest , Block , Mute ,
2020-12-03 20:23:51 -05:00
AccountModerationNote , AccountPin , AccountStat , ListAccount ,
PollVote , Mention
]
owned_classes << AccountDeletionRequest if ActiveRecord :: Base . connection . table_exists? ( :account_deletion_requests )
owned_classes << AccountNote if ActiveRecord :: Base . connection . table_exists? ( :account_notes )
2021-05-12 17:19:44 -04:00
owned_classes << FollowRecommendationSuppression if ActiveRecord :: Base . connection . table_exists? ( :follow_recommendation_suppressions )
2022-01-23 09:46:30 -05:00
owned_classes << AccountIdentityProof if ActiveRecord :: Base . connection . table_exists? ( :account_identity_proofs )
2022-03-12 02:33:11 -05:00
owned_classes << Appeal if ActiveRecord :: Base . connection . table_exists? ( :appeals )
2023-09-26 05:11:52 -04:00
owned_classes << BulkImport if ActiveRecord :: Base . connection . table_exists? ( :bulk_imports )
2020-12-03 20:23:51 -05:00
owned_classes . each do | klass |
klass . where ( account_id : other_account . id ) . find_each do | record |
2023-02-18 17:09:40 -05:00
record . update_attribute ( :account_id , id )
rescue ActiveRecord :: RecordNotUnique
next
2020-12-03 20:23:51 -05:00
end
end
target_classes = [ Follow , FollowRequest , Block , Mute , AccountModerationNote , AccountPin ]
target_classes << AccountNote if ActiveRecord :: Base . connection . table_exists? ( :account_notes )
target_classes . each do | klass |
klass . where ( target_account_id : other_account . id ) . find_each do | record |
2023-02-18 17:09:40 -05:00
record . update_attribute ( :target_account_id , id )
rescue ActiveRecord :: RecordNotUnique
next
2020-12-03 20:23:51 -05:00
end
end
2021-05-12 17:19:44 -04:00
if ActiveRecord :: Base . connection . table_exists? ( :canonical_email_blocks )
CanonicalEmailBlock . where ( reference_account_id : other_account . id ) . find_each do | record |
record . update_attribute ( :reference_account_id , id )
end
end
2022-03-12 02:33:11 -05:00
if ActiveRecord :: Base . connection . table_exists? ( :appeals )
Appeal . where ( account_warning_id : other_account . id ) . find_each do | record |
record . update_attribute ( :account_warning_id , id )
end
end
2020-12-03 20:23:51 -05:00
end
2020-11-19 11:37:49 -05:00
end
class User < ApplicationRecord
belongs_to :account , inverse_of : :user
end
desc 'fix-duplicates' , 'Fix duplicates in database and rebuild indexes'
long_desc << ~ LONG_DESC
Delete or merge duplicate accounts , statuses , emojis , etc . and rebuild indexes .
This is useful if your database indexes are corrupted because of issues such as https : / / wiki . postgresql . org / wiki / Locale_data_changes
Mastodon has to be stopped to run this task , which will take a long time and may be destructive .
LONG_DESC
def fix_duplicates
2023-11-07 10:21:58 -05:00
verify_system_ready!
2022-01-23 09:46:30 -05:00
2023-11-07 10:21:58 -05:00
process_deduplications
2020-11-19 11:37:49 -05:00
2023-11-07 10:21:58 -05:00
deduplication_cleanup_tasks
say 'Finished!'
end
2020-11-19 11:37:49 -05:00
2023-11-07 10:21:58 -05:00
private
def verify_system_ready!
verify_schema_version!
verify_sidekiq_not_active!
verify_backup_warning!
end
def process_deduplications
2020-11-19 11:37:49 -05:00
deduplicate_users!
deduplicate_account_domain_blocks!
deduplicate_account_identity_proofs!
deduplicate_announcement_reactions!
deduplicate_conversations!
deduplicate_custom_emojis!
deduplicate_custom_emoji_categories!
deduplicate_domain_allows!
deduplicate_domain_blocks!
deduplicate_unavailable_domains!
deduplicate_email_domain_blocks!
deduplicate_media_attachments!
deduplicate_preview_cards!
deduplicate_statuses!
2021-03-19 06:07:56 -04:00
deduplicate_accounts!
2020-11-19 11:37:49 -05:00
deduplicate_tags!
deduplicate_webauthn_credentials!
2022-11-13 15:01:38 -05:00
deduplicate_webhooks!
2023-09-26 05:11:52 -04:00
deduplicate_software_updates!
2023-11-07 10:21:58 -05:00
end
2020-11-19 11:37:49 -05:00
2023-11-07 10:21:58 -05:00
def deduplication_cleanup_tasks
refresh_instances_view if schema_has_instances_view?
2020-11-19 11:37:49 -05:00
Rails . cache . clear
2023-11-07 10:21:58 -05:00
end
2020-11-19 11:37:49 -05:00
2023-11-07 10:21:58 -05:00
def refresh_instances_view
Scenic . database . refresh_materialized_view ( 'instances' , concurrently : true , cascade : false )
2020-11-19 11:37:49 -05:00
end
2023-11-07 10:21:58 -05:00
def schema_has_instances_view?
ActiveRecord :: Migrator . current_version > = 2020_12_06_004238
end
def verify_schema_version!
if ActiveRecord :: Migrator . current_version < MIN_SUPPORTED_VERSION
say 'Your version of the database schema is too old and is not supported by this script.' , :red
say 'Please update to at least Mastodon 3.0.0 before running this script.' , :red
exit ( 1 )
elsif ActiveRecord :: Migrator . current_version > MAX_SUPPORTED_VERSION
say 'Your version of the database schema is more recent than this script, this may cause unexpected errors.' , :yellow
exit ( 1 ) unless yes? ( 'Continue anyway? (Yes/No)' )
end
end
def verify_sidekiq_not_active!
if Sidekiq :: ProcessSet . new . any?
say 'It seems Sidekiq is running. All Mastodon processes need to be stopped when using this script.' , :red
exit ( 1 )
end
end
def verify_backup_warning!
say 'This task will take a long time to run and is potentially destructive.' , :yellow
say 'Please make sure to stop Mastodon and have a backup.' , :yellow
exit ( 1 ) unless yes? ( 'Continue? (Yes/No)' )
end
2020-11-19 11:37:49 -05:00
def deduplicate_accounts!
remove_index_if_exists! ( :accounts , 'index_accounts_on_username_and_domain_lower' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.'
2020-11-19 11:37:49 -05:00
find_duplicate_accounts . each do | row |
accounts = Account . where ( id : row [ 'ids' ] . split ( ',' ) ) . to_a
if accounts . first . local?
deduplicate_local_accounts! ( accounts )
else
deduplicate_remote_accounts! ( accounts )
end
end
2023-05-31 13:40:16 -04:00
say 'Restoring index_accounts_on_username_and_domain_lower…'
2023-02-17 21:05:57 -05:00
if ActiveRecord :: Migrator . current_version < 2020_06_20_164023
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . add_index :accounts , 'lower (username), lower(domain)' , name : 'index_accounts_on_username_and_domain_lower' , unique : true
else
ActiveRecord :: Base . connection . add_index :accounts , " lower (username), COALESCE(lower(domain), '') " , name : 'index_accounts_on_username_and_domain_lower' , unique : true
end
2021-01-08 21:51:32 -05:00
2023-05-31 13:40:16 -04:00
say 'Reindexing textual indexes on accounts…'
2021-01-08 21:51:32 -05:00
ActiveRecord :: Base . connection . execute ( 'REINDEX INDEX search_index;' )
ActiveRecord :: Base . connection . execute ( 'REINDEX INDEX index_accounts_on_uri;' )
ActiveRecord :: Base . connection . execute ( 'REINDEX INDEX index_accounts_on_url;' )
2023-09-26 05:11:52 -04:00
ActiveRecord :: Base . connection . execute ( 'REINDEX INDEX index_accounts_on_domain_and_id;' ) if ActiveRecord :: Migrator . current_version > = 2023_05_24_190515
2020-11-19 11:37:49 -05:00
end
def deduplicate_users!
remove_index_if_exists! ( :users , 'index_users_on_confirmation_token' )
remove_index_if_exists! ( :users , 'index_users_on_email' )
remove_index_if_exists! ( :users , 'index_users_on_remember_token' )
remove_index_if_exists! ( :users , 'index_users_on_reset_password_token' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating user records…'
2020-11-19 11:37:49 -05:00
# Deduplicating email
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1 " ) . each do | row |
users = User . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :updated_at ) . reverse
ref_user = users . shift
2023-05-31 13:40:16 -04:00
say " Multiple users registered with e-mail address #{ ref_user . email } . " , :yellow
2023-11-07 08:01:09 -05:00
say " e-mail will be disabled for the following accounts: #{ user . map { | user | user . account . acct } . join ( ', ' ) } " , :yellow
2023-05-31 13:40:16 -04:00
say 'Please reach out to them and set another address with `tootctl account modify` or delete them.' , :yellow
2020-11-19 11:37:49 -05:00
2023-05-30 10:08:47 -04:00
users . each_with_index do | user , index |
user . update! ( email : " #{ index } " + user . email )
2020-11-19 11:37:49 -05:00
end
end
2023-05-26 03:42:16 -04:00
deduplicate_users_process_confirmation_token
deduplicate_users_process_remember_token
deduplicate_users_process_password_token
2023-05-31 13:40:16 -04:00
say 'Restoring users indexes…'
2023-05-26 03:42:16 -04:00
ActiveRecord :: Base . connection . add_index :users , [ 'confirmation_token' ] , name : 'index_users_on_confirmation_token' , unique : true
ActiveRecord :: Base . connection . add_index :users , [ 'email' ] , name : 'index_users_on_email' , unique : true
ActiveRecord :: Base . connection . add_index :users , [ 'remember_token' ] , name : 'index_users_on_remember_token' , unique : true if ActiveRecord :: Migrator . current_version < 2022_01_18_183010
if ActiveRecord :: Migrator . current_version < 2022_03_10_060641
ActiveRecord :: Base . connection . add_index :users , [ 'reset_password_token' ] , name : 'index_users_on_reset_password_token' , unique : true
else
ActiveRecord :: Base . connection . add_index :users , [ 'reset_password_token' ] , name : 'index_users_on_reset_password_token' , unique : true , where : 'reset_password_token IS NOT NULL' , opclass : :text_pattern_ops
end
2023-09-26 05:11:52 -04:00
ActiveRecord :: Base . connection . execute ( 'REINDEX INDEX index_users_on_unconfirmed_email;' ) if ActiveRecord :: Migrator . current_version > = 2023_07_02_151753
2023-05-26 03:42:16 -04:00
end
def deduplicate_users_process_confirmation_token
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1 " ) . each do | row |
users = User . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :created_at ) . reverse . drop ( 1 )
2023-11-07 08:01:09 -05:00
say " Unsetting confirmation token for those accounts: #{ users . map { | user | user . account . acct } . join ( ', ' ) } " , :yellow
2020-11-19 11:37:49 -05:00
users . each do | user |
user . update! ( confirmation_token : nil )
end
end
2023-05-26 03:42:16 -04:00
end
2020-11-19 11:37:49 -05:00
2023-05-26 03:42:16 -04:00
def deduplicate_users_process_remember_token
2023-02-17 21:05:57 -05:00
if ActiveRecord :: Migrator . current_version < 2022_01_18_183010
2022-01-23 09:46:30 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1 " ) . each do | row |
users = User . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :updated_at ) . reverse . drop ( 1 )
2023-11-07 08:01:09 -05:00
say " Unsetting remember token for those accounts: #{ users . map { | user | user . account . acct } . join ( ', ' ) } " , :yellow
2020-11-19 11:37:49 -05:00
2022-01-23 09:46:30 -05:00
users . each do | user |
user . update! ( remember_token : nil )
end
2020-11-19 11:37:49 -05:00
end
end
2023-05-26 03:42:16 -04:00
end
2020-11-19 11:37:49 -05:00
2023-05-26 03:42:16 -04:00
def deduplicate_users_process_password_token
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1 " ) . each do | row |
users = User . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :updated_at ) . reverse . drop ( 1 )
2023-11-07 08:01:09 -05:00
say " Unsetting password reset token for those accounts: #{ users . map { | user | user . account . acct } . join ( ', ' ) } " , :yellow
2020-11-19 11:37:49 -05:00
users . each do | user |
user . update! ( reset_password_token : nil )
end
end
end
def deduplicate_account_domain_blocks!
remove_index_if_exists! ( :account_domain_blocks , 'index_account_domain_blocks_on_account_id_and_domain' )
2023-05-31 13:40:16 -04:00
say 'Removing duplicate account domain blocks…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1 " ) . each do | row |
AccountDomainBlock . where ( id : row [ 'ids' ] . split ( ',' ) . drop ( 1 ) ) . delete_all
end
2023-05-31 13:40:16 -04:00
say 'Restoring account domain blocks indexes…'
2023-02-20 00:14:10 -05:00
ActiveRecord :: Base . connection . add_index :account_domain_blocks , %w( account_id domain ) , name : 'index_account_domain_blocks_on_account_id_and_domain' , unique : true
2020-11-19 11:37:49 -05:00
end
def deduplicate_account_identity_proofs!
2022-01-23 09:46:30 -05:00
return unless ActiveRecord :: Base . connection . table_exists? ( :account_identity_proofs )
2020-11-19 11:37:49 -05:00
remove_index_if_exists! ( :account_identity_proofs , 'index_account_proofs_on_account_and_provider_and_username' )
2023-05-31 13:40:16 -04:00
say 'Removing duplicate account identity proofs…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1 " ) . each do | row |
AccountIdentityProof . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse . drop ( 1 ) . each ( & :destroy )
end
2023-05-31 13:40:16 -04:00
say 'Restoring account identity proofs indexes…'
2023-02-20 00:14:10 -05:00
ActiveRecord :: Base . connection . add_index :account_identity_proofs , %w( account_id provider provider_username ) , name : 'index_account_proofs_on_account_and_provider_and_username' , unique : true
2020-11-19 11:37:49 -05:00
end
def deduplicate_announcement_reactions!
return unless ActiveRecord :: Base . connection . table_exists? ( :announcement_reactions )
remove_index_if_exists! ( :announcement_reactions , 'index_announcement_reactions_on_account_id_and_announcement_id' )
2023-05-31 13:40:16 -04:00
say 'Removing duplicate account identity proofs…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1 " ) . each do | row |
AnnouncementReaction . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse . drop ( 1 ) . each ( & :destroy )
end
2023-05-31 13:40:16 -04:00
say 'Restoring announcement_reactions indexes…'
2023-02-20 00:14:10 -05:00
ActiveRecord :: Base . connection . add_index :announcement_reactions , %w( account_id announcement_id name ) , name : 'index_announcement_reactions_on_account_id_and_announcement_id' , unique : true
2020-11-19 11:37:49 -05:00
end
def deduplicate_conversations!
remove_index_if_exists! ( :conversations , 'index_conversations_on_uri' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating conversations…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1 " ) . each do | row |
conversations = Conversation . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse
ref_conversation = conversations . shift
conversations . each do | other |
merge_conversations! ( ref_conversation , other )
other . destroy
end
end
2023-05-31 13:40:16 -04:00
say 'Restoring conversations indexes…'
2023-02-17 21:05:57 -05:00
if ActiveRecord :: Migrator . current_version < 2022_03_07_083603
2022-03-12 02:33:11 -05:00
ActiveRecord :: Base . connection . add_index :conversations , [ 'uri' ] , name : 'index_conversations_on_uri' , unique : true
else
ActiveRecord :: Base . connection . add_index :conversations , [ 'uri' ] , name : 'index_conversations_on_uri' , unique : true , where : 'uri IS NOT NULL' , opclass : :text_pattern_ops
end
2020-11-19 11:37:49 -05:00
end
def deduplicate_custom_emojis!
remove_index_if_exists! ( :custom_emojis , 'index_custom_emojis_on_shortcode_and_domain' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating custom_emojis…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1 " ) . each do | row |
emojis = CustomEmoji . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse
ref_emoji = emojis . shift
emojis . each do | other |
merge_custom_emojis! ( ref_emoji , other )
other . destroy
end
end
2023-05-31 13:40:16 -04:00
say 'Restoring custom_emojis indexes…'
2023-02-20 00:14:10 -05:00
ActiveRecord :: Base . connection . add_index :custom_emojis , %w( shortcode domain ) , name : 'index_custom_emojis_on_shortcode_and_domain' , unique : true
2020-11-19 11:37:49 -05:00
end
def deduplicate_custom_emoji_categories!
remove_index_if_exists! ( :custom_emoji_categories , 'index_custom_emoji_categories_on_name' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating custom_emoji_categories…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1 " ) . each do | row |
categories = CustomEmojiCategory . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse
ref_category = categories . shift
categories . each do | other |
merge_custom_emoji_categories! ( ref_category , other )
other . destroy
end
end
2023-05-31 13:40:16 -04:00
say 'Restoring custom_emoji_categories indexes…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . add_index :custom_emoji_categories , [ 'name' ] , name : 'index_custom_emoji_categories_on_name' , unique : true
end
def deduplicate_domain_allows!
remove_index_if_exists! ( :domain_allows , 'index_domain_allows_on_domain' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating domain_allows…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1 " ) . each do | row |
DomainAllow . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse . drop ( 1 ) . each ( & :destroy )
end
2023-05-31 13:40:16 -04:00
say 'Restoring domain_allows indexes…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . add_index :domain_allows , [ 'domain' ] , name : 'index_domain_allows_on_domain' , unique : true
end
def deduplicate_domain_blocks!
remove_index_if_exists! ( :domain_blocks , 'index_domain_blocks_on_domain' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating domain_allows…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1 " ) . each do | row |
domain_blocks = DomainBlock . where ( id : row [ 'ids' ] . split ( ',' ) ) . by_severity . reverse . to_a
reject_media = domain_blocks . any? ( & :reject_media? )
reject_reports = domain_blocks . any? ( & :reject_reports? )
reference_block = domain_blocks . shift
private_comment = domain_blocks . reduce ( reference_block . private_comment . presence ) { | a , b | a || b . private_comment . presence }
public_comment = domain_blocks . reduce ( reference_block . public_comment . presence ) { | a , b | a || b . public_comment . presence }
reference_block . update! ( reject_media : reject_media , reject_reports : reject_reports , private_comment : private_comment , public_comment : public_comment )
domain_blocks . each ( & :destroy )
end
2023-05-31 13:40:16 -04:00
say 'Restoring domain_blocks indexes…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . add_index :domain_blocks , [ 'domain' ] , name : 'index_domain_blocks_on_domain' , unique : true
end
def deduplicate_unavailable_domains!
return unless ActiveRecord :: Base . connection . table_exists? ( :unavailable_domains )
remove_index_if_exists! ( :unavailable_domains , 'index_unavailable_domains_on_domain' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating unavailable_domains…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1 " ) . each do | row |
UnavailableDomain . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse . drop ( 1 ) . each ( & :destroy )
end
2023-05-31 13:40:16 -04:00
say 'Restoring domain_allows indexes…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . add_index :unavailable_domains , [ 'domain' ] , name : 'index_unavailable_domains_on_domain' , unique : true
end
def deduplicate_email_domain_blocks!
remove_index_if_exists! ( :email_domain_blocks , 'index_email_domain_blocks_on_domain' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating email_domain_blocks…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1 " ) . each do | row |
domain_blocks = EmailDomainBlock . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by { | b | b . parent . nil? ? 1 : 0 } . to_a
domain_blocks . drop ( 1 ) . each ( & :destroy )
end
2023-05-31 13:40:16 -04:00
say 'Restoring email_domain_blocks indexes…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . add_index :email_domain_blocks , [ 'domain' ] , name : 'index_email_domain_blocks_on_domain' , unique : true
end
def deduplicate_media_attachments!
remove_index_if_exists! ( :media_attachments , 'index_media_attachments_on_shortcode' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating media_attachments…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1 " ) . each do | row |
MediaAttachment . where ( id : row [ 'ids' ] . split ( ',' ) . drop ( 1 ) ) . update_all ( shortcode : nil )
end
2023-05-31 13:40:16 -04:00
say 'Restoring media_attachments indexes…'
2023-02-17 21:05:57 -05:00
if ActiveRecord :: Migrator . current_version < 2022_03_10_060626
2022-03-12 02:33:11 -05:00
ActiveRecord :: Base . connection . add_index :media_attachments , [ 'shortcode' ] , name : 'index_media_attachments_on_shortcode' , unique : true
else
ActiveRecord :: Base . connection . add_index :media_attachments , [ 'shortcode' ] , name : 'index_media_attachments_on_shortcode' , unique : true , where : 'shortcode IS NOT NULL' , opclass : :text_pattern_ops
end
2020-11-19 11:37:49 -05:00
end
def deduplicate_preview_cards!
remove_index_if_exists! ( :preview_cards , 'index_preview_cards_on_url' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating preview_cards…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1 " ) . each do | row |
PreviewCard . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse . drop ( 1 ) . each ( & :destroy )
end
2023-05-31 13:40:16 -04:00
say 'Restoring preview_cards indexes…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . add_index :preview_cards , [ 'url' ] , name : 'index_preview_cards_on_url' , unique : true
end
def deduplicate_statuses!
remove_index_if_exists! ( :statuses , 'index_statuses_on_uri' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating statuses…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1 " ) . each do | row |
statuses = Status . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id )
ref_status = statuses . shift
statuses . each do | status |
merge_statuses! ( ref_status , status ) if status . account_id == ref_status . account_id
status . destroy
end
end
2023-05-31 13:40:16 -04:00
say 'Restoring statuses indexes…'
2023-02-17 21:05:57 -05:00
if ActiveRecord :: Migrator . current_version < 2022_03_10_060706
2022-03-12 02:33:11 -05:00
ActiveRecord :: Base . connection . add_index :statuses , [ 'uri' ] , name : 'index_statuses_on_uri' , unique : true
else
ActiveRecord :: Base . connection . add_index :statuses , [ 'uri' ] , name : 'index_statuses_on_uri' , unique : true , where : 'uri IS NOT NULL' , opclass : :text_pattern_ops
end
2020-11-19 11:37:49 -05:00
end
def deduplicate_tags!
remove_index_if_exists! ( :tags , 'index_tags_on_name_lower' )
2022-11-14 02:33:48 -05:00
remove_index_if_exists! ( :tags , 'index_tags_on_name_lower_btree' )
2020-11-19 11:37:49 -05:00
2023-05-31 13:40:16 -04:00
say 'Deduplicating tags…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1 " ) . each do | row |
tags = Tag . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by { | t | [ t . usable? , t . trendable? , t . listable? ] . count ( false ) }
ref_tag = tags . shift
tags . each do | tag |
merge_tags! ( ref_tag , tag )
tag . destroy
end
end
2023-05-31 13:40:16 -04:00
say 'Restoring tags indexes…'
2023-02-17 21:05:57 -05:00
if ActiveRecord :: Migrator . current_version < 2021_04_21_121431
2022-11-13 15:01:38 -05:00
ActiveRecord :: Base . connection . add_index :tags , 'lower((name)::text)' , name : 'index_tags_on_name_lower' , unique : true
else
ActiveRecord :: Base . connection . execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)'
2021-05-12 17:19:44 -04:00
end
2020-11-19 11:37:49 -05:00
end
def deduplicate_webauthn_credentials!
return unless ActiveRecord :: Base . connection . table_exists? ( :webauthn_credentials )
remove_index_if_exists! ( :webauthn_credentials , 'index_webauthn_credentials_on_external_id' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating webauthn_credentials…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1 " ) . each do | row |
WebauthnCredential . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse . drop ( 1 ) . each ( & :destroy )
end
2023-05-31 13:40:16 -04:00
say 'Restoring webauthn_credentials indexes…'
2020-11-19 11:37:49 -05:00
ActiveRecord :: Base . connection . add_index :webauthn_credentials , [ 'external_id' ] , name : 'index_webauthn_credentials_on_external_id' , unique : true
end
2022-11-13 15:01:38 -05:00
def deduplicate_webhooks!
return unless ActiveRecord :: Base . connection . table_exists? ( :webhooks )
remove_index_if_exists! ( :webhooks , 'index_webhooks_on_url' )
2023-05-31 13:40:16 -04:00
say 'Deduplicating webhooks…'
2022-11-13 15:01:38 -05:00
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1 " ) . each do | row |
Webhooks . where ( id : row [ 'ids' ] . split ( ',' ) ) . sort_by ( & :id ) . reverse . drop ( 1 ) . each ( & :destroy )
end
2023-05-31 13:40:16 -04:00
say 'Restoring webhooks indexes…'
2022-11-13 15:01:38 -05:00
ActiveRecord :: Base . connection . add_index :webhooks , [ 'url' ] , name : 'index_webhooks_on_url' , unique : true
end
2023-09-26 05:11:52 -04:00
def deduplicate_software_updates!
# Not bothering with this, it's data that will be recovered with the scheduler
SoftwareUpdate . delete_all
end
2020-11-19 11:37:49 -05:00
def deduplicate_local_accounts! ( accounts )
accounts = accounts . sort_by ( & :id ) . reverse
2023-05-31 13:40:16 -04:00
say " Multiple local accounts were found for username ' #{ accounts . first . username } '. " , :yellow
say 'All those accounts are distinct accounts but only the most recently-created one is fully-functional.' , :yellow
2020-11-19 11:37:49 -05:00
accounts . each_with_index do | account , idx |
2023-05-31 13:40:16 -04:00
say format (
2023-05-30 10:21:53 -04:00
'%<index>2d. %<username>s: created at: %<created_at>s; updated at: %<updated_at>s; last logged in at: %<last_log_in_at>s; statuses: %<status_count>5d; last status at: %<last_status_at>s' ,
index : idx ,
username : account . username ,
created_at : account . created_at ,
updated_at : account . updated_at ,
last_log_in_at : account . user & . last_sign_in_at & . to_s || 'N/A' ,
status_count : account . account_stat & . statuses_count || 0 ,
last_status_at : account . account_stat & . last_status_at || 'N/A'
)
2020-11-19 11:37:49 -05:00
end
2023-05-31 13:40:16 -04:00
say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
2020-11-19 11:37:49 -05:00
2023-05-31 13:40:16 -04:00
ref_id = ask ( 'Account to keep unchanged:' ) do | q |
2020-11-19 11:37:49 -05:00
q . required true
q . default 0
q . convert :int
end
accounts . delete_at ( ref_id )
i = 0
accounts . each do | account |
i += 1
username = account . username + " _ #{ i } "
while Account . local . exists? ( username : username )
i += 1
username = account . username + " _ #{ i } "
end
account . update! ( username : username )
end
end
def deduplicate_remote_accounts! ( accounts )
accounts = accounts . sort_by ( & :updated_at ) . reverse
reference_account = accounts . shift
accounts . each do | other_account |
if other_account . public_key == reference_account . public_key
# The accounts definitely point to the same resource, so
# it's safe to re-attribute content and relationships
2020-11-23 11:50:16 -05:00
reference_account . merge_with! ( other_account )
2020-11-19 11:37:49 -05:00
end
other_account . destroy
end
end
def merge_conversations! ( main_conv , duplicate_conv )
owned_classes = [ ConversationMute , AccountConversation ]
owned_classes . each do | klass |
klass . where ( conversation_id : duplicate_conv . id ) . find_each do | record |
2023-02-18 17:09:40 -05:00
record . update_attribute ( :account_id , main_conv . id )
rescue ActiveRecord :: RecordNotUnique
next
2020-11-19 11:37:49 -05:00
end
end
end
def merge_custom_emojis! ( main_emoji , duplicate_emoji )
owned_classes = [ AnnouncementReaction ]
owned_classes . each do | klass |
klass . where ( custom_emoji_id : duplicate_emoji . id ) . update_all ( custom_emoji_id : main_emoji . id )
end
end
def merge_custom_emoji_categories! ( main_category , duplicate_category )
owned_classes = [ CustomEmoji ]
owned_classes . each do | klass |
klass . where ( category_id : duplicate_category . id ) . update_all ( category_id : main_category . id )
end
end
def merge_statuses! ( main_status , duplicate_status )
owned_classes = [ Favourite , Mention , Poll ]
owned_classes << Bookmark if ActiveRecord :: Base . connection . table_exists? ( :bookmarks )
owned_classes . each do | klass |
klass . where ( status_id : duplicate_status . id ) . find_each do | record |
record . update_attribute ( :status_id , main_status . id )
rescue ActiveRecord :: RecordNotUnique
next
end
end
2023-02-18 17:09:40 -05:00
StatusPin . where ( account_id : main_status . account_id , status_id : duplicate_status . id ) . find_each do | record |
record . update_attribute ( :status_id , main_status . id )
rescue ActiveRecord :: RecordNotUnique
next
end
2020-11-19 11:37:49 -05:00
Status . where ( in_reply_to_id : duplicate_status . id ) . find_each do | record |
2023-02-18 17:09:40 -05:00
record . update_attribute ( :in_reply_to_id , main_status . id )
rescue ActiveRecord :: RecordNotUnique
next
2020-11-19 11:37:49 -05:00
end
Status . where ( reblog_of_id : duplicate_status . id ) . find_each do | record |
2023-02-18 17:09:40 -05:00
record . update_attribute ( :reblog_of_id , main_status . id )
rescue ActiveRecord :: RecordNotUnique
next
2020-11-19 11:37:49 -05:00
end
end
def merge_tags! ( main_tag , duplicate_tag )
[ FeaturedTag ] . each do | klass |
klass . where ( tag_id : duplicate_tag . id ) . find_each do | record |
2023-02-18 17:09:40 -05:00
record . update_attribute ( :tag_id , main_tag . id )
rescue ActiveRecord :: RecordNotUnique
next
2020-11-19 11:37:49 -05:00
end
end
end
def find_duplicate_accounts
ActiveRecord :: Base . connection . select_all ( " SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1 " )
end
def remove_index_if_exists! ( table , name )
ActiveRecord :: Base . connection . remove_index ( table , name : name )
2023-05-02 06:57:11 -04:00
rescue ArgumentError , ActiveRecord :: StatementInvalid
2020-11-19 11:37:49 -05:00
nil
end
end
end