Add trending links (#16917)

* Add trending links

* Add overriding specific links trendability

* Add link type to preview cards and only trend articles

Change trends review notifications from being sent every 5 minutes to being sent every 2 hours

Change threshold from 5 unique accounts to 15 unique accounts

* Fix tests
This commit is contained in:
Eugen Rochko 2021-11-25 13:07:38 +01:00 committed by GitHub
parent 46e62fc4b3
commit 6e50134a42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
97 changed files with 2071 additions and 722 deletions

View file

@ -31,7 +31,7 @@ class TagsIndex < Chewy::Index
end end
field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? } field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } } field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day.accounts } }
field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at } field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
end end
end end

View file

@ -4,7 +4,7 @@ module Admin
class DashboardController < BaseController class DashboardController < BaseController
def index def index
@system_checks = Admin::SystemCheck.perform @system_checks = Admin::SystemCheck.perform
@time_period = (1.month.ago.to_date...Time.now.utc.to_date) @time_period = (29.days.ago.to_date...Time.now.utc.to_date)
@pending_users_count = User.pending.count @pending_users_count = User.pending.count
@pending_reports_count = Report.unresolved.count @pending_reports_count = Report.unresolved.count
@pending_tags_count = Tag.pending_review.count @pending_tags_count = Tag.pending_review.count

View file

@ -2,38 +2,12 @@
module Admin module Admin
class TagsController < BaseController class TagsController < BaseController
before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all] before_action :set_tag
before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all]
before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all]
def index
authorize :tag, :index?
@tags = filtered_tags.page(params[:page])
@form = Form::TagBatch.new
end
def batch
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_tags_path(filter_params)
end
def approve_all
Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'approve').save
redirect_to admin_tags_path(filter_params)
end
def reject_all
Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'reject').save
redirect_to admin_tags_path(filter_params)
end
def show def show
authorize @tag, :show? authorize @tag, :show?
@time_period = (6.days.ago.to_date...Time.now.utc.to_date)
end end
def update def update
@ -52,52 +26,8 @@ module Admin
@tag = Tag.find(params[:id]) @tag = Tag.find(params[:id])
end end
def set_usage_by_domain
@usage_by_domain = @tag.statuses
.with_public_visibility
.excluding_silenced_accounts
.where(Status.arel_table[:id].gteq(Mastodon::Snowflake.id_at(Time.now.utc.beginning_of_day)))
.joins(:account)
.group('accounts.domain')
.reorder(statuses_count: :desc)
.pluck(Arel.sql('accounts.domain, count(*) AS statuses_count'))
end
def set_counters
@accounts_today = @tag.history.first[:accounts]
@accounts_week = Redis.current.pfcount(*current_week_days.map { |day| "activity:tags:#{@tag.id}:#{day}:accounts" })
end
def filtered_tags
TagFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
end
def tag_params def tag_params
params.require(:tag).permit(:name, :trendable, :usable, :listable) params.require(:tag).permit(:name, :trendable, :usable, :listable)
end end
def current_week_days
now = Time.now.utc.beginning_of_day.to_date
(Date.commercial(now.cwyear, now.cweek)..now).map do |date|
date.to_time(:utc).beginning_of_day.to_i
end
end
def form_tag_batch_params
params.require(:form_tag_batch).permit(:action, tag_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end end
end end

View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseController
def index
authorize :preview_card_provider, :index?
@preview_card_providers = filtered_preview_card_providers.page(params[:page])
@form = Form::PreviewCardProviderBatch.new
end
def batch
@form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_links_preview_card_providers_path(filter_params)
end
private
def filtered_preview_card_providers
PreviewCardProviderFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS)
end
def form_preview_card_provider_batch_params
params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end

View file

@ -0,0 +1,45 @@
# frozen_string_literal: true
class Admin::Trends::LinksController < Admin::BaseController
def index
authorize :preview_card, :index?
@preview_cards = filtered_preview_cards.page(params[:page])
@form = Form::PreviewCardBatch.new
end
def batch
@form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_links_path(filter_params)
end
private
def filtered_preview_cards
PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
end
def filter_params
params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS)
end
def form_preview_card_batch_params
params.require(:form_preview_card_batch).permit(:action, preview_card_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:approve_all]
'approve_all'
elsif params[:reject]
'reject'
elsif params[:reject_all]
'reject_all'
end
end
end

View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Trends::TagsController < Admin::BaseController
def index
authorize :tag, :index?
@tags = filtered_tags.page(params[:page])
@form = Form::TagBatch.new
end
def batch
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_tags_path(filter_params)
end
private
def filtered_tags
TagFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
end
def form_tag_batch_params
params.require(:form_tag_batch).permit(:action, tag_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end

View file

@ -17,7 +17,8 @@ class Api::V1::Admin::DimensionsController < Api::BaseController
params[:keys], params[:keys],
params[:start_at], params[:start_at],
params[:end_at], params[:end_at],
params[:limit] params[:limit],
params
) )
end end
end end

View file

@ -16,7 +16,8 @@ class Api::V1::Admin::MeasuresController < Api::BaseController
@measures = Admin::Metrics::Measure.retrieve( @measures = Admin::Metrics::Measure.retrieve(
params[:keys], params[:keys],
params[:start_at], params[:start_at],
params[:end_at] params[:end_at],
params
) )
end end
end end

View file

@ -0,0 +1,16 @@
# frozen_string_literal: true
class Api::V1::Admin::Trends::TagsController < Api::BaseController
before_action :require_staff!
before_action :set_tags
def index
render json: @tags, each_serializer: REST::Admin::TagSerializer
end
private
def set_tags
@tags = Trends.tags.get(false, limit_param(10))
end
end

View file

@ -1,16 +0,0 @@
# frozen_string_literal: true
class Api::V1::Admin::TrendsController < Api::BaseController
before_action :require_staff!
before_action :set_trends
def index
render json: @trends, each_serializer: REST::Admin::TagSerializer
end
private
def set_trends
@trends = TrendingTags.get(10, filtered: false)
end
end

View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
class Api::V1::Trends::LinksController < Api::BaseController
before_action :set_links
def index
render json: @links, each_serializer: REST::Trends::LinkSerializer
end
private
def set_links
@links = begin
if Setting.trends
Trends.links.get(true, limit_param(10))
else
[]
end
end
end
end

View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
class Api::V1::Trends::TagsController < Api::BaseController
before_action :set_tags
def index
render json: @tags, each_serializer: REST::TagSerializer
end
private
def set_tags
@tags = begin
if Setting.trends
Trends.tags.get(true, limit_param(10))
else
[]
end
end
end
end

View file

@ -1,15 +0,0 @@
# frozen_string_literal: true
class Api::V1::TrendsController < Api::BaseController
before_action :set_tags
def index
render json: @tags, each_serializer: REST::TagSerializer
end
private
def set_tags
@tags = TrendingTags.get(limit_param(10))
end
end

View file

@ -6,6 +6,8 @@ module Admin::FilterHelper
CustomEmojiFilter::KEYS, CustomEmojiFilter::KEYS,
ReportFilter::KEYS, ReportFilter::KEYS,
TagFilter::KEYS, TagFilter::KEYS,
PreviewCardProviderFilter::KEYS,
PreviewCardFilter::KEYS,
InstanceFilter::KEYS, InstanceFilter::KEYS,
InviteFilter::KEYS, InviteFilter::KEYS,
RelationshipFilter::KEYS, RelationshipFilter::KEYS,

View file

@ -0,0 +1,94 @@
# frozen_string_literal: true
module LanguagesHelper
HUMAN_LOCALES = {
af: 'Afrikaans',
ar: 'العربية',
ast: 'Asturianu',
bg: 'Български',
bn: 'বাংলা',
br: 'Breton',
ca: 'Català',
co: 'Corsu',
cs: 'Čeština',
cy: 'Cymraeg',
da: 'Dansk',
de: 'Deutsch',
el: 'Ελληνικά',
en: 'English',
eo: 'Esperanto',
'es-AR': 'Español (Argentina)',
'es-MX': 'Español (México)',
es: 'Español',
et: 'Eesti',
eu: 'Euskara',
fa: 'فارسی',
fi: 'Suomi',
fr: 'Français',
ga: 'Gaeilge',
gd: 'Gàidhlig',
gl: 'Galego',
he: 'עברית',
hi: 'हिन्दी',
hr: 'Hrvatski',
hu: 'Magyar',
hy: 'Հայերեն',
id: 'Bahasa Indonesia',
io: 'Ido',
is: 'Íslenska',
it: 'Italiano',
ja: '日本語',
ka: 'ქართული',
kab: 'Taqbaylit',
kk: 'Қазақша',
kmr: 'Kurmancî',
kn: 'ಕನ್ನಡ',
ko: '한국어',
ku: 'سۆرانی',
lt: 'Lietuvių',
lv: 'Latviešu',
mk: 'Македонски',
ml: 'മലയാളം',
mr: 'मराठी',
ms: 'Bahasa Melayu',
nl: 'Nederlands',
nn: 'Nynorsk',
no: 'Norsk',
oc: 'Occitan',
pl: 'Polski',
'pt-BR': 'Português (Brasil)',
'pt-PT': 'Português (Portugal)',
pt: 'Português',
ro: 'Română',
ru: 'Русский',
sa: 'संस्कृतम्',
sc: 'Sardu',
si: 'සිංහල',
sk: 'Slovenčina',
sl: 'Slovenščina',
sq: 'Shqip',
'sr-Latn': 'Srpski (latinica)',
sr: 'Српски',
sv: 'Svenska',
ta: 'தமிழ்',
te: 'తెలుగు',
th: 'ไทย',
tr: 'Türkçe',
uk: 'Українська',
ur: 'اُردُو',
vi: 'Tiếng Việt',
zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
'zh-CN': '简体中文',
'zh-HK': '繁體中文(香港)',
'zh-TW': '繁體中文(臺灣)',
zh: '中文',
}.freeze
def human_locale(locale)
if locale == 'und'
I18n.t('generic.none')
else
HUMAN_LOCALES[locale.to_sym] || locale
end
end
end

View file

@ -1,95 +1,8 @@
# frozen_string_literal: true # frozen_string_literal: true
module SettingsHelper module SettingsHelper
HUMAN_LOCALES = {
af: 'Afrikaans',
ar: 'العربية',
ast: 'Asturianu',
bg: 'Български',
bn: 'বাংলা',
br: 'Breton',
ca: 'Català',
co: 'Corsu',
cs: 'Čeština',
cy: 'Cymraeg',
da: 'Dansk',
de: 'Deutsch',
el: 'Ελληνικά',
en: 'English',
eo: 'Esperanto',
'es-AR': 'Español (Argentina)',
'es-MX': 'Español (México)',
es: 'Español',
et: 'Eesti',
eu: 'Euskara',
fa: 'فارسی',
fi: 'Suomi',
fr: 'Français',
ga: 'Gaeilge',
gd: 'Gàidhlig',
gl: 'Galego',
he: 'עברית',
hi: 'हिन्दी',
hr: 'Hrvatski',
hu: 'Magyar',
hy: 'Հայերեն',
id: 'Bahasa Indonesia',
io: 'Ido',
is: 'Íslenska',
it: 'Italiano',
ja: '日本語',
ka: 'ქართული',
kab: 'Taqbaylit',
kk: 'Қазақша',
kmr: 'Kurmancî',
kn: 'ಕನ್ನಡ',
ko: '한국어',
ku: 'سۆرانی',
lt: 'Lietuvių',
lv: 'Latviešu',
mk: 'Македонски',
ml: 'മലയാളം',
mr: 'मराठी',
ms: 'Bahasa Melayu',
nl: 'Nederlands',
nn: 'Nynorsk',
no: 'Norsk',
oc: 'Occitan',
pl: 'Polski',
'pt-BR': 'Português (Brasil)',
'pt-PT': 'Português (Portugal)',
pt: 'Português',
ro: 'Română',
ru: 'Русский',
sa: 'संस्कृतम्',
sc: 'Sardu',
si: 'සිංහල',
sk: 'Slovenčina',
sl: 'Slovenščina',
sq: 'Shqip',
'sr-Latn': 'Srpski (latinica)',
sr: 'Српски',
sv: 'Svenska',
ta: 'தமிழ்',
te: 'తెలుగు',
th: 'ไทย',
tr: 'Türkçe',
uk: 'Українська',
ur: 'اُردُو',
vi: 'Tiếng Việt',
zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
'zh-CN': '简体中文',
'zh-HK': '繁體中文(香港)',
'zh-TW': '繁體中文(臺灣)',
zh: '中文',
}.freeze
def human_locale(locale)
HUMAN_LOCALES[locale]
end
def filterable_languages def filterable_languages
LanguageDetector.instance.language_names.select(&HUMAN_LOCALES.method(:key?)) LanguageDetector.instance.language_names.select(&LanguagesHelper::HUMAN_LOCALES.method(:key?))
end end
def hash_to_object(hash) def hash_to_object(hash)

View file

@ -32,6 +32,7 @@ export default class Counter extends React.PureComponent {
end_at: PropTypes.string.isRequired, end_at: PropTypes.string.isRequired,
label: PropTypes.string.isRequired, label: PropTypes.string.isRequired,
href: PropTypes.string, href: PropTypes.string,
params: PropTypes.object,
}; };
state = { state = {
@ -40,9 +41,9 @@ export default class Counter extends React.PureComponent {
}; };
componentDidMount () { componentDidMount () {
const { measure, start_at, end_at } = this.props; const { measure, start_at, end_at, params } = this.props;
api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at }).then(res => { api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at, [measure]: params }).then(res => {
this.setState({ this.setState({
loading: false, loading: false,
data: res.data, data: res.data,

View file

@ -13,6 +13,7 @@ export default class Dimension extends React.PureComponent {
end_at: PropTypes.string.isRequired, end_at: PropTypes.string.isRequired,
limit: PropTypes.number.isRequired, limit: PropTypes.number.isRequired,
label: PropTypes.string.isRequired, label: PropTypes.string.isRequired,
params: PropTypes.object,
}; };
state = { state = {
@ -21,9 +22,9 @@ export default class Dimension extends React.PureComponent {
}; };
componentDidMount () { componentDidMount () {
const { start_at, end_at, dimension, limit } = this.props; const { start_at, end_at, dimension, limit, params } = this.props;
api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit }).then(res => { api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit, [dimension]: params }).then(res => {
this.setState({ this.setState({
loading: false, loading: false,
data: res.data, data: res.data,

View file

@ -19,7 +19,7 @@ export default class Trends extends React.PureComponent {
componentDidMount () { componentDidMount () {
const { limit } = this.props; const { limit } = this.props;
api().get('/api/v1/admin/trends', { params: { limit } }).then(res => { api().get('/api/v1/admin/trends/tags', { params: { limit } }).then(res => {
this.setState({ this.setState({
loading: false, loading: false,
data: res.data, data: res.data,

View file

@ -325,3 +325,19 @@
margin-top: 10px; margin-top: 10px;
} }
} }
.batch-table__row--muted .pending-account__header {
&,
a,
strong {
color: lighten($ui-base-color, 26%);
}
}
.batch-table__row--attention .pending-account__header {
&,
a,
strong {
color: $gold-star;
}
}

View file

@ -100,6 +100,16 @@
transition: all 200ms ease-out; transition: all 200ms ease-out;
} }
&.positive {
background: lighten($ui-base-color, 4%);
color: $valid-value-color;
}
&.negative {
background: lighten($ui-base-color, 4%);
color: $error-value-color;
}
span { span {
flex: 1 1 auto; flex: 1 1 auto;
} }

View file

@ -129,8 +129,6 @@ class ActivityPub::Activity
end end
def crawl_links(status) def crawl_links(status)
return if status.spoiler_text?
# Spread out crawling randomly to avoid DDoSing the link # Spread out crawling randomly to avoid DDoSing the link
LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id) LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id)
end end

View file

@ -22,9 +22,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
visibility: visibility_from_audience visibility: visibility_from_audience
) )
original_status.tags.each do |tag| Trends.tags.register(@status)
tag.use!(@account) Trends.links.register(@status)
end
distribute(@status) distribute(@status)
end end

View file

@ -164,9 +164,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def attach_tags(status) def attach_tags(status)
@tags.each do |tag| @tags.each do |tag|
status.tags << tag status.tags << tag
tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility? tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
end end
# If we're processing an old status, this may register tags as being used now
# as opposed to when the status was really published, but this is probably
# not a big deal
Trends.tags.register(status)
@mentions.each do |mention| @mentions.each do |mention|
mention.status = status mention.status = status
mention.save mention.save

View file

@ -7,9 +7,14 @@ class Admin::Metrics::Dimension
servers: Admin::Metrics::Dimension::ServersDimension, servers: Admin::Metrics::Dimension::ServersDimension,
space_usage: Admin::Metrics::Dimension::SpaceUsageDimension, space_usage: Admin::Metrics::Dimension::SpaceUsageDimension,
software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension, software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension,
tag_servers: Admin::Metrics::Dimension::TagServersDimension,
tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension,
}.freeze }.freeze
def self.retrieve(dimension_keys, start_at, end_at, limit) def self.retrieve(dimension_keys, start_at, end_at, limit, params)
Array(dimension_keys).map { |key| DIMENSIONS[key.to_sym]&.new(start_at, end_at, limit) }.compact Array(dimension_keys).map do |key|
klass = DIMENSIONS[key.to_sym]
klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil)
end.compact
end end
end end

View file

@ -1,10 +1,15 @@
# frozen_string_literal: true # frozen_string_literal: true
class Admin::Metrics::Dimension::BaseDimension class Admin::Metrics::Dimension::BaseDimension
def initialize(start_at, end_at, limit) def self.with_params?
false
end
def initialize(start_at, end_at, limit, params)
@start_at = start_at&.to_datetime @start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime @end_at = end_at&.to_datetime
@limit = limit&.to_i @limit = limit&.to_i
@params = params
end end
def key def key
@ -26,6 +31,10 @@ class Admin::Metrics::Dimension::BaseDimension
protected protected
def time_period def time_period
(@start_at...@end_at) (@start_at..@end_at)
end
def params
raise NotImplementedError
end end
end end

View file

@ -1,6 +1,8 @@
# frozen_string_literal: true # frozen_string_literal: true
class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension
include LanguagesHelper
def key def key
'languages' 'languages'
end end
@ -18,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]]) rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]])
rows.map { |row| { key: row['locale'], human_key: SettingsHelper::HUMAN_LOCALES[row['locale'].to_sym], value: row['value'].to_s } } rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } }
end end
end end

View file

@ -0,0 +1,36 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension
include LanguagesHelper
def self.with_params?
true
end
def key
'tag_languages'
end
def data
sql = <<-SQL.squish
SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value
FROM statuses
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
WHERE statuses_tags.tag_id = $1
AND statuses.id BETWEEN $2 AND $3
GROUP BY COALESCE(statuses.language, 'und')
ORDER BY count(*) DESC
LIMIT $4
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } }
end
private
def params
@params.permit(:id)
end
end

View file

@ -0,0 +1,35 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension
def self.with_params?
true
end
def key
'tag_servers'
end
def data
sql = <<-SQL.squish
SELECT accounts.domain, count(*) AS value
FROM statuses
INNER JOIN accounts ON accounts.id = statuses.account_id
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
WHERE statuses_tags.tag_id = $1
AND statuses.id BETWEEN $2 AND $3
GROUP BY accounts.domain
ORDER BY count(*) DESC
LIMIT $4
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } }
end
private
def params
@params.permit(:id)
end
end

View file

@ -7,9 +7,15 @@ class Admin::Metrics::Measure
interactions: Admin::Metrics::Measure::InteractionsMeasure, interactions: Admin::Metrics::Measure::InteractionsMeasure,
opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure, opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure,
resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure, resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure,
tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure,
tag_uses: Admin::Metrics::Measure::TagUsesMeasure,
tag_servers: Admin::Metrics::Measure::TagServersMeasure,
}.freeze }.freeze
def self.retrieve(measure_keys, start_at, end_at) def self.retrieve(measure_keys, start_at, end_at, params)
Array(measure_keys).map { |key| MEASURES[key.to_sym]&.new(start_at, end_at) }.compact Array(measure_keys).map do |key|
klass = MEASURES[key.to_sym]
klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil)
end.compact
end end
end end

View file

@ -24,10 +24,10 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas
end end
def time_period def time_period
(@start_at.to_date...@end_at.to_date) (@start_at.to_date..@end_at.to_date)
end end
def previous_time_period def previous_time_period
((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period)) ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end end
end end

View file

@ -1,9 +1,14 @@
# frozen_string_literal: true # frozen_string_literal: true
class Admin::Metrics::Measure::BaseMeasure class Admin::Metrics::Measure::BaseMeasure
def initialize(start_at, end_at) def self.with_params?
false
end
def initialize(start_at, end_at, params)
@start_at = start_at&.to_datetime @start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime @end_at = end_at&.to_datetime
@params = params
end end
def key def key
@ -33,14 +38,18 @@ class Admin::Metrics::Measure::BaseMeasure
protected protected
def time_period def time_period
(@start_at...@end_at) (@start_at..@end_at)
end end
def previous_time_period def previous_time_period
((@start_at - length_of_period)...(@end_at - length_of_period)) ((@start_at - length_of_period)..(@end_at - length_of_period))
end end
def length_of_period def length_of_period
@length_of_period ||= @end_at - @start_at @length_of_period ||= @end_at - @start_at
end end
def params
raise NotImplementedError
end
end end

View file

@ -24,10 +24,10 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba
end end
def time_period def time_period
(@start_at.to_date...@end_at.to_date) (@start_at.to_date..@end_at.to_date)
end end
def previous_time_period def previous_time_period
((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period)) ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end end
end end

View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_accounts'
end
def total
tag.history.aggregate(time_period).accounts
end
def previous_total
tag.history.aggregate(previous_time_period).accounts
end
def data
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def time_period
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
def params
@params.permit(:id)
end
end

View file

@ -0,0 +1,47 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_servers'
end
def total
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain')
end
def previous_total
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain')
end
def data
sql = <<-SQL.squish
SELECT axis.*, (
SELECT count(*) AS value
FROM statuses
WHERE statuses.id BETWEEN $1 AND $2
AND date_trunc('day', statuses.created_at)::date = axis.day
)
FROM (
SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day
) as axis
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]])
rows.map { |row| { date: row['day'], value: row['value'].to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def params
@params.permit(:id)
end
end

View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_uses'
end
def total
tag.history.aggregate(time_period).uses
end
def previous_total
tag.history.aggregate(previous_time_period).uses
end
def data
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def time_period
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
def params
@params.permit(:id)
end
end

View file

@ -4,6 +4,11 @@ class LinkDetailsExtractor
include ActionView::Helpers::TagHelper include ActionView::Helpers::TagHelper
class StructuredData class StructuredData
SUPPORTED_TYPES = %w(
NewsArticle
WebPage
).freeze
def initialize(data) def initialize(data)
@data = data @data = data
end end
@ -16,6 +21,14 @@ class LinkDetailsExtractor
json['description'] json['description']
end end
def language
json['inLanguage']
end
def type
json['@type']
end
def image def image
obj = first_of_value(json['image']) obj = first_of_value(json['image'])
@ -44,6 +57,10 @@ class LinkDetailsExtractor
publisher['name'] publisher['name']
end end
def publisher_logo
publisher.dig('logo', 'url')
end
private private
def author def author
@ -58,8 +75,12 @@ class LinkDetailsExtractor
arr.is_a?(Array) ? arr.first : arr arr.is_a?(Array) ? arr.first : arr
end end
def root_array(root)
root.is_a?(Array) ? root : [root]
end
def json def json
@json ||= first_of_value(Oj.load(@data)) @json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {}
end end
end end
@ -75,6 +96,7 @@ class LinkDetailsExtractor
description: description || '', description: description || '',
image_remote_url: image, image_remote_url: image,
type: type, type: type,
link_type: link_type,
width: width || 0, width: width || 0,
height: height || 0, height: height || 0,
html: html || '', html: html || '',
@ -83,6 +105,7 @@ class LinkDetailsExtractor
author_name: author_name || '', author_name: author_name || '',
author_url: author_url || '', author_url: author_url || '',
embed_url: embed_url || '', embed_url: embed_url || '',
language: language,
} }
end end
@ -90,6 +113,14 @@ class LinkDetailsExtractor
player_url.present? ? :video : :link player_url.present? ? :video : :link
end end
def link_type
if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article'
:article
else
:unknown
end
end
def html def html
player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
end end
@ -138,6 +169,14 @@ class LinkDetailsExtractor
valid_url_or_nil(opengraph_tag('twitter:player:stream')) valid_url_or_nil(opengraph_tag('twitter:player:stream'))
end end
def language
valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first)
end
def icon
valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon'))
end
private private
def player_url def player_url
@ -162,6 +201,14 @@ class LinkDetailsExtractor
nil nil
end end
def valid_locale_or_nil(str)
return nil if str.blank?
code, = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA
locale = ISO_639.find(code)
locale&.alpha2
end
def link_tag(name) def link_tag(name)
document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
end end

View file

@ -25,13 +25,25 @@ class AdminMailer < ApplicationMailer
end end
end end
def new_trending_tag(recipient, tag) def new_trending_tags(recipient, tags)
@tag = tag @tags = tags
@me = recipient @me = recipient
@instance = Rails.configuration.x.local_domain @instance = Rails.configuration.x.local_domain
@lowest_trending_tag = Trends.tags.get(true, Trends::Tags::REVIEW_THRESHOLD).last
locale_for_account(@me) do locale_for_account(@me) do
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name) mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance)
end
end
def new_trending_links(recipient, links)
@links = links
@me = recipient
@instance = Rails.configuration.x.local_domain
@lowest_trending_link = Trends.links.get(true, Trends::Links::REVIEW_THRESHOLD).last
locale_for_account(@me) do
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance)
end end
end end
end end

View file

@ -4,8 +4,8 @@
# #
# Table name: account_statuses_cleanup_policies # Table name: account_statuses_cleanup_policies
# #
# id :bigint not null, primary key # id :bigint(8) not null, primary key
# account_id :bigint not null # account_id :bigint(8) not null
# enabled :boolean default(TRUE), not null # enabled :boolean default(TRUE), not null
# min_status_age :integer default(1209600), not null # min_status_age :integer default(1209600), not null
# keep_direct :boolean default(TRUE), not null # keep_direct :boolean default(TRUE), not null

View file

@ -0,0 +1,65 @@
# frozen_string_literal: true
class Form::PreviewCardBatch
include ActiveModel::Model
include Authorization
attr_accessor :preview_card_ids, :action, :current_account, :precision
def save
case action
when 'approve'
approve!
when 'approve_all'
approve_all!
when 'reject'
reject!
when 'reject_all'
reject_all!
end
end
private
def preview_cards
@preview_cards ||= PreviewCard.where(id: preview_card_ids)
end
def preview_card_providers
@preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) }
end
def approve!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.update_all(trendable: true)
end
def approve_all!
preview_card_providers.each do |provider|
authorize(provider, :update?)
provider.update(trendable: true, reviewed_at: action_time)
end
# Reset any individual overrides
preview_cards.update_all(trendable: nil)
end
def reject!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.update_all(trendable: false)
end
def reject_all!
preview_card_providers.each do |provider|
authorize(provider, :update?)
provider.update(trendable: false, reviewed_at: action_time)
end
# Reset any individual overrides
preview_cards.update_all(trendable: nil)
end
def action_time
@action_time ||= Time.now.utc
end
end

View file

@ -0,0 +1,33 @@
# frozen_string_literal: true
class Form::PreviewCardProviderBatch
include ActiveModel::Model
include Authorization
attr_accessor :preview_card_provider_ids, :action, :current_account
def save
case action
when 'approve'
approve!
when 'reject'
reject!
end
end
private
def preview_card_providers
PreviewCardProvider.where(id: preview_card_provider_ids)
end
def approve!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc)
end
def reject!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc)
end
end

View file

@ -23,11 +23,15 @@ class Form::TagBatch
def approve! def approve!
tags.each { |tag| authorize(tag, :update?) } tags.each { |tag| authorize(tag, :update?) }
tags.update_all(trendable: true, reviewed_at: Time.now.utc) tags.update_all(trendable: true, reviewed_at: action_time)
end end
def reject! def reject!
tags.each { |tag| authorize(tag, :update?) } tags.each { |tag| authorize(tag, :update?) }
tags.update_all(trendable: false, reviewed_at: Time.now.utc) tags.update_all(trendable: false, reviewed_at: action_time)
end
def action_time
@action_time ||= Time.now.utc
end end
end end

View file

@ -24,6 +24,11 @@
# embed_url :string default(""), not null # embed_url :string default(""), not null
# image_storage_schema_version :integer # image_storage_schema_version :integer
# blurhash :string # blurhash :string
# language :string
# max_score :float
# max_score_at :datetime
# trendable :boolean
# link_type :integer
# #
class PreviewCard < ApplicationRecord class PreviewCard < ApplicationRecord
@ -40,6 +45,7 @@ class PreviewCard < ApplicationRecord
self.inheritance_column = false self.inheritance_column = false
enum type: [:link, :photo, :video, :rich] enum type: [:link, :photo, :video, :rich]
enum link_type: [:unknown, :article]
has_and_belongs_to_many :statuses has_and_belongs_to_many :statuses
@ -54,6 +60,32 @@ class PreviewCard < ApplicationRecord
before_save :extract_dimensions, if: :link? before_save :extract_dimensions, if: :link?
def appropriate_for_trends?
link? && article? && title.present? && description.present? && image.present? && provider_name.present?
end
def domain
@domain ||= Addressable::URI.parse(url).normalized_host
end
def provider
@provider ||= PreviewCardProvider.matching_domain(domain)
end
def trendable?
if attributes['trendable'].nil?
provider&.trendable?
else
attributes['trendable']
end
end
def requires_review_notification?
attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?)
end
attr_writer :provider
def local? def local?
false false
end end
@ -69,11 +101,14 @@ class PreviewCard < ApplicationRecord
save! save!
end end
def history
@history ||= Trends::History.new('links', id)
end
class << self class << self
private private
# rubocop:disable Naming/MethodParameterName def image_styles(file)
def image_styles(f)
styles = { styles = {
original: { original: {
geometry: '400x400>', geometry: '400x400>',
@ -83,10 +118,9 @@ class PreviewCard < ApplicationRecord
}, },
} }
styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif' styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif'
styles styles
end end
# rubocop:enable Naming/MethodParameterName
end end
private private

View file

@ -0,0 +1,53 @@
# frozen_string_literal: true
class PreviewCardFilter
KEYS = %i(
trending
).freeze
attr_reader :params
def initialize(params)
@params = params
end
def results
scope = PreviewCard.unscoped
params.each do |key, value|
next if key.to_s == 'page'
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope
end
private
def scope_for(key, value)
case key.to_s
when 'trending'
trending_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope(value)
ids = begin
case value.to_s
when 'allowed'
Trends.links.currently_trending_ids(true, -1)
else
Trends.links.currently_trending_ids(false, -1)
end
end
if ids.empty?
PreviewCard.none
else
PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering')
end
end
end

View file

@ -0,0 +1,57 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: preview_card_providers
#
# id :bigint(8) not null, primary key
# domain :string default(""), not null
# icon_file_name :string
# icon_content_type :string
# icon_file_size :bigint(8)
# icon_updated_at :datetime
# trendable :boolean
# reviewed_at :datetime
# requested_review_at :datetime
# created_at :datetime not null
# updated_at :datetime not null
#
class PreviewCardProvider < ApplicationRecord
include DomainNormalizable
include Attachmentable
ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze
LIMIT = 1.megabyte
validates :domain, presence: true, uniqueness: true, domain: true
has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false
validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT }
remotable_attachment :icon, LIMIT
scope :trendable, -> { where(trendable: true) }
scope :not_trendable, -> { where(trendable: false) }
scope :reviewed, -> { where.not(reviewed_at: nil) }
scope :pending_review, -> { where(reviewed_at: nil) }
def requires_review?
reviewed_at.nil?
end
def reviewed?
reviewed_at.present?
end
def requested_review?
requested_review_at.present?
end
def requires_review_notification?
requires_review? && !requested_review?
end
def self.matching_domain(domain)
segments = domain.split('.')
where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first
end
end

View file

@ -0,0 +1,49 @@
# frozen_string_literal: true
class PreviewCardProviderFilter
KEYS = %i(
status
).freeze
attr_reader :params
def initialize(params)
@params = params
end
def results
scope = PreviewCardProvider.unscoped
params.each do |key, value|
next if key.to_s == 'page'
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope.order(domain: :asc)
end
private
def scope_for(key, value)
case key.to_s
when 'status'
status_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def status_scope(value)
case value.to_s
when 'approved'
PreviewCardProvider.trendable
when 'rejected'
PreviewCardProvider.not_trendable
when 'pending_review'
PreviewCardProvider.pending_review
else
raise "Unknown status: #{value}"
end
end
end

View file

@ -36,6 +36,7 @@ class Tag < ApplicationRecord
scope :usable, -> { where(usable: [true, nil]) } scope :usable, -> { where(usable: [true, nil]) }
scope :listable, -> { where(listable: [true, nil]) } scope :listable, -> { where(listable: [true, nil]) }
scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) } scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
scope :not_trendable, -> { where(trendable: false) }
scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) } scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
@ -75,28 +76,12 @@ class Tag < ApplicationRecord
requested_review_at.present? requested_review_at.present?
end end
def use!(account, status: nil, at_time: Time.now.utc) def requires_review_notification?
TrendingTags.record_use!(self, account, status: status, at_time: at_time) requires_review? && !requested_review?
end
def trending?
TrendingTags.trending?(self)
end end
def history def history
days = [] @history ||= Trends::History.new('tags', id)
7.times do |i|
day = i.days.ago.beginning_of_day.to_i
days << {
day: day.to_s,
uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
}
end
days
end end
class << self class << self

View file

@ -2,13 +2,8 @@
class TagFilter class TagFilter
KEYS = %i( KEYS = %i(
directory trending
reviewed status
unreviewed
pending_review
popular
active
name
).freeze ).freeze
attr_reader :params attr_reader :params
@ -18,7 +13,13 @@ class TagFilter
end end
def results def results
scope = Tag.unscoped scope = begin
if params[:status] == 'pending_review'
Tag.unscoped
else
trending_scope
end
end
params.each do |key, value| params.each do |key, value|
next if key.to_s == 'page' next if key.to_s == 'page'
@ -26,27 +27,40 @@ class TagFilter
scope.merge!(scope_for(key, value.to_s.strip)) if value.present? scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end end
scope.order(id: :desc) scope
end end
private private
def scope_for(key, value) def scope_for(key, value)
case key.to_s case key.to_s
when 'reviewed' when 'status'
Tag.reviewed.order(reviewed_at: :desc) status_scope(value)
when 'unreviewed'
Tag.unreviewed
when 'pending_review'
Tag.pending_review.order(requested_review_at: :desc)
when 'popular'
Tag.order('max_score DESC NULLS LAST')
when 'active'
Tag.order('last_status_at DESC NULLS LAST')
when 'name'
Tag.matches_name(value)
else else
raise "Unknown filter: #{key}" raise "Unknown filter: #{key}"
end end
end end
def trending_scope
ids = Trends.tags.currently_trending_ids(false, -1)
if ids.empty?
Tag.none
else
Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering')
end
end
def status_scope(value)
case value.to_s
when 'approved'
Tag.trendable
when 'rejected'
Tag.not_trendable
when 'pending_review'
Tag.pending_review
else
raise "Unknown status: #{value}"
end
end
end end

View file

@ -1,128 +0,0 @@
# frozen_string_literal: true
class TrendingTags
KEY = 'trending_tags'
EXPIRE_HISTORY_AFTER = 7.days.seconds
EXPIRE_TRENDS_AFTER = 1.day.seconds
THRESHOLD = 5
LIMIT = 10
REVIEW_THRESHOLD = 3
MAX_SCORE_COOLDOWN = 2.days.freeze
MAX_SCORE_HALFLIFE = 2.hours.freeze
class << self
include Redisable
def record_use!(tag, account, status: nil, at_time: Time.now.utc)
return unless tag.usable? && !account.silenced?
# Even if a tag is not allowed to trend, we still need to
# record the stats since they can be displayed in other places
increment_historical_use!(tag.id, at_time)
increment_unique_use!(tag.id, account.id, at_time)
increment_use!(tag.id, at_time)
# Only update when the tag was last used once every 12 hours
# and only if a status is given (lets use ignore reblogs)
tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago))
end
def update!(at_time = Time.now.utc)
tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
tags = Tag.trendable.where(id: tag_ids.uniq)
# First pass to calculate scores and update the set
tags.each do |tag|
expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
expected = 1.0 if expected.zero?
observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
max_time = tag.max_score_at
max_score = tag.max_score
max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN)
score = begin
if expected > observed || observed < THRESHOLD
0
else
((observed - expected)**2) / expected
end
end
if score > max_score
max_score = score
max_time = at_time
# Not interested in triggering any callbacks for this
tag.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f))
if decaying_score.zero?
redis.zrem(KEY, tag.id)
else
redis.zadd(KEY, decaying_score, tag.id)
end
end
users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?)
# Second pass to notify about previously unreviewed trends
tags.each do |tag|
current_rank = redis.zrevrank(KEY, tag.id)
needs_review_notification = tag.requires_review? && !tag.requested_review?
rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD
next unless !tag.trendable? && rank_passes_threshold && needs_review_notification
tag.touch(:requested_review_at)
users_for_review.each do |user|
AdminMailer.new_trending_tag(user.account, tag).deliver_later!
end
end
# Trim older items
redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
redis.zremrangebyscore(KEY, '(0.3', '-inf')
end
def get(limit, filtered: true)
tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
tags = Tag.where(id: tag_ids)
tags = tags.trendable if filtered
tags = tags.index_by(&:id)
tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
end
def trending?(tag)
rank = redis.zrevrank(KEY, tag.id)
rank.present? && rank < LIMIT
end
private
def increment_historical_use!(tag_id, at_time)
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
redis.incrby(key, 1)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
def increment_unique_use!(tag_id, account_id, at_time)
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
redis.pfadd(key, account_id)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
def increment_use!(tag_id, at_time)
key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}"
redis.sadd(key, tag_id)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
end
end

27
app/models/trends.rb Normal file
View file

@ -0,0 +1,27 @@
# frozen_string_literal: true
module Trends
def self.table_name_prefix
'trends_'
end
def self.links
@links ||= Trends::Links.new
end
def self.tags
@tags ||= Trends::Tags.new
end
def self.refresh!
[links, tags].each(&:refresh)
end
def self.request_review!
[links, tags].each(&:request_review) if enabled?
end
def self.enabled?
Setting.trends
end
end

80
app/models/trends/base.rb Normal file
View file

@ -0,0 +1,80 @@
# frozen_string_literal: true
class Trends::Base
include Redisable
class_attribute :default_options
attr_reader :options
# @param [Hash] options
# @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score
# @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review
# @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from
# @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays
def initialize(options = {})
@options = self.class.default_options.merge(options)
end
def register(_status)
raise NotImplementedError
end
def add(*)
raise NotImplementedError
end
def refresh(*)
raise NotImplementedError
end
def request_review
raise NotImplementedError
end
def get(*)
raise NotImplementedError
end
def score(id)
redis.zscore("#{key_prefix}:all", id) || 0
end
def rank(id)
redis.zrevrank("#{key_prefix}:allowed", id)
end
def currently_trending_ids(allowed, limit)
redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i)
end
protected
def key_prefix
raise NotImplementedError
end
def recently_used_ids(at_time = Time.now.utc)
redis.smembers(used_key(at_time)).map(&:to_i)
end
def record_used_id(id, at_time = Time.now.utc)
redis.sadd(used_key(at_time), id)
redis.expire(used_key(at_time), 1.day.seconds)
end
def trim_older_items
redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1')
redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1')
end
def score_at_rank(rank)
redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
end
private
def used_key(at_time)
"#{key_prefix}:used:#{at_time.beginning_of_day.to_i}"
end
end

View file

@ -0,0 +1,98 @@
# frozen_string_literal: true
class Trends::History
include Enumerable
class Aggregate
include Redisable
def initialize(prefix, id, date_range)
@days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) }
end
def uses
redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum
end
def accounts
redis.pfcount(*@days.map { |day| day.key_for(:accounts) })
end
end
class Day
include Redisable
EXPIRE_AFTER = 14.days.seconds
def initialize(prefix, id, day)
@prefix = prefix
@id = id
@day = day.beginning_of_day
end
attr_reader :day
def accounts
redis.pfcount(key_for(:accounts))
end
def uses
redis.get(key_for(:uses))&.to_i || 0
end
def add(account_id)
redis.pipelined do
redis.incrby(key_for(:uses), 1)
redis.pfadd(key_for(:accounts), account_id)
redis.expire(key_for(:uses), EXPIRE_AFTER)
redis.expire(key_for(:accounts), EXPIRE_AFTER)
end
end
def as_json
{ day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s }
end
def key_for(suffix)
case suffix
when :accounts
"#{key_prefix}:#{suffix}"
when :uses
key_prefix
end
end
def key_prefix
"activity:#{@prefix}:#{@id}:#{day.to_i}"
end
end
def initialize(prefix, id)
@prefix = prefix
@id = id
end
def get(date)
Day.new(@prefix, @id, date)
end
def add(account_id, at_time = Time.now.utc)
Day.new(@prefix, @id, at_time).add(account_id)
end
def aggregate(date_range)
Aggregate.new(@prefix, @id, date_range)
end
def each(&block)
if block_given?
(0...7).map { |i| block.call(get(i.days.ago)) }
else
to_enum(:each)
end
end
def as_json(*)
map(&:as_json)
end
end

117
app/models/trends/links.rb Normal file
View file

@ -0,0 +1,117 @@
# frozen_string_literal: true
class Trends::Links < Trends::Base
PREFIX = 'trending_links'
self.default_options = {
threshold: 15,
review_threshold: 10,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 8.hours.freeze,
}
def register(status, at_time = Time.now.utc)
original_status = status.reblog? ? status.reblog : status
return unless original_status.public_visibility? && status.public_visibility? &&
!original_status.account.silenced? && !status.account.silenced? &&
!original_status.spoiler_text?
original_status.preview_cards.each do |preview_card|
add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends?
end
end
def add(preview_card, account_id, at_time = Time.now.utc)
preview_card.history.add(account_id, at_time)
record_used_id(preview_card.id, at_time)
end
def get(allowed, limit)
preview_card_ids = currently_trending_ids(allowed, limit)
preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id)
preview_card_ids.map { |id| preview_cards[id] }.compact
end
def refresh(at_time = Time.now.utc)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
calculate_scores(preview_cards, at_time)
trim_older_items
end
def request_review
preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
preview_cards_requiring_review = preview_cards.filter_map do |preview_card|
next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
if preview_card.provider.nil?
preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
else
preview_card.provider.touch(:requested_review_at)
end
preview_card
end
return if preview_cards_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
def key_prefix
PREFIX
end
private
def calculate_scores(preview_cards, at_time)
preview_cards.each do |preview_card|
expected = preview_card.history.get(at_time - 1.day).accounts.to_f
expected = 1.0 if expected.zero?
observed = preview_card.history.get(at_time).accounts.to_f
max_time = preview_card.max_score_at
max_score = preview_card.max_score
max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
score = begin
if expected > observed || observed < options[:threshold]
0
else
((observed - expected)**2) / expected
end
end
if score > max_score
max_score = score
max_time = at_time
# Not interested in triggering any callbacks for this
preview_card.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", preview_card.id)
redis.zrem("#{PREFIX}:allowed", preview_card.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id)
if preview_card.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id)
else
redis.zrem("#{PREFIX}:allowed", preview_card.id)
end
end
end
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

111
app/models/trends/tags.rb Normal file
View file

@ -0,0 +1,111 @@
# frozen_string_literal: true
class Trends::Tags < Trends::Base
PREFIX = 'trending_tags'
self.default_options = {
threshold: 15,
review_threshold: 10,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 4.hours.freeze,
}
def register(status, at_time = Time.now.utc)
original_status = status.reblog? ? status.reblog : status
return unless original_status.public_visibility? && status.public_visibility? &&
!original_status.account.silenced? && !status.account.silenced?
original_status.tags.each do |tag|
add(tag, status.account_id, at_time) if tag.usable?
end
end
def add(tag, account_id, at_time = Time.now.utc)
tag.history.add(account_id, at_time)
record_used_id(tag.id, at_time)
end
def refresh(at_time = Time.now.utc)
tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
calculate_scores(tags, at_time)
trim_older_items
end
def get(allowed, limit)
tag_ids = currently_trending_ids(allowed, limit)
tags = Tag.where(id: tag_ids).index_by(&:id)
tag_ids.map { |id| tags[id] }.compact
end
def request_review
tags = Tag.where(id: currently_trending_ids(false, -1))
tags_requiring_review = tags.filter_map do |tag|
next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
tag.touch(:requested_review_at)
tag
end
return if tags_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
def key_prefix
PREFIX
end
private
def calculate_scores(tags, at_time)
tags.each do |tag|
expected = tag.history.get(at_time - 1.day).accounts.to_f
expected = 1.0 if expected.zero?
observed = tag.history.get(at_time).accounts.to_f
max_time = tag.max_score_at
max_score = tag.max_score
max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
score = begin
if expected > observed || observed < options[:threshold]
0
else
((observed - expected)**2) / expected
end
end
if score > max_score
max_score = score
max_time = at_time
# Not interested in triggering any callbacks for this
tag.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", tag.id)
redis.zrem("#{PREFIX}:allowed", tag.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, tag.id)
if tag.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id)
else
redis.zrem("#{PREFIX}:allowed", tag.id)
end
end
end
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class PreviewCardPolicy < ApplicationPolicy
def index?
staff?
end
def update?
staff?
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class PreviewCardProviderPolicy < ApplicationPolicy
def index?
staff?
end
def update?
staff?
end
end

View file

@ -0,0 +1,5 @@
# frozen_string_literal: true
class REST::Trends::LinkSerializer < REST::PreviewCardSerializer
attributes :history
end

View file

@ -50,7 +50,7 @@ class FetchLinkCardService < BaseService
# We follow redirects, and ideally we want to save the preview card for # We follow redirects, and ideally we want to save the preview card for
# the destination URL and not any link shortener in-between, so here # the destination URL and not any link shortener in-between, so here
# we set the URL to the one of the last response in the redirect chain # we set the URL to the one of the last response in the redirect chain
@url = res.request.uri.to_s.to_s @url = res.request.uri.to_s
@card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url @card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url
if res.code == 200 && res.mime_type == 'text/html' if res.code == 200 && res.mime_type == 'text/html'
@ -66,6 +66,7 @@ class FetchLinkCardService < BaseService
def attach_card def attach_card
@status.preview_cards << @card @status.preview_cards << @card
Rails.cache.delete(@status) Rails.cache.delete(@status)
Trends.links.register(@status)
end end
def parse_urls def parse_urls

View file

@ -91,7 +91,8 @@ class PostStatusService < BaseService
end end
def postprocess_status! def postprocess_status!
LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text? Trends.tags.register(@status)
LinkCrawlWorker.perform_async(@status.id)
DistributionWorker.perform_async(@status.id) DistributionWorker.perform_async(@status.id)
ActivityPub::DistributionWorker.perform_async(@status.id) ActivityPub::DistributionWorker.perform_async(@status.id)
PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll

View file

@ -8,7 +8,7 @@ class ProcessHashtagsService < BaseService
Tag.find_or_create_by_names(tags) do |tag| Tag.find_or_create_by_names(tags) do |tag|
status.tags << tag status.tags << tag
records << tag records << tag
tag.use!(status.account, status: status, at_time: status.created_at) if status.public_visibility? tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
end end
return unless status.distributable? return unless status.distributable?

View file

@ -30,12 +30,13 @@ class ReblogService < BaseService
reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit]) reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit])
Trends.tags.register(reblog)
Trends.links.register(reblog)
DistributionWorker.perform_async(reblog.id) DistributionWorker.perform_async(reblog.id)
ActivityPub::DistributionWorker.perform_async(reblog.id) ActivityPub::DistributionWorker.perform_async(reblog.id)
create_notification(reblog) create_notification(reblog)
bump_potential_friendship(account, reblog) bump_potential_friendship(account, reblog)
record_use(account, reblog)
reblog reblog
end end
@ -60,16 +61,6 @@ class ReblogService < BaseService
PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog) PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog)
end end
def record_use(account, reblog)
return unless reblog.public_visibility?
original_status = reblog.reblog
original_status.tags.each do |tag|
tag.use!(account)
end
end
def build_json(reblog) def build_json(reblog)
Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account)) Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account))
end end

View file

@ -42,7 +42,7 @@
%span= t('admin.dashboard.pending_users_html', count: @pending_users_count) %span= t('admin.dashboard.pending_users_html', count: @pending_users_count)
= fa_icon 'chevron-right fw' = fa_icon 'chevron-right fw'
= link_to admin_tags_path(pending_review: '1'), class: 'dashboard__quick-access' do = link_to admin_trends_tags_path(status: 'pending_review'), class: 'dashboard__quick-access' do
%span= t('admin.dashboard.pending_tags_html', count: @pending_tags_count) %span= t('admin.dashboard.pending_tags_html', count: @pending_tags_count)
= fa_icon 'chevron-right fw' = fa_icon 'chevron-right fw'

View file

@ -1,19 +0,0 @@
.batch-table__row
- if batch_available
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id
.directory__tag
= link_to admin_tag_path(tag.id) do
%h4
= fa_icon 'hashtag'
= tag.name
%small
= t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
- if tag.trending?
= fa_icon 'fire fw'
= t('admin.tags.trending_right_now')
.trends__item__current= friendly_number_to_human tag.history.first[:uses]

View file

@ -1,74 +0,0 @@
- content_for :page_title do
= t('admin.tags.title')
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
.filters
.filter-subset
%strong= t('admin.tags.review')
%ul
%li= filter_link_to t('generic.all'), reviewed: nil, unreviewed: nil, pending_review: nil
%li= filter_link_to t('admin.tags.unreviewed'), unreviewed: '1', reviewed: nil, pending_review: nil
%li= filter_link_to t('admin.tags.reviewed'), reviewed: '1', unreviewed: nil, pending_review: nil
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), pending_review: '1', reviewed: nil, unreviewed: nil
.filter-subset
%strong= t('generic.order_by')
%ul
%li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil
%li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil
%li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil
= form_tag admin_tags_url, method: 'GET', class: 'simple_form' do
.fields-group
- TagFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
- %i(name).each do |key|
.input.string.optional
= text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.tags.#{key}")
.actions
%button.button= t('admin.accounts.search')
= link_to t('admin.accounts.reset'), admin_tags_path, class: 'button negative'
%hr.spacer/
= form_for(@form, url: batch_admin_tags_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- TagFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table.optional
.batch-table__toolbar
- if params[:pending_review] == '1' || params[:unreviewed] == '1'
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.accounts.approve')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.accounts.reject')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
- else
.batch-table__toolbar__actions
%span.neutral-hint= t('generic.no_batch_actions_available')
.batch-table__body
- if @tags.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'tag', collection: @tags, locals: { f: f, batch_available: params[:pending_review] == '1' || params[:unreviewed] == '1' }
= paginate @tags
- if params[:pending_review] == '1' || params[:unreviewed] == '1'
%hr.spacer/
%div.action-buttons
%div
= link_to t('admin.accounts.approve_all'), approve_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button'
%div
= link_to t('admin.accounts.reject_all'), reject_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button button--destructive'

View file

@ -1,15 +1,50 @@
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
- content_for :page_title do - content_for :page_title do
= "##{@tag.name}" = "##{@tag.name}"
.dashboard__counters - content_for :heading_actions do
%div = l(@time_period.first)
= link_to tag_url(@tag), target: '_blank', rel: 'noopener noreferrer' do = ' - '
.dashboard__counters__num= number_with_delimiter @accounts_today = l(@time_period.last)
.dashboard__counters__label= t 'admin.tags.accounts_today'
%div .dashboard
%div .dashboard__item
.dashboard__counters__num= number_with_delimiter @accounts_week = react_admin_component :counter, measure: 'tag_accounts', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_accounts_measure')
.dashboard__counters__label= t 'admin.tags.accounts_week' .dashboard__item
= react_admin_component :counter, measure: 'tag_uses', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_uses_measure')
.dashboard__item
= react_admin_component :counter, measure: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_servers_measure')
.dashboard__item
= react_admin_component :dimension, dimension: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_servers_dimension')
.dashboard__item
= react_admin_component :dimension, dimension: 'tag_languages', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_languages_dimension')
.dashboard__item
= link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.usable? ? 'positive' : 'negative'] do
- if @tag.usable?
%span= t('admin.trends.tags.usable')
= fa_icon 'check fw'
- else
%span= t('admin.trends.tags.not_usable')
= fa_icon 'lock fw'
= link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.trendable? ? 'positive' : 'negative'] do
- if @tag.trendable?
%span= t('admin.trends.tags.trendable')
= fa_icon 'check fw'
- else
%span= t('admin.trends.tags.not_trendable')
= fa_icon 'lock fw'
= link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.listable? ? 'positive' : 'negative'] do
- if @tag.listable?
%span= t('admin.trends.tags.listable')
= fa_icon 'check fw'
- else
%span= t('admin.trends.tags.not_listable')
= fa_icon 'lock fw'
%hr.spacer/ %hr.spacer/
@ -26,18 +61,3 @@
.actions .actions
= f.button :button, t('generic.save_changes'), type: :submit = f.button :button, t('generic.save_changes'), type: :submit
%hr.spacer/
%h3= t 'admin.tags.breakdown'
.table-wrapper
%table.table
%tbody
- total = @usage_by_domain.sum(&:last).to_f
- @usage_by_domain.each do |(domain, count)|
%tr
%th= domain || site_hostname
%td= number_to_percentage((count / total) * 100, precision: 1)
%td= number_with_delimiter count

View file

@ -0,0 +1,30 @@
.batch-table__row{ class: [preview_card.provider&.requires_review? && 'batch-table__row--attention', !preview_card.provider&.requires_review? && !preview_card.trendable? && 'batch-table__row--muted'] }
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :preview_card_ids, { multiple: true, include_hidden: false }, preview_card.id
.batch-table__row__content.pending-account
.pending-account__header
= link_to preview_card.title, preview_card.url
%br/
- if preview_card.provider_name.present?
= preview_card.provider_name
- if preview_card.language.present?
= human_locale(preview_card.language)
= t('admin.trends.links.shared_by_over_week', count: preview_card.history.reduce(0) { |sum, day| sum + day.accounts })
- if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id))
%abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
- if preview_card.max_score_at && preview_card.max_score_at >= Trends::Links::MAX_SCORE_COOLDOWN.ago && preview_card.max_score_at < 1.day.ago
= t('admin.trends.tags.peaked_on_and_decaying', date: l(preview_card.max_score_at.to_date, format: :short))
- elsif preview_card.provider&.requires_review?
= t('admin.trends.pending_review')

View file

@ -0,0 +1,41 @@
- content_for :page_title do
= t('admin.trends.links.title')
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
.filters
.filter-subset
%strong= t('admin.trends.trending')
%ul
%li= filter_link_to t('generic.all'), trending: nil
%li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed'
.back-link
= link_to admin_trends_links_preview_card_providers_path do
= t('admin.trends.preview_card_providers.title')
= fa_icon 'chevron-right fw'
%hr.spacer/
= form_for(@form, url: batch_admin_trends_links_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- PreviewCardFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
.batch-table__body
- if @preview_cards.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'preview_card', collection: @preview_cards, locals: { f: f }
= paginate @preview_cards

View file

@ -0,0 +1,16 @@
.batch-table__row{ class: [preview_card_provider.requires_review? && 'batch-table__row--attention', !preview_card_provider.requires_review? && !preview_card_provider.trendable? && 'batch-table__row--muted'] }
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :preview_card_provider_ids, { multiple: true, include_hidden: false }, preview_card_provider.id
.batch-table__row__content.pending-account
.pending-account__header
%strong= preview_card_provider.domain
%br/
- if preview_card_provider.requires_review?
= t('admin.trends.pending_review')
- elsif preview_card_provider.trendable?
= t('admin.trends.preview_card_providers.allowed')
- else
= t('admin.trends.preview_card_providers.rejected')

View file

@ -0,0 +1,43 @@
- content_for :page_title do
= t('admin.trends.preview_card_providers.title')
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
.filters
.filter-subset
%strong= t('admin.tags.review')
%ul
%li= filter_link_to t('generic.all'), status: nil
%li= filter_link_to t('admin.trends.approved'), status: 'approved'
%li= filter_link_to t('admin.trends.rejected'), status: 'rejected'
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{PreviewCardProvider.pending_review.count})"], ' '), status: 'pending_review'
.back-link
= link_to admin_trends_links_path do
= fa_icon 'chevron-left fw'
= t('admin.trends.links.title')
%hr.spacer/
= form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- PreviewCardProviderFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table.optional
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
.batch-table__body
- if @preview_card_providers.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'preview_card_provider', collection: @preview_card_providers, locals: { f: f }
= paginate @preview_card_providers

View file

@ -0,0 +1,24 @@
.batch-table__row{ class: [tag.requires_review? && 'batch-table__row--attention', !tag.requires_review? && !tag.trendable? && 'batch-table__row--muted'] }
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id
.batch-table__row__content.pending-account
.pending-account__header
= link_to admin_tag_path(tag.id) do
= fa_icon 'hashtag'
= tag.name
%br/
= t('admin.trends.tags.used_by_over_week', count: tag.history.reduce(0) { |sum, day| sum + day.accounts })
- if tag.trendable? && (rank = Trends.tags.rank(tag.id))
%abbr{ title: t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
- if tag.max_score_at && tag.max_score_at >= Trends::Tags::MAX_SCORE_COOLDOWN.ago && tag.max_score_at < 1.day.ago
= t('admin.trends.tags.peaked_on_and_decaying', date: l(tag.max_score_at.to_date, format: :short))
- elsif tag.requires_review?
= t('admin.trends.pending_review')

View file

@ -0,0 +1,38 @@
- content_for :page_title do
= t('admin.trends.tags.title')
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
.filters
.filter-subset
%strong= t('admin.tags.review')
%ul
%li= filter_link_to t('generic.all'), status: nil
%li= filter_link_to t('admin.trends.approved'), status: 'approved'
%li= filter_link_to t('admin.trends.rejected'), status: 'rejected'
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review'
%hr.spacer/
= form_for(@form, url: batch_admin_trends_tags_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- TagFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table.optional
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
.batch-table__body
- if @tags.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'tag', collection: @tags, locals: { f: f }
= paginate @tags

View file

@ -0,0 +1,16 @@
<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
<%= raw t('admin_mailer.new_trending_links.body') %>
<% @links.each do |link| %>
- <%= link.title %> • <%= link.url %>
<%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %>
<% end %>
<% if @lowest_trending_link %>
<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %>
<% else %>
<%= t('admin_mailer.new_trending_links.no_approved_links') %>
<% end %>
<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %>

View file

@ -1,5 +0,0 @@
<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %>
<%= raw t('application_mailer.view')%> <%= admin_tags_url(pending_review: '1') %>

View file

@ -0,0 +1,16 @@
<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
<%= raw t('admin_mailer.new_trending_tags.body') %>
<% @tags.each do |tag| %>
- #<%= tag.name %>
<%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %>
<% end %>
<% if @lowest_trending_tag %>
<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %>
<% else %>
<%= t('admin_mailer.new_trending_tags.no_approved_tags') %>
<% end %>
<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %>

View file

@ -6,7 +6,7 @@
%p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html') %p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html')
- if Setting.trends && !(user_signed_in? && !current_user.setting_trends) - if Setting.trends && !(user_signed_in? && !current_user.setting_trends)
- trends = TrendingTags.get(3) - trends = Trends.tags.get(true, 3)
- unless trends.empty? - unless trends.empty?
.endorsements-widget.trends-widget .endorsements-widget.trends-widget

View file

@ -1,11 +1,11 @@
# frozen_string_literal: true # frozen_string_literal: true
class Scheduler::TrendingTagsScheduler class Scheduler::Trends::RefreshScheduler
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options retry: 0 sidekiq_options retry: 0
def perform def perform
TrendingTags.update! if Setting.trends Trends.refresh!
end end
end end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class Scheduler::Trends::ReviewNotificationsScheduler
include Sidekiq::Worker
sidekiq_options retry: 0
def perform
Trends.request_review!
end
end

View file

@ -67,7 +67,7 @@
"check_name": "SQL", "check_name": "SQL",
"message": "Possible SQL injection", "message": "Possible SQL injection",
"file": "app/models/account.rb", "file": "app/models/account.rb",
"line": 479, "line": 484,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "find_by_sql([\" WITH first_degree AS (\\n SELECT target_account_id\\n FROM follows\\n WHERE account_id = ?\\n UNION ALL\\n SELECT ?\\n )\\n SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)\\n WHERE accounts.id IN (SELECT * FROM first_degree)\\n AND #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, account.id, limit, offset])", "code": "find_by_sql([\" WITH first_degree AS (\\n SELECT target_account_id\\n FROM follows\\n WHERE account_id = ?\\n UNION ALL\\n SELECT ?\\n )\\n SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)\\n WHERE accounts.id IN (SELECT * FROM first_degree)\\n AND #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, account.id, limit, offset])",
"render_path": null, "render_path": null,
@ -100,6 +100,26 @@
"confidence": "Weak", "confidence": "Weak",
"note": "" "note": ""
}, },
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "75fcd147b7611763ab6915faf8c5b0709e612b460f27c05c72d8b9bd0a6a77f8",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "lib/mastodon/snowflake.rb",
"line": 87,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "connection.execute(\"CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\nRETURNS bigint AS\\n$$\\n DECLARE\\n time_part bigint;\\n sequence_base bigint;\\n tail bigint;\\n BEGIN\\n time_part := (\\n -- Get the time in milliseconds\\n ((date_part('epoch', now()) * 1000))::bigint\\n -- And shift it over two bytes\\n << 16);\\n\\n sequence_base := (\\n 'x' ||\\n -- Take the first two bytes (four hex characters)\\n substr(\\n -- Of the MD5 hash of the data we documented\\n md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),\\n 1, 4\\n )\\n -- And turn it into a bigint\\n )::bit(16)::bigint;\\n\\n -- Finally, add our sequence number to our base, and chop\\n -- it to the last two bytes\\n tail := (\\n (sequence_base + nextval(table_name || '_id_seq'))\\n & 65535);\\n\\n -- Return the time part and the sequence part. OR appears\\n -- faster here than addition, but they're equivalent:\\n -- time_part has no trailing two bytes, and tail is only\\n -- the last two bytes.\\n RETURN time_part | tail;\\n END\\n$$ LANGUAGE plpgsql VOLATILE;\\n\")",
"render_path": null,
"location": {
"type": "method",
"class": "Mastodon::Snowflake",
"method": "define_timestamp_id"
},
"user_input": "SecureRandom.hex(16)",
"confidence": "Medium",
"note": ""
},
{ {
"warning_type": "Mass Assignment", "warning_type": "Mass Assignment",
"warning_code": 105, "warning_code": 105,
@ -140,6 +160,26 @@
"confidence": "High", "confidence": "High",
"note": "" "note": ""
}, },
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "8c1d8c4b76c1cd3960e90dff999f854a6ff742fcfd8de6c7184ac5a1b1a4d7dd",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/preview_card_filter.rb",
"line": 50,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "PreviewCard.joins(\"join unnest(array[#{(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id\")",
"render_path": null,
"location": {
"type": "method",
"class": "PreviewCardFilter",
"method": "trending_scope"
},
"user_input": "(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")",
"confidence": "Medium",
"note": ""
},
{ {
"warning_type": "SQL Injection", "warning_type": "SQL Injection",
"warning_code": 0, "warning_code": 0,
@ -147,7 +187,7 @@
"check_name": "SQL", "check_name": "SQL",
"message": "Possible SQL injection", "message": "Possible SQL injection",
"file": "app/models/account.rb", "file": "app/models/account.rb",
"line": 448, "line": 453,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "find_by_sql([\" SELECT\\n accounts.*,\\n ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, limit, offset])", "code": "find_by_sql([\" SELECT\\n accounts.*,\\n ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, limit, offset])",
"render_path": null, "render_path": null,
@ -160,26 +200,6 @@
"confidence": "Medium", "confidence": "Medium",
"note": "" "note": ""
}, },
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "9ccb9ba6a6947400e187d515e0bf719d22993d37cfc123c824d7fafa6caa9ac3",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "lib/mastodon/snowflake.rb",
"line": 87,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "connection.execute(\" CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\n RETURNS bigint AS\\n $$\\n DECLARE\\n time_part bigint;\\n sequence_base bigint;\\n tail bigint;\\n BEGIN\\n time_part := (\\n -- Get the time in milliseconds\\n ((date_part('epoch', now()) * 1000))::bigint\\n -- And shift it over two bytes\\n << 16);\\n\\n sequence_base := (\\n 'x' ||\\n -- Take the first two bytes (four hex characters)\\n substr(\\n -- Of the MD5 hash of the data we documented\\n md5(table_name ||\\n '#{SecureRandom.hex(16)}' ||\\n time_part::text\\n ),\\n 1, 4\\n )\\n -- And turn it into a bigint\\n )::bit(16)::bigint;\\n\\n -- Finally, add our sequence number to our base, and chop\\n -- it to the last two bytes\\n tail := (\\n (sequence_base + nextval(table_name || '_id_seq'))\\n & 65535);\\n\\n -- Return the time part and the sequence part. OR appears\\n -- faster here than addition, but they're equivalent:\\n -- time_part has no trailing two bytes, and tail is only\\n -- the last two bytes.\\n RETURN time_part | tail;\\n END\\n $$ LANGUAGE plpgsql VOLATILE;\\n\")",
"render_path": null,
"location": {
"type": "method",
"class": "Mastodon::Snowflake",
"method": "define_timestamp_id"
},
"user_input": "SecureRandom.hex(16)",
"confidence": "Medium",
"note": ""
},
{ {
"warning_type": "Redirect", "warning_type": "Redirect",
"warning_code": 18, "warning_code": 18,
@ -201,23 +221,53 @@
"note": "" "note": ""
}, },
{ {
"warning_type": "Redirect", "warning_type": "SQL Injection",
"warning_code": 18, "warning_code": 0,
"fingerprint": "ba699ddcc6552c422c4ecd50d2cd217f616a2446659e185a50b05a0f2dad8d33", "fingerprint": "c32a484ccd9da46abd3bc93d08b72029d7dbc0576ccf4e878a9627e9a83cad2e",
"check_name": "Redirect", "check_name": "SQL",
"message": "Possible unprotected redirect", "message": "Possible SQL injection",
"file": "app/controllers/media_controller.rb", "file": "app/models/tag_filter.rb",
"line": 20, "line": 50,
"link": "https://brakemanscanner.org/docs/warning_types/redirect/", "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "redirect_to(MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original))", "code": "Tag.joins(\"join unnest(array[#{Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id\")",
"render_path": null, "render_path": null,
"location": { "location": {
"type": "method", "type": "method",
"class": "MediaController", "class": "TagFilter",
"method": "show" "method": "trending_scope"
}, },
"user_input": "MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original)", "user_input": "Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")",
"confidence": "High", "confidence": "Medium",
"note": ""
},
{
"warning_type": "Cross-Site Scripting",
"warning_code": 4,
"fingerprint": "cd5cfd7f40037fbfa753e494d7129df16e358bfc43ef0da3febafbf4ee1ed3ac",
"check_name": "LinkToHref",
"message": "Potentially unsafe model attribute in `link_to` href",
"file": "app/views/admin/trends/links/_preview_card.html.haml",
"line": 7,
"link": "https://brakemanscanner.org/docs/warning_types/link_to_href",
"code": "link_to((Unresolved Model).new.title, (Unresolved Model).new.url)",
"render_path": [
{
"type": "template",
"name": "admin/trends/links/index",
"line": 37,
"file": "app/views/admin/trends/links/index.html.haml",
"rendered": {
"name": "admin/trends/links/_preview_card",
"file": "app/views/admin/trends/links/_preview_card.html.haml"
}
}
],
"location": {
"type": "template",
"template": "admin/trends/links/_preview_card"
},
"user_input": "(Unresolved Model).new.url",
"confidence": "Weak",
"note": "" "note": ""
}, },
{ {
@ -227,7 +277,7 @@
"check_name": "SQL", "check_name": "SQL",
"message": "Possible SQL injection", "message": "Possible SQL injection",
"file": "app/models/account.rb", "file": "app/models/account.rb",
"line": 495, "line": 500,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "find_by_sql([\" SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, limit, offset])", "code": "find_by_sql([\" SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, limit, offset])",
"render_path": null, "render_path": null,
@ -261,6 +311,6 @@
"note": "" "note": ""
} }
], ],
"updated": "2021-05-11 20:22:27 +0900", "updated": "2021-11-14 05:26:09 +0100",
"brakeman_version": "5.0.1" "brakeman_version": "5.1.2"
} }

View file

@ -674,8 +674,8 @@ en:
desc_html: Affects hashtags that have not been previously disallowed desc_html: Affects hashtags that have not been previously disallowed
title: Allow hashtags to trend without prior review title: Allow hashtags to trend without prior review
trends: trends:
desc_html: Publicly display previously reviewed hashtags that are currently trending desc_html: Publicly display previously reviewed content that is currently trending
title: Trending hashtags title: Trends
site_uploads: site_uploads:
delete: Delete uploaded file delete: Delete uploaded file
destroyed_msg: Site upload successfully deleted! destroyed_msg: Site upload successfully deleted!
@ -702,21 +702,51 @@ en:
sidekiq_process_check: sidekiq_process_check:
message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration
tags: tags:
accounts_today: Unique uses today
accounts_week: Unique uses this week
breakdown: Breakdown of today's usage by source
last_active: Recently used
most_popular: Most popular
most_recent: Recently created
name: Hashtag
review: Review status review: Review status
reviewed: Reviewed
title: Hashtags
trending_right_now: Trending right now
unique_uses_today: "%{count} posting today"
unreviewed: Not reviewed
updated_msg: Hashtag settings updated successfully updated_msg: Hashtag settings updated successfully
title: Administration title: Administration
trends:
allow: Allow
approved: Approved
disallow: Disallow
links:
allow: Allow link
allow_provider: Allow publisher
disallow: Disallow link
disallow_provider: Disallow publisher
shared_by_over_week:
one: Shared by one person over the last week
other: Shared by %{count} people over the last week
title: Trending links
usage_comparison: Shared %{today} times today, compared to %{yesterday} yesterday
pending_review: Pending review
preview_card_providers:
allowed: Links from this publisher can trend
rejected: Links from this publisher won't trend
title: Publishers
rejected: Rejected
tags:
current_score: Current score %{score}
dashboard:
tag_accounts_measure: unique uses
tag_languages_dimension: Top languages
tag_servers_dimension: Top servers
tag_servers_measure: different servers
tag_uses_measure: total uses
listable: Can be suggested
not_listable: Won't be suggested
not_trendable: Won't appear under trends
not_usable: Cannot be used
peaked_on_and_decaying: Peaked on %{date}, now decaying
title: Trending hashtags
trendable: Can appear under trends
trending_rank: 'Trending #%{rank}'
usable: Can be used
usage_comparison: Used %{today} times today, compared to %{yesterday} yesterday
used_by_over_week:
one: Used by one person over the last week
other: Used by %{count} people over the last week
title: Trends
warning_presets: warning_presets:
add_new: Add new add_new: Add new
delete: Delete delete: Delete
@ -731,9 +761,16 @@ en:
body: "%{reporter} has reported %{target}" body: "%{reporter} has reported %{target}"
body_remote: Someone from %{domain} has reported %{target} body_remote: Someone from %{domain} has reported %{target}
subject: New report for %{instance} (#%{id}) subject: New report for %{instance} (#%{id})
new_trending_tag: new_trending_links:
body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.' body: The following links are trending today, but their publishers have not been previously reviewed. They will not be displayed publicly unless you approve them. Further notifications from the same publishers will not be generated.
subject: New hashtag up for review on %{instance} (#%{name}) no_approved_links: There are currently no approved trending links.
requirements: The lowest approved trending link is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.
subject: New trending links up for review on %{instance}
new_trending_tags:
body: 'The following hashtags are trending today, but they have not been previously reviewed. They will not be displayed publicly unless you approve them:'
no_approved_tags: There are currently no approved trending hashtags.
requirements: 'The lowest approved trending hashtag is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.'
subject: New trending hashtags up for review on %{instance}
aliases: aliases:
add_new: Create alias add_new: Create alias
created_msg: Successfully created a new alias. You can now initiate the move from the old account. created_msg: Successfully created a new alias. You can now initiate the move from the old account.
@ -940,7 +977,7 @@ en:
changes_saved_msg: Changes successfully saved! changes_saved_msg: Changes successfully saved!
copy: Copy copy: Copy
delete: Delete delete: Delete
no_batch_actions_available: No batch actions available on this page none: None
order_by: Order by order_by: Order by
save_changes: Save changes save_changes: Save changes
validation_errors: validation_errors:

View file

@ -204,8 +204,8 @@ en:
mention: Someone mentioned you mention: Someone mentioned you
pending_account: New account needs review pending_account: New account needs review
reblog: Someone boosted your post reblog: Someone boosted your post
report: New report is submitted report: A new report is submitted
trending_tag: An unreviewed hashtag is trending trending_tag: A new trend requires approval
rule: rule:
text: Rule text: Rule
tag: tag:

View file

@ -34,12 +34,16 @@ SimpleNavigation::Configuration.run do |navigation|
n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: proc { Setting.min_invite_role == 'user' && current_user.functional? } n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: proc { Setting.min_invite_role == 'user' && current_user.functional? }
n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? } n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? }
n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_tags_path, if: proc { current_user.staff? } do |s|
s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags}
s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links}
end
n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), admin_reports_url, if: proc { current_user.staff? } do |s| n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), admin_reports_url, if: proc { current_user.staff? } do |s|
s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_url s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_url
s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports} s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts} s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations} s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations}
s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? } s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? } s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }

View file

@ -301,14 +301,29 @@ Rails.application.routes.draw do
resources :account_moderation_notes, only: [:create, :destroy] resources :account_moderation_notes, only: [:create, :destroy]
resource :follow_recommendations, only: [:show, :update] resource :follow_recommendations, only: [:show, :update]
resources :tags, only: [:show, :update]
resources :tags, only: [:index, :show, :update] do namespace :trends do
resources :links, only: [:index] do
collection do collection do
post :approve_all
post :reject_all
post :batch post :batch
end end
end end
resources :tags, only: [:index] do
collection do
post :batch
end
end
namespace :links do
resources :preview_card_providers, only: [:index], path: :publishers do
collection do
post :batch
end
end
end
end
end end
get '/admin', to: redirect('/admin/dashboard', status: 302) get '/admin', to: redirect('/admin/dashboard', status: 302)
@ -399,7 +414,7 @@ Rails.application.routes.draw do
resources :favourites, only: [:index] resources :favourites, only: [:index]
resources :bookmarks, only: [:index] resources :bookmarks, only: [:index]
resources :reports, only: [:create] resources :reports, only: [:create]
resources :trends, only: [:index] resources :trends, only: [:index], controller: 'trends/tags'
resources :filters, only: [:index, :create, :show, :update, :destroy] resources :filters, only: [:index, :create, :show, :update, :destroy]
resources :endorsements, only: [:index] resources :endorsements, only: [:index]
resources :markers, only: [:index, :create] resources :markers, only: [:index, :create]
@ -410,6 +425,11 @@ Rails.application.routes.draw do
resources :apps, only: [:create] resources :apps, only: [:create]
namespace :trends do
resources :links, only: [:index]
resources :tags, only: [:index]
end
namespace :emails do namespace :emails do
resources :confirmations, only: [:create] resources :confirmations, only: [:create]
end end
@ -512,7 +532,9 @@ Rails.application.routes.draw do
end end
end end
resources :trends, only: [:index] namespace :trends do
resources :tags, only: [:index]
end
post :measures, to: 'measures#create' post :measures, to: 'measures#create'
post :dimensions, to: 'dimensions#create' post :dimensions, to: 'dimensions#create'

View file

@ -13,9 +13,13 @@
every: '5m' every: '5m'
class: Scheduler::ScheduledStatusesScheduler class: Scheduler::ScheduledStatusesScheduler
queue: scheduler queue: scheduler
trending_tags_scheduler: trends_refresh_scheduler:
every: '5m' every: '5m'
class: Scheduler::TrendingTagsScheduler class: Scheduler::Trends::RefreshScheduler
queue: scheduler
trends_review_notifications_scheduler:
every: '2h'
class: Scheduler::Trends::ReviewNotificationsScheduler
queue: scheduler queue: scheduler
media_cleanup_scheduler: media_cleanup_scheduler:
cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'

View file

@ -0,0 +1,12 @@
class CreatePreviewCardProviders < ActiveRecord::Migration[6.1]
def change
create_table :preview_card_providers do |t|
t.string :domain, null: false, default: '', index: { unique: true }
t.attachment :icon
t.boolean :trendable
t.datetime :reviewed_at
t.datetime :requested_review_at
t.timestamps
end
end
end

View file

@ -0,0 +1,7 @@
class AddLanguageToPreviewCards < ActiveRecord::Migration[6.1]
def change
add_column :preview_cards, :language, :string
add_column :preview_cards, :max_score, :float
add_column :preview_cards, :max_score_at, :datetime
end
end

View file

@ -0,0 +1,5 @@
class AddTrendableToPreviewCards < ActiveRecord::Migration[6.1]
def change
add_column :preview_cards, :trendable, :boolean
end
end

View file

@ -0,0 +1,5 @@
class AddLinkTypeToPreviewCards < ActiveRecord::Migration[6.1]
def change
add_column :preview_cards, :link_type, :int
end
end

View file

@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2021_08_08_071221) do ActiveRecord::Schema.define(version: 2021_11_23_212714) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
@ -689,6 +689,20 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
t.index ["status_id"], name: "index_polls_on_status_id" t.index ["status_id"], name: "index_polls_on_status_id"
end end
create_table "preview_card_providers", force: :cascade do |t|
t.string "domain", default: "", null: false
t.string "icon_file_name"
t.string "icon_content_type"
t.bigint "icon_file_size"
t.datetime "icon_updated_at"
t.boolean "trendable"
t.datetime "reviewed_at"
t.datetime "requested_review_at"
t.datetime "created_at", precision: 6, null: false
t.datetime "updated_at", precision: 6, null: false
t.index ["domain"], name: "index_preview_card_providers_on_domain", unique: true
end
create_table "preview_cards", force: :cascade do |t| create_table "preview_cards", force: :cascade do |t|
t.string "url", default: "", null: false t.string "url", default: "", null: false
t.string "title", default: "", null: false t.string "title", default: "", null: false
@ -710,6 +724,11 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
t.string "embed_url", default: "", null: false t.string "embed_url", default: "", null: false
t.integer "image_storage_schema_version" t.integer "image_storage_schema_version"
t.string "blurhash" t.string "blurhash"
t.string "language"
t.float "max_score"
t.datetime "max_score_at"
t.boolean "trendable"
t.integer "link_type"
t.index ["url"], name: "index_preview_cards_on_url", unique: true t.index ["url"], name: "index_preview_cards_on_url", unique: true
end end

View file

@ -84,10 +84,7 @@ module Mastodon::Snowflake
-- Take the first two bytes (four hex characters) -- Take the first two bytes (four hex characters)
substr( substr(
-- Of the MD5 hash of the data we documented -- Of the MD5 hash of the data we documented
md5(table_name || md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),
'#{SecureRandom.hex(16)}' ||
time_part::text
),
1, 4 1, 4
) )
-- And turn it into a bigint -- And turn it into a bigint

View file

@ -96,7 +96,7 @@ namespace :repo do
end.uniq.compact end.uniq.compact
missing_available_locales = locales_in_files - I18n.available_locales missing_available_locales = locales_in_files - I18n.available_locales
missing_locale_names = I18n.available_locales.reject { |locale| SettingsHelper::HUMAN_LOCALES.key?(locale) } missing_locale_names = I18n.available_locales.reject { |locale| LanguagesHelper::HUMAN_LOCALES.key?(locale) }
critical = false critical = false

View file

@ -9,18 +9,6 @@ RSpec.describe Admin::TagsController, type: :controller do
sign_in Fabricate(:user, admin: true) sign_in Fabricate(:user, admin: true)
end end
describe 'GET #index' do
let!(:tag) { Fabricate(:tag) }
before do
get :index
end
it 'returns status 200' do
expect(response).to have_http_status(200)
end
end
describe 'GET #show' do describe 'GET #show' do
let!(:tag) { Fabricate(:tag) } let!(:tag) { Fabricate(:tag) }

View file

@ -0,0 +1,22 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Api::V1::Trends::TagsController, type: :controller do
render_views
describe 'GET #index' do
before do
trending_tags = double()
allow(trending_tags).to receive(:get).and_return(Fabricate.times(10, :tag))
allow(Trends).to receive(:tags).and_return(trending_tags)
get :index
end
it 'returns http success' do
expect(response).to have_http_status(200)
end
end
end

View file

@ -1,18 +0,0 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Api::V1::TrendsController, type: :controller do
render_views
describe 'GET #index' do
before do
allow(TrendingTags).to receive(:get).and_return(Fabricate.times(10, :tag))
get :index
end
it 'returns http success' do
expect(response).to have_http_status(200)
end
end
end

View file

@ -2,20 +2,15 @@
require 'rails_helper' require 'rails_helper'
describe SettingsHelper do describe LanguagesHelper do
describe 'the HUMAN_LOCALES constant' do describe 'the HUMAN_LOCALES constant' do
it 'includes all I18n locales' do it 'includes all I18n locales' do
options = I18n.available_locales expect(described_class::HUMAN_LOCALES.keys).to include(*I18n.available_locales)
expect(described_class::HUMAN_LOCALES.keys).to include(*options)
end end
end end
describe 'human_locale' do describe 'human_locale' do
it 'finds the human readable local description from a key' do it 'finds the human readable local description from a key' do
# Ensure the value is as we expect
expect(described_class::HUMAN_LOCALES[:en]).to eq('English')
expect(helper.human_locale(:en)).to eq('English') expect(helper.human_locale(:en)).to eq('English')
end end
end end

View file

@ -5,4 +5,14 @@ class AdminMailerPreview < ActionMailer::Preview
def new_pending_account def new_pending_account
AdminMailer.new_pending_account(Account.first, User.pending.first) AdminMailer.new_pending_account(Account.first, User.pending.first)
end end
# Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_tags
def new_trending_tags
AdminMailer.new_trending_tags(Account.first, Tag.limit(3))
end
# Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_links
def new_trending_links
AdminMailer.new_trending_links(Account.first, PreviewCard.limit(3))
end
end end

View file

@ -1,68 +0,0 @@
require 'rails_helper'
RSpec.describe TrendingTags do
describe '.record_use!' do
pending
end
describe '.update!' do
let!(:at_time) { Time.now.utc }
let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
before do
allow(Redis.current).to receive(:pfcount) do |key|
case key
when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
2
when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts"
16
when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
0
when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts"
4
when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
13
end
end
Redis.current.zadd('trending_tags', 0.9, tag3.id)
Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id])
tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours)
described_class.update!(at_time)
end
it 'calculates and re-calculates scores' do
expect(described_class.get(10, filtered: false)).to eq [tag1, tag3]
end
it 'omits hashtags below threshold' do
expect(described_class.get(10, filtered: false)).to_not include(tag2)
end
it 'decays scores' do
expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9
end
end
describe '.trending?' do
let(:tag) { Fabricate(:tag) }
before do
10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) }
end
it 'returns true if the hashtag is within limit' do
Redis.current.zadd('trending_tags', 11, tag.id)
expect(described_class.trending?(tag)).to be true
end
it 'returns false if the hashtag is outside the limit' do
Redis.current.zadd('trending_tags', 0, tag.id)
expect(described_class.trending?(tag)).to be false
end
end
end

View file

@ -0,0 +1,67 @@
require 'rails_helper'
RSpec.describe Trends::Tags do
subject { described_class.new(threshold: 5, review_threshold: 10) }
let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) }
describe '#add' do
let(:tag) { Fabricate(:tag) }
before do
subject.add(tag, 1, at_time)
end
it 'records history' do
expect(tag.history.get(at_time).accounts).to eq 1
end
it 'records use' do
expect(subject.send(:recently_used_ids, at_time)).to eq [tag.id]
end
end
describe '#get' do
pending
end
describe '#refresh' do
let!(:today) { at_time }
let!(:yesterday) { today - 1.day }
let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
before do
2.times { |i| subject.add(tag1, i, yesterday) }
13.times { |i| subject.add(tag3, i, yesterday) }
16.times { |i| subject.add(tag1, i, today) }
4.times { |i| subject.add(tag2, i, today) }
end
context do
before do
subject.refresh(yesterday + 12.hours)
subject.refresh(at_time)
end
it 'calculates and re-calculates scores' do
expect(subject.get(false, 10)).to eq [tag1, tag3]
end
it 'omits hashtags below threshold' do
expect(subject.get(false, 10)).to_not include(tag2)
end
end
it 'decays scores' do
subject.refresh(yesterday + 12.hours)
original_score = subject.score(tag3.id)
expect(original_score).to eq 144.0
subject.refresh(yesterday + 12.hours + subject.options[:max_score_halflife])
decayed_score = subject.score(tag3.id)
expect(decayed_score).to be <= original_score / 2
end
end
end