~cytrogen/masto-fe

ref: 937dc42f101be905e3af41b879901a4445b0223a masto-fe/app/models/form/import.rb -rw-r--r-- 5.5 KiB
937dc42f — Matt Jankowski Extract methods for file movement in `CLI::Upgrade` (#25120) 2 years ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
# frozen_string_literal: true

require 'csv'

# A non-ActiveRecord helper class for CSV uploads.
# Handles saving contents to database.
class Form::Import
  include ActiveModel::Model

  MODES = %i(merge overwrite).freeze

  FILE_SIZE_LIMIT       = 20.megabytes
  ROWS_PROCESSING_LIMIT = 20_000

  EXPECTED_HEADERS_BY_TYPE = {
    following: ['Account address', 'Show boosts', 'Notify on new posts', 'Languages'],
    blocking: ['Account address'],
    muting: ['Account address', 'Hide notifications'],
    domain_blocking: ['#domain'],
    bookmarks: ['#uri'],
    lists: ['List name', 'Account address'],
  }.freeze

  KNOWN_FIRST_HEADERS = EXPECTED_HEADERS_BY_TYPE.values.map(&:first).uniq.freeze

  ATTRIBUTE_BY_HEADER = {
    'Account address' => 'acct',
    'Show boosts' => 'show_reblogs',
    'Notify on new posts' => 'notify',
    'Languages' => 'languages',
    'Hide notifications' => 'hide_notifications',
    '#domain' => 'domain',
    '#uri' => 'uri',
    'List name' => 'list_name',
  }.freeze

  class EmptyFileError < StandardError; end

  attr_accessor :current_account, :data, :type, :overwrite, :bulk_import

  validates :type, presence: true
  validates :data, presence: true
  validate :validate_data

  def guessed_type
    return :muting if csv_data.headers.include?('Hide notifications')
    return :following if csv_data.headers.include?('Show boosts') || csv_data.headers.include?('Notify on new posts') || csv_data.headers.include?('Languages')
    return :following if data.original_filename&.start_with?('follows') || data.original_filename&.start_with?('following_accounts')
    return :blocking if data.original_filename&.start_with?('blocks') || data.original_filename&.start_with?('blocked_accounts')
    return :muting if data.original_filename&.start_with?('mutes') || data.original_filename&.start_with?('muted_accounts')
    return :domain_blocking if data.original_filename&.start_with?('domain_blocks') || data.original_filename&.start_with?('blocked_domains')
    return :bookmarks if data.original_filename&.start_with?('bookmarks')
    return :lists if data.original_filename&.start_with?('lists')
  end

  # Whether the uploaded CSV file seems to correspond to a different import type than the one selected
  def likely_mismatched?
    guessed_type.present? && guessed_type != type.to_sym
  end

  def save
    return false unless valid?

    ApplicationRecord.transaction do
      now = Time.now.utc
      @bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?)
      nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length # rubocop:disable Rails/SkipsModelValidations
      @bulk_import.update(total_items: nb_items)
    end
  end

  def mode
    overwrite ? :overwrite : :merge
  end

  def mode=(str)
    self.overwrite = str.to_sym == :overwrite
  end

  private

  def default_csv_headers
    case type.to_sym
    when :following, :blocking, :muting
      ['Account address']
    when :domain_blocking
      ['#domain']
    when :bookmarks
      ['#uri']
    when :lists
      ['List name', 'Account address']
    end
  end

  def csv_data
    return @csv_data if defined?(@csv_data)

    csv_converter = lambda do |field, field_info|
      case field_info.header
      when 'Show boosts', 'Notify on new posts', 'Hide notifications'
        ActiveModel::Type::Boolean.new.cast(field)
      when 'Languages'
        field&.split(',')&.map(&:strip)&.presence
      when 'Account address'
        field.strip.gsub(/\A@/, '')
      when '#domain', '#uri', 'List name'
        field.strip
      else
        field
      end
    end

    @csv_data = CSV.open(data.path, encoding: 'UTF-8', skip_blanks: true, headers: true, converters: csv_converter)
    @csv_data.take(1) # Ensure the headers are read
    raise EmptyFileError if @csv_data.headers == true

    @csv_data = CSV.open(data.path, encoding: 'UTF-8', skip_blanks: true, headers: default_csv_headers, converters: csv_converter) unless KNOWN_FIRST_HEADERS.include?(@csv_data.headers&.first)
    @csv_data
  end

  def csv_row_count
    return @csv_row_count if defined?(@csv_row_count)

    csv_data.rewind
    @csv_row_count = csv_data.take(ROWS_PROCESSING_LIMIT + 2).count
  end

  def parsed_rows
    csv_data.rewind

    expected_headers = EXPECTED_HEADERS_BY_TYPE[type.to_sym]

    csv_data.take(ROWS_PROCESSING_LIMIT + 1).map do |row|
      row.to_h.slice(*expected_headers).transform_keys { |key| ATTRIBUTE_BY_HEADER[key] }
    end
  end

  def validate_data
    return if data.nil?
    return errors.add(:data, I18n.t('imports.errors.too_large')) if data.size > FILE_SIZE_LIMIT
    return errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless default_csv_headers.all? { |header| csv_data.headers.include?(header) }

    errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if csv_row_count > ROWS_PROCESSING_LIMIT

    if type.to_sym == :following
      base_limit = FollowLimitValidator.limit_for_account(current_account)
      limit = base_limit
      limit -= current_account.following_count unless overwrite
      errors.add(:data, I18n.t('users.follow_limit_reached', limit: base_limit)) if csv_row_count > limit
    end
  rescue CSV::MalformedCSVError => e
    errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
  rescue EmptyFileError
    errors.add(:data, I18n.t('imports.errors.empty'))
  end
end