~cytrogen/masto-fe

f7f23b4a19a84371f44ec5297125e96ba81681a1 — Eugen Rochko 6 years ago e9a11dc
Add audio uploads (#11123)

* Add audio uploads

Fix #4827

Accept uploads of OGG, WAV, FLAC, OPUS and MP3 files, and converts
them to OGG. Media attachments get a new `audio` type. In the UI,
audio uploads are displayed identically to video uploads.

* Improve code style
M app/controllers/media_controller.rb => app/controllers/media_controller.rb +10 -2
@@ 7,6 7,8 @@ class MediaController < ApplicationController

  before_action :set_media_attachment
  before_action :verify_permitted_status!
  before_action :check_playable, only: :player
  before_action :allow_iframing, only: :player

  content_security_policy only: :player do |p|
    p.frame_ancestors(false)


@@ 18,8 20,6 @@ class MediaController < ApplicationController

  def player
    @body_classes = 'player'
    response.headers['X-Frame-Options'] = 'ALLOWALL'
    raise ActiveRecord::RecordNotFound unless @media_attachment.video? || @media_attachment.gifv?
  end

  private


@@ 34,4 34,12 @@ class MediaController < ApplicationController
    # Reraise in order to get a 404 instead of a 403 error code
    raise ActiveRecord::RecordNotFound
  end

  def check_playable
    not_found unless @media_attachment.larger_media_format?
  end

  def allow_iframing
    response.headers['X-Frame-Options'] = 'ALLOWALL'
  end
end

M app/javascript/mastodon/components/status.js => app/javascript/mastodon/components/status.js +6 -6
@@ 333,17 333,17 @@ class Status extends ImmutablePureComponent {
            media={status.get('media_attachments')}
          />
        );
      } else if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
        const video = status.getIn(['media_attachments', 0]);
      } else if (['video', 'audio'].includes(status.getIn(['media_attachments', 0, 'type']))) {
        const attachment = status.getIn(['media_attachments', 0]);

        media = (
          <Bundle fetchComponent={Video} loading={this.renderLoadingVideoPlayer} >
            {Component => (
              <Component
                preview={video.get('preview_url')}
                blurhash={video.get('blurhash')}
                src={video.get('url')}
                alt={video.get('description')}
                preview={attachment.get('preview_url')}
                blurhash={attachment.get('blurhash')}
                src={attachment.get('url')}
                alt={attachment.get('description')}
                width={this.props.cachedMediaWidth}
                height={110}
                inline

M app/javascript/mastodon/features/compose/components/upload_button.js => app/javascript/mastodon/features/compose/components/upload_button.js +5 -3
@@ 7,9 7,11 @@ import ImmutablePureComponent from 'react-immutable-pure-component';
import ImmutablePropTypes from 'react-immutable-proptypes';

const messages = defineMessages({
  upload: { id: 'upload_button.label', defaultMessage: 'Add media (JPEG, PNG, GIF, WebM, MP4, MOV)' },
  upload: { id: 'upload_button.label', defaultMessage: 'Add media ({formats})' },
});

const SUPPORTED_FORMATS = 'JPEG, PNG, GIF, WebM, MP4, MOV, OGG, WAV, MP3, FLAC';

const makeMapStateToProps = () => {
  const mapStateToProps = state => ({
    acceptContentTypes: state.getIn(['media_attachments', 'accept_content_types']),


@@ 60,9 62,9 @@ class UploadButton extends ImmutablePureComponent {

    return (
      <div className='compose-form__upload-button'>
        <IconButton icon='camera' title={intl.formatMessage(messages.upload)} disabled={disabled} onClick={this.handleClick} className='compose-form__upload-button-icon' size={18} inverted style={iconStyle} />
        <IconButton icon='camera' title={intl.formatMessage(messages.upload, { formats: SUPPORTED_FORMATS })} disabled={disabled} onClick={this.handleClick} className='compose-form__upload-button-icon' size={18} inverted style={iconStyle} />
        <label>
          <span style={{ display: 'none' }}>{intl.formatMessage(messages.upload)}</span>
          <span style={{ display: 'none' }}>{intl.formatMessage(messages.upload, { formats: SUPPORTED_FORMATS })}</span>
          <input
            key={resetFileKey}
            ref={this.setRef}

M app/javascript/mastodon/features/compose/containers/upload_button_container.js => app/javascript/mastodon/features/compose/containers/upload_button_container.js +1 -1
@@ 3,7 3,7 @@ import UploadButton from '../components/upload_button';
import { uploadCompose } from '../../../actions/compose';

const mapStateToProps = state => ({
  disabled: state.getIn(['compose', 'is_uploading']) || (state.getIn(['compose', 'media_attachments']).size > 3 || state.getIn(['compose', 'media_attachments']).some(m => m.get('type') === 'video')),
  disabled: state.getIn(['compose', 'is_uploading']) || (state.getIn(['compose', 'media_attachments']).size > 3 || state.getIn(['compose', 'media_attachments']).some(m => ['video', 'audio'].includes(m.get('type')))),
  unavailable: state.getIn(['compose', 'poll']) !== null,
  resetFileKey: state.getIn(['compose', 'resetFileKey']),
});

M app/javascript/mastodon/features/status/components/detailed_status.js => app/javascript/mastodon/features/status/components/detailed_status.js +6 -6
@@ 107,15 107,15 @@ export default class DetailedStatus extends ImmutablePureComponent {
    }

    if (status.get('media_attachments').size > 0) {
      if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
        const video = status.getIn(['media_attachments', 0]);
      if (['video', 'audio'].includes(status.getIn(['media_attachments', 0, 'type']))) {
        const attachment = status.getIn(['media_attachments', 0]);

        media = (
          <Video
            preview={video.get('preview_url')}
            blurhash={video.get('blurhash')}
            src={video.get('url')}
            alt={video.get('description')}
            preview={attachment.get('preview_url')}
            blurhash={attachment.get('blurhash')}
            src={attachment.get('url')}
            alt={attachment.get('description')}
            width={300}
            height={150}
            inline

M app/javascript/mastodon/locales/en.json => app/javascript/mastodon/locales/en.json +1 -1
@@ 369,7 369,7 @@
  "trends.count_by_accounts": "{count} {rawCount, plural, one {person} other {people}} talking",
  "ui.beforeunload": "Your draft will be lost if you leave Mastodon.",
  "upload_area.title": "Drag & drop to upload",
  "upload_button.label": "Add media (JPEG, PNG, GIF, WebM, MP4, MOV)",
  "upload_button.label": "Add media ({formats})",
  "upload_error.limit": "File upload limit exceeded.",
  "upload_error.poll": "File upload not allowed with polls.",
  "upload_form.description": "Describe for the visually impaired",

M app/models/media_attachment.rb => app/models/media_attachment.rb +43 -21
@@ 24,14 24,16 @@
class MediaAttachment < ApplicationRecord
  self.inheritance_column = nil

  enum type: [:image, :gifv, :video, :unknown]
  enum type: [:image, :gifv, :video, :unknown, :audio]

  IMAGE_FILE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp'].freeze
  VIDEO_FILE_EXTENSIONS = ['.webm', '.mp4', '.m4v', '.mov'].freeze
  AUDIO_FILE_EXTENSIONS = ['.ogg', '.oga', '.mp3', '.wav', '.flac', '.opus'].freeze

  IMAGE_MIME_TYPES             = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
  VIDEO_MIME_TYPES             = ['video/webm', 'video/mp4', 'video/quicktime'].freeze
  VIDEO_CONVERTIBLE_MIME_TYPES = ['video/webm', 'video/quicktime'].freeze
  AUDIO_MIME_TYPES             = ['audio/wave', 'audio/wav', 'audio/x-wav', 'audio/x-pn-wave', 'audio/ogg', 'audio/mpeg', 'audio/webm', 'audio/flac'].freeze

  BLURHASH_OPTIONS = {
    x_comp: 4,


@@ 65,6 67,13 @@ class MediaAttachment < ApplicationRecord
    },
  }.freeze

  AUDIO_STYLES = {
    original: {
      format: 'ogg',
      convert_options: {},
    },
  }.freeze

  VIDEO_FORMAT = {
    format: 'mp4',
    convert_options: {


@@ 83,6 92,11 @@ class MediaAttachment < ApplicationRecord
    },
  }.freeze

  VIDEO_CONVERTED_STYLES = {
    small: VIDEO_STYLES[:small],
    original: VIDEO_FORMAT,
  }.freeze

  IMAGE_LIMIT = 8.megabytes
  VIDEO_LIMIT = 40.megabytes



@@ 95,9 109,9 @@ class MediaAttachment < ApplicationRecord
                    processors: ->(f) { file_processors f },
                    convert_options: { all: '-quality 90 -strip' }

  validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES
  validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :video_or_gifv?
  validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :video_or_gifv?
  validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
  validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format?
  validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format?
  remotable_attachment :file, VIDEO_LIMIT

  include Attachmentable


@@ 120,8 134,12 @@ class MediaAttachment < ApplicationRecord
    file.blank? && remote_url.present?
  end

  def video_or_gifv?
    video? || gifv?
  def larger_media_format?
    video? || gifv? || audio?
  end

  def audio_or_video?
    audio? || video?
  end

  def to_param


@@ 156,28 174,24 @@ class MediaAttachment < ApplicationRecord
    private

    def file_styles(f)
      if f.instance.file_content_type == 'image/gif'
        {
          small: IMAGE_STYLES[:small],
          original: VIDEO_FORMAT,
        }
      elsif IMAGE_MIME_TYPES.include? f.instance.file_content_type
      if f.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type)
        VIDEO_CONVERTED_STYLES
      elsif IMAGE_MIME_TYPES.include?(f.instance.file_content_type)
        IMAGE_STYLES
      elsif VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type)
        {
          small: VIDEO_STYLES[:small],
          original: VIDEO_FORMAT,
        }
      else
      elsif VIDEO_MIME_TYPES.include?(f.instance.file_content_type)
        VIDEO_STYLES
      else
        AUDIO_STYLES
      end
    end

    def file_processors(f)
      if f.file_content_type == 'image/gif'
        [:gif_transcoder, :blurhash_transcoder]
      elsif VIDEO_MIME_TYPES.include? f.file_content_type
      elsif VIDEO_MIME_TYPES.include?(f.file_content_type)
        [:video_transcoder, :blurhash_transcoder]
      elsif AUDIO_MIME_TYPES.include?(f.file_content_type)
        [:transcoder]
      else
        [:lazy_thumbnail, :blurhash_transcoder]
      end


@@ 202,7 216,15 @@ class MediaAttachment < ApplicationRecord
  end

  def set_type_and_extension
    self.type = VIDEO_MIME_TYPES.include?(file_content_type) ? :video : :image
    self.type = begin
      if VIDEO_MIME_TYPES.include?(file_content_type)
        :video
      elsif AUDIO_MIME_TYPES.include?(file_content_type)
        :audio
      else
        :image
      end
    end
  end

  def set_meta


@@ 245,7 267,7 @@ class MediaAttachment < ApplicationRecord
      frame_rate: movie.frame_rate,
      duration: movie.duration,
      bitrate: movie.bitrate,
    }
    }.compact
  end

  def reset_parent_cache

M app/serializers/initial_state_serializer.rb => app/serializers/initial_state_serializer.rb +1 -1
@@ 60,7 60,7 @@ class InitialStateSerializer < ActiveModel::Serializer
  end

  def media_attachments
    { accept_content_types: MediaAttachment::IMAGE_FILE_EXTENSIONS + MediaAttachment::VIDEO_FILE_EXTENSIONS + MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES }
    { accept_content_types: MediaAttachment::IMAGE_FILE_EXTENSIONS + MediaAttachment::VIDEO_FILE_EXTENSIONS + MediaAttachment::AUDIO_FILE_EXTENSIONS + MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES + MediaAttachment::AUDIO_MIME_TYPES }
  end

  private

M app/services/post_status_service.rb => app/services/post_status_service.rb +1 -1
@@ 100,7 100,7 @@ class PostStatusService < BaseService

    @media = @account.media_attachments.where(status_id: nil).where(id: @options[:media_ids].take(4).map(&:to_i))

    raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:video?)
    raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:audio_or_video?)
  end

  def language_from_option(str)

M app/views/stream_entries/_detailed_status.html.haml => app/views/stream_entries/_detailed_status.html.haml +1 -1
@@ 27,7 27,7 @@
          = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }

  - if !status.media_attachments.empty?
    - if status.media_attachments.first.video?
    - if status.media_attachments.first.audio_or_video?
      - video = status.media_attachments.first
      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do
        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }

M app/views/stream_entries/_simple_status.html.haml => app/views/stream_entries/_simple_status.html.haml +1 -1
@@ 31,7 31,7 @@
          = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }

  - if !status.media_attachments.empty?
    - if status.media_attachments.first.video?
    - if status.media_attachments.first.audio_or_video?
      - video = status.media_attachments.first
      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description do
        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }