32 files changed, 575 insertions(+), 77 deletions(-)

M Gemfile
M Gemfile.lock
M app/chewy/accounts_index.rb
M app/controllers/api/v1/directories_controller.rb
M app/controllers/api/v1/emails/confirmations_controller.rb
M app/javascript/mastodon/actions/server.js
M app/javascript/mastodon/features/about/index.jsx
M app/javascript/mastodon/features/compose/components/action_bar.jsx
A app/javascript/mastodon/features/firehose/index.jsx
M app/javascript/mastodon/features/home_timeline/index.jsx
M app/javascript/mastodon/features/ui/components/header.jsx
M app/javascript/mastodon/features/ui/components/navigation_panel.jsx
M app/javascript/mastodon/features/ui/index.jsx
M app/javascript/mastodon/features/ui/util/async-components.js
M app/javascript/mastodon/locales/en.json
M app/javascript/mastodon/reducers/index.ts
M app/javascript/mastodon/reducers/server.js
M app/javascript/mastodon/reducers/settings.js
M app/models/account.rb
M app/models/concerns/account_search.rb
M app/services/account_search_service.rb
M app/services/resolve_url_service.rb
M app/services/search_service.rb
M config/routes.rb
M config/routes/admin.rb
M crowdin.yml
A db/migrate/20230630145300_add_index_backups_on_user_id.rb
M db/schema.rb
M spec/controllers/api/v1/directories_controller_spec.rb
M spec/controllers/api/v1/emails/confirmations_controller_spec.rb
M spec/services/resolve_url_service_spec.rb
M spec/services/search_service_spec.rb

@@ 3,8 3,6 @@
 source 'https://rubygems.org'
 ruby '>= 3.0.0'
 
-gem 'pkg-config', '~> 1.5'
-
 gem 'puma', '~> 6.3'
 gem 'rails', '~> 6.1.7'
 gem 'sprockets', '~> 3.7.2'

@@ 478,7 478,6 @@ GEM
     pg (1.5.3)
     pghero (3.3.3)
       activerecord (>= 6)
-    pkg-config (1.5.1)
     posix-spawn (0.3.15)
     premailer (1.21.0)
       addressable


@@ 717,7 716,7 @@ GEM
       unf_ext
     unf_ext (0.0.8.2)
     unicode-display_width (2.4.2)
-    uri (0.12.1)
+    uri (0.12.2)
     validate_email (0.1.6)
       activemodel (>= 3.0)
       mail (>= 2.2.5)


@@ 833,7 832,6 @@ DEPENDENCIES
   parslet
   pg (~> 1.5)
   pghero
-  pkg-config (~> 1.5)
   posix-spawn
   premailer-rails
   private_address_check (~> 0.5)

@@ 2,8 2,37 @@
 
 class AccountsIndex < Chewy::Index
   settings index: { refresh_interval: '30s' }, analysis: {
+    filter: {
+      english_stop: {
+        type: 'stop',
+        stopwords: '_english_',
+      },
+
+      english_stemmer: {
+        type: 'stemmer',
+        language: 'english',
+      },
+
+      english_possessive_stemmer: {
+        type: 'stemmer',
+        language: 'possessive_english',
+      },
+    },
+
     analyzer: {
-      content: {
+      natural: {
+        tokenizer: 'uax_url_email',
+        filter: %w(
+          english_possessive_stemmer
+          lowercase
+          asciifolding
+          cjk_width
+          english_stop
+          english_stemmer
+        ),
+      },
+
+      verbatim: {
         tokenizer: 'whitespace',
         filter: %w(lowercase asciifolding cjk_width),
       },


@@ 26,18 55,13 @@ class AccountsIndex < Chewy::Index
   index_scope ::Account.searchable.includes(:account_stat)
 
   root date_detection: false do
-    field :id, type: 'long'
-
-    field :display_name, type: 'text', analyzer: 'content' do
-      field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
-    end
-
-    field :acct, type: 'text', analyzer: 'content', value: ->(account) { [account.username, account.domain].compact.join('@') } do
-      field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
-    end
-
-    field :following_count, type: 'long', value: ->(account) { account.following_count }
-    field :followers_count, type: 'long', value: ->(account) { account.followers_count }
-    field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
+    field(:id, type: 'long')
+    field(:following_count, type: 'long')
+    field(:followers_count, type: 'long')
+    field(:properties, type: 'keyword', value: ->(account) { account.searchable_properties })
+    field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at })
+    field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
+    field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
+    field(:text, type: 'text', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
   end
 end

@@ 21,11 21,35 @@ class Api::V1::DirectoriesController < Api::BaseController
 
   def accounts_scope
     Account.discoverable.tap do |scope|
-      scope.merge!(Account.local)                                          if truthy_param?(:local)
-      scope.merge!(Account.by_recent_status)                               if params[:order].blank? || params[:order] == 'active'
-      scope.merge!(Account.order(id: :desc))                               if params[:order] == 'new'
-      scope.merge!(Account.not_excluded_by_account(current_account))       if current_account
-      scope.merge!(Account.not_domain_blocked_by_account(current_account)) if current_account && !truthy_param?(:local)
+      scope.merge!(account_order_scope)
+      scope.merge!(local_account_scope) if local_accounts?
+      scope.merge!(account_exclusion_scope) if current_account
+      scope.merge!(account_domain_block_scope) if current_account && !local_accounts?
     end
   end
+
+  def local_accounts?
+    truthy_param?(:local)
+  end
+
+  def account_order_scope
+    case params[:order]
+    when 'new'
+      Account.order(id: :desc)
+    when 'active', nil
+      Account.by_recent_status
+    end
+  end
+
+  def local_account_scope
+    Account.local
+  end
+
+  def account_exclusion_scope
+    Account.not_excluded_by_account(current_account)
+  end
+
+  def account_domain_block_scope
+    Account.not_domain_blocked_by_account(current_account)
+  end
 end

@@ 5,6 5,7 @@ class Api::V1::Emails::ConfirmationsController < Api::BaseController
   before_action -> { doorkeeper_authorize! :write, :'write:accounts' }, except: :check
   before_action :require_user_owned_by_application!, except: :check
   before_action :require_user_not_confirmed!, except: :check
+  before_action :require_authenticated_user!, only: :check
 
   def create
     current_user.update!(email: params[:email]) if params.key?(:email)

@@ 19,6 19,10 @@ export const SERVER_DOMAIN_BLOCKS_FETCH_SUCCESS = 'SERVER_DOMAIN_BLOCKS_FETCH_SU
 export const SERVER_DOMAIN_BLOCKS_FETCH_FAIL    = 'SERVER_DOMAIN_BLOCKS_FETCH_FAIL';
 
 export const fetchServer = () => (dispatch, getState) => {
+  if (getState().getIn(['server', 'server', 'isLoading'])) {
+    return;
+  }
+
   dispatch(fetchServerRequest());
 
   api(getState)


@@ 66,6 70,10 @@ const fetchServerTranslationLanguagesFail = error => ({
 });
 
 export const fetchExtendedDescription = () => (dispatch, getState) => {
+  if (getState().getIn(['server', 'extendedDescription', 'isLoading'])) {
+    return;
+  }
+
   dispatch(fetchExtendedDescriptionRequest());
 
   api(getState)


@@ 89,6 97,10 @@ const fetchExtendedDescriptionFail = error => ({
 });
 
 export const fetchDomainBlocks = () => (dispatch, getState) => {
+  if (getState().getIn(['server', 'domainBlocks', 'isLoading'])) {
+    return;
+  }
+
   dispatch(fetchDomainBlocksRequest());
 
   api(getState)

@@ 161,7 161,7 @@ class About extends PureComponent {
           </Section>
 
           <Section title={intl.formatMessage(messages.rules)}>
-            {!isLoading && (server.get('rules').isEmpty() ? (
+            {!isLoading && (server.get('rules', []).isEmpty() ? (
               <p><FormattedMessage id='about.not_available' defaultMessage='This information has not been made available on this server.' /></p>
             ) : (
               <ol className='rules-list'>

@@ 60,7 60,7 @@ class ActionBar extends PureComponent {
     return (
       <div className='compose__action-bar'>
         <div className='compose__action-bar-dropdown'>
-          <DropdownMenuContainer items={menu} icon='ellipsis-v' size={18} direction='right' />
+          <DropdownMenuContainer items={menu} icon='bars' size={18} direction='right' />
         </div>
       </div>
     );

@@ 0,0 1,210 @@
+import PropTypes from 'prop-types';
+import { useRef, useCallback, useEffect } from 'react';
+
+import { useIntl, defineMessages, FormattedMessage } from 'react-intl';
+
+import { Helmet } from 'react-helmet';
+import { NavLink } from 'react-router-dom';
+
+import { addColumn } from 'mastodon/actions/columns';
+import { changeSetting } from 'mastodon/actions/settings';
+import { connectPublicStream, connectCommunityStream } from 'mastodon/actions/streaming';
+import { expandPublicTimeline, expandCommunityTimeline } from 'mastodon/actions/timelines';
+import DismissableBanner from 'mastodon/components/dismissable_banner';
+import initialState, { domain } from 'mastodon/initial_state';
+import { useAppDispatch, useAppSelector } from 'mastodon/store';
+
+import Column from '../../components/column';
+import ColumnHeader from '../../components/column_header';
+import SettingToggle from '../notifications/components/setting_toggle';
+import StatusListContainer from '../ui/containers/status_list_container';
+
+const messages = defineMessages({
+  title: { id: 'column.firehose', defaultMessage: 'Live feeds' },
+});
+
+// TODO: use a proper React context later on
+const useIdentity = () => ({
+  signedIn: !!initialState.meta.me,
+  accountId: initialState.meta.me,
+  disabledAccountId: initialState.meta.disabled_account_id,
+  accessToken: initialState.meta.access_token,
+  permissions: initialState.role ? initialState.role.permissions : 0,
+});
+
+const ColumnSettings = () => {
+  const dispatch = useAppDispatch();
+  const settings = useAppSelector((state) => state.getIn(['settings', 'firehose']));
+  const onChange = useCallback(
+    (key, checked) => dispatch(changeSetting(['firehose', ...key], checked)),
+    [dispatch],
+  );
+
+  return (
+    <div>
+      <div className='column-settings__row'>
+        <SettingToggle
+          settings={settings}
+          settingPath={['onlyMedia']}
+          onChange={onChange}
+          label={<FormattedMessage id='community.column_settings.media_only' defaultMessage='Media only' />}
+        />
+      </div>
+    </div>
+  );
+};
+
+const Firehose = ({ feedType, multiColumn }) => {
+  const dispatch = useAppDispatch();
+  const intl = useIntl();
+  const { signedIn } = useIdentity();
+  const columnRef = useRef(null);
+
+  const onlyMedia = useAppSelector((state) => state.getIn(['settings', 'firehose', 'onlyMedia'], false));
+  const hasUnread = useAppSelector((state) => state.getIn(['timelines', `${feedType}${onlyMedia ? ':media' : ''}`, 'unread'], 0) > 0);
+
+  const handlePin = useCallback(
+    () => {
+      switch(feedType) {
+      case 'community':
+        dispatch(addColumn('COMMUNITY', { other: { onlyMedia } }));
+        break;
+      case 'public':
+        dispatch(addColumn('PUBLIC', { other: { onlyMedia } }));
+        break;
+      case 'public:remote':
+        dispatch(addColumn('REMOTE', { other: { onlyMedia, onlyRemote: true } }));
+        break;
+      }
+    },
+    [dispatch, onlyMedia, feedType],
+  );
+
+  const handleLoadMore = useCallback(
+    (maxId) => {
+      switch(feedType) {
+      case 'community':
+        dispatch(expandCommunityTimeline({ onlyMedia }));
+        break;
+      case 'public':
+        dispatch(expandPublicTimeline({ maxId, onlyMedia }));
+        break;
+      case 'public:remote':
+        dispatch(expandPublicTimeline({ maxId, onlyMedia, onlyRemote: true }));
+        break;
+      }
+    },
+    [dispatch, onlyMedia, feedType],
+  );
+
+  const handleHeaderClick = useCallback(() => columnRef.current?.scrollTop(), []);
+
+  useEffect(() => {
+    let disconnect;
+
+    switch(feedType) {
+    case 'community':
+      dispatch(expandCommunityTimeline({ onlyMedia }));
+      if (signedIn) {
+        disconnect = dispatch(connectCommunityStream({ onlyMedia }));
+      }
+      break;
+    case 'public':
+      dispatch(expandPublicTimeline({ onlyMedia }));
+      if (signedIn) {
+        disconnect = dispatch(connectPublicStream({ onlyMedia }));
+      }
+      break;
+    case 'public:remote':
+      dispatch(expandPublicTimeline({ onlyMedia, onlyRemote: true }));
+      if (signedIn) {
+        disconnect = dispatch(connectPublicStream({ onlyMedia, onlyRemote: true }));
+      }
+      break;
+    }
+
+    return () => disconnect?.();
+  }, [dispatch, signedIn, feedType, onlyMedia]);
+
+  const prependBanner = feedType === 'community' ? (
+    <DismissableBanner id='community_timeline'>
+      <FormattedMessage
+        id='dismissable_banner.community_timeline'
+        defaultMessage='These are the most recent public posts from people whose accounts are hosted by {domain}.'
+        values={{ domain }}
+      />
+    </DismissableBanner>
+  ) : (
+   <DismissableBanner id='public_timeline'>
+     <FormattedMessage
+       id='dismissable_banner.public_timeline'
+       defaultMessage='These are the most recent public posts from people on this and other servers of the decentralized network that this server knows about.'
+     />
+   </DismissableBanner>
+  );
+
+  const emptyMessage = feedType === 'community' ? (
+    <FormattedMessage
+      id='empty_column.community'
+      defaultMessage='The local timeline is empty. Write something publicly to get the ball rolling!'
+    />
+  ) : (
+   <FormattedMessage
+     id='empty_column.public'
+     defaultMessage='There is nothing here! Write something publicly, or manually follow users from other servers to fill it up'
+   />
+  );
+
+  return (
+    <Column bindToDocument={!multiColumn} ref={columnRef} label={intl.formatMessage(messages.title)}>
+      <ColumnHeader
+        icon='globe'
+        active={hasUnread}
+        title={intl.formatMessage(messages.title)}
+        onPin={handlePin}
+        onClick={handleHeaderClick}
+        multiColumn={multiColumn}
+      >
+        <ColumnSettings />
+      </ColumnHeader>
+
+      <div className='scrollable scrollable--flex'>
+        <div className='account__section-headline'>
+          <NavLink exact to='/public/local'>
+            <FormattedMessage tagName='div' id='firehose.local' defaultMessage='Local' />
+          </NavLink>
+
+          <NavLink exact to='/public/remote'>
+            <FormattedMessage tagName='div' id='firehose.remote' defaultMessage='Remote' />
+          </NavLink>
+
+          <NavLink exact to='/public'>
+            <FormattedMessage tagName='div' id='firehose.all' defaultMessage='All' />
+          </NavLink>
+        </div>
+
+        <StatusListContainer
+          prepend={prependBanner}
+          timelineId={`${feedType}${onlyMedia ? ':media' : ''}`}
+          onLoadMore={handleLoadMore}
+          trackScroll
+          scrollKey='firehose'
+          emptyMessage={emptyMessage}
+          bindToDocument={!multiColumn}
+        />
+      </div>
+
+      <Helmet>
+        <title>{intl.formatMessage(messages.title)}</title>
+        <meta name='robots' content='noindex' />
+      </Helmet>
+    </Column>
+  );
+}
+
+Firehose.propTypes = {
+  multiColumn: PropTypes.bool,
+  feedType: PropTypes.string,
+};
+
+export default Firehose;

@@ 37,7 37,7 @@ const getHomeFeedSpeed = createSelector([
   state => state.get('statuses'),
 ], (statusIds, pendingStatusIds, statusMap) => {
   const recentStatusIds = pendingStatusIds.size > 0 ? pendingStatusIds : statusIds;
-  const statuses = recentStatusIds.map(id => statusMap.get(id)).filter(status => status?.get('account') !== me).take(20);
+  const statuses = recentStatusIds.filter(id => id !== null).map(id => statusMap.get(id)).filter(status => status?.get('account') !== me).take(20);
   const oldest = new Date(statuses.getIn([statuses.size - 1, 'created_at'], 0));
   const newest = new Date(statuses.getIn([0, 'created_at'], 0));
   const averageGap = (newest - oldest) / (1000 * (statuses.size + 1)); // Average gap between posts on first page in seconds

@@ 91,7 91,6 @@ class Header extends PureComponent {
 
       content = (
         <>
-          {location.pathname !== '/search' && <Link to='/search' className='button button-secondary' aria-label={intl.formatMessage(messages.search)}><Icon id='search' /></Link>}
           {signupButton}
           <a href='/auth/sign_in' className='button button-tertiary'><FormattedMessage id='sign_in_banner.sign_in' defaultMessage='Login' /></a>
         </>

@@ 20,8 20,7 @@ const messages = defineMessages({
   home: { id: 'tabs_bar.home', defaultMessage: 'Home' },
   notifications: { id: 'tabs_bar.notifications', defaultMessage: 'Notifications' },
   explore: { id: 'explore.title', defaultMessage: 'Explore' },
-  local: { id: 'tabs_bar.local_timeline', defaultMessage: 'Local' },
-  federated: { id: 'tabs_bar.federated_timeline', defaultMessage: 'Federated' },
+  firehose: { id: 'column.firehose', defaultMessage: 'Live feeds' },
   direct: { id: 'navigation_bar.direct', defaultMessage: 'Private mentions' },
   favourites: { id: 'navigation_bar.favourites', defaultMessage: 'Favourites' },
   bookmarks: { id: 'navigation_bar.bookmarks', defaultMessage: 'Bookmarks' },


@@ 43,6 42,10 @@ class NavigationPanel extends Component {
     intl: PropTypes.object.isRequired,
   };
 
+  isFirehoseActive = (match, location) => {
+    return match || location.pathname.startsWith('/public');
+  };
+
   render () {
     const { intl } = this.props;
     const { signedIn, disabledAccountId } = this.context.identity;


@@ 69,10 72,7 @@ class NavigationPanel extends Component {
         )}
 
         {(signedIn || timelinePreview) && (
-          <>
-            <ColumnLink transparent to='/public/local' icon='users' text={intl.formatMessage(messages.local)} />
-            <ColumnLink transparent exact to='/public' icon='globe' text={intl.formatMessage(messages.federated)} />
-          </>
+          <ColumnLink transparent to='/public/local' isActive={this.isFirehoseActive} icon='globe' text={intl.formatMessage(messages.firehose)} />
         )}
 
         {!signedIn && (

@@ 36,8 36,7 @@ import {
   Status,
   GettingStarted,
   KeyboardShortcuts,
-  PublicTimeline,
-  CommunityTimeline,
+  Firehose,
   AccountTimeline,
   AccountGallery,
   HomeTimeline,


@@ 188,8 187,11 @@ class SwitchingColumnsArea extends PureComponent {
           <WrappedRoute path='/privacy-policy' component={PrivacyPolicy} content={children} />
 
           <WrappedRoute path={['/home', '/timelines/home']} component={HomeTimeline} content={children} />
-          <WrappedRoute path={['/public', '/timelines/public']} exact component={PublicTimeline} content={children} />
-          <WrappedRoute path={['/public/local', '/timelines/public/local']} exact component={CommunityTimeline} content={children} />
+          <Redirect from='/timelines/public' to='/public' exact />
+          <Redirect from='/timelines/public/local' to='/public/local' exact />
+          <WrappedRoute path='/public' exact component={Firehose} componentParams={{ feedType: 'public' }} content={children} />
+          <WrappedRoute path='/public/local' exact component={Firehose} componentParams={{ feedType: 'community' }} content={children} />
+          <WrappedRoute path='/public/remote' exact component={Firehose} componentParams={{ feedType: 'public:remote' }} content={children} />
           <WrappedRoute path={['/conversations', '/timelines/direct']} component={DirectTimeline} content={children} />
           <WrappedRoute path='/tags/:id' component={HashtagTimeline} content={children} />
           <WrappedRoute path='/lists/:id' component={ListTimeline} content={children} />

@@ 22,6 22,10 @@ export function CommunityTimeline () {
   return import(/* webpackChunkName: "features/community_timeline" */'../../community_timeline');
 }
 
+export function Firehose () {
+  return import(/* webpackChunkName: "features/firehose" */'../../firehose');
+}
+
 export function HashtagTimeline () {
   return import(/* webpackChunkName: "features/hashtag_timeline" */'../../hashtag_timeline');
 }

@@ 114,6 114,7 @@
   "column.directory": "Browse profiles",
   "column.domain_blocks": "Blocked domains",
   "column.favourites": "Favourites",
+  "column.firehose": "Live feeds",
   "column.follow_requests": "Follow requests",
   "column.home": "Home",
   "column.lists": "Lists",


@@ 267,6 268,9 @@
   "filter_modal.select_filter.subtitle": "Use an existing category or create a new one",
   "filter_modal.select_filter.title": "Filter this post",
   "filter_modal.title.status": "Filter a post",
+  "firehose.all": "All",
+  "firehose.local": "Local",
+  "firehose.remote": "Remote",
   "follow_request.authorize": "Authorize",
   "follow_request.reject": "Reject",
   "follow_requests.unlocked_explanation": "Even though your account is not locked, the {domain} staff thought you might want to review follow requests from these accounts manually.",


@@ 649,9 653,7 @@
   "subscribed_languages.target": "Change subscribed languages for {target}",
   "suggestions.dismiss": "Dismiss suggestion",
   "suggestions.header": "You might be interested in…",
-  "tabs_bar.federated_timeline": "Federated",
   "tabs_bar.home": "Home",
-  "tabs_bar.local_timeline": "Local",
   "tabs_bar.notifications": "Notifications",
   "time_remaining.days": "{number, plural, one {# day} other {# days}} left",
   "time_remaining.hours": "{number, plural, one {# hour} other {# hours}} left",

@@ 1,3 1,5 @@
+import { Record as ImmutableRecord } from 'immutable';
+
 import { loadingBarReducer } from 'react-redux-loading-bar';
 import { combineReducers } from 'redux-immutable';
 


@@ 88,6 90,22 @@ const reducers = {
   followed_tags,
 };
 
-const rootReducer = combineReducers(reducers);
+// We want the root state to be an ImmutableRecord, which is an object with a defined list of keys,
+// so it is properly typed and keys can be accessed using `state.<key>` syntax.
+// This will allow an easy conversion to a plain object once we no longer call `get` or `getIn` on the root state
+
+// By default with `combineReducers` it is a Collection, so we provide our own implementation to get a Record
+const initialRootState = Object.fromEntries(
+  Object.entries(reducers).map(([name, reducer]) => [
+    name,
+    reducer(undefined, {
+      // empty action
+    }),
+  ])
+);
+
+const RootStateRecord = ImmutableRecord(initialRootState, 'RootState');
+
+const rootReducer = combineReducers(reducers, RootStateRecord);
 
 export { rootReducer };

@@ 17,15 17,15 @@ import {
 
 const initialState = ImmutableMap({
   server: ImmutableMap({
-    isLoading: true,
+    isLoading: false,
   }),
 
   extendedDescription: ImmutableMap({
-    isLoading: true,
+    isLoading: false,
   }),
 
   domainBlocks: ImmutableMap({
-    isLoading: true,
+    isLoading: false,
     isAvailable: true,
     items: ImmutableList(),
   }),

@@ 79,6 79,10 @@ const initialState = ImmutableMap({
     }),
   }),
 
+  firehose: ImmutableMap({
+    onlyMedia: false,
+  }),
+
   community: ImmutableMap({
     regex: ImmutableMap({
       body: '',

@@ 116,7 116,7 @@ class Account < ApplicationRecord
   scope :matches_username, ->(value) { where('lower((username)::text) LIKE lower(?)', "#{value}%") }
   scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) }
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
-  scope :without_unapproved, -> { left_outer_joins(:user).remote.or(left_outer_joins(:user).merge(User.approved.confirmed)) }
+  scope :without_unapproved, -> { left_outer_joins(:user).merge(User.approved.confirmed).or(remote) }
   scope :searchable, -> { without_unapproved.without_suspended.where(moved_to_account_id: nil) }
   scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
   scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) }

@@ 106,6 106,17 @@ module AccountSearch
     LIMIT :limit OFFSET :offset
   SQL
 
+  def searchable_text
+    PlainTextFormatter.new(note, local?).to_s if discoverable?
+  end
+
+  def searchable_properties
+    [].tap do |properties|
+      properties << 'bot' if bot?
+      properties << 'verified' if fields.any?(&:verified?)
+    end
+  end
+
   class_methods do
     def search_for(terms, limit: 10, offset: 0)
       tsquery = generate_query_for_search(terms)

@@ 9,12 9,11 @@ class AccountSearchService < BaseService
   MIN_QUERY_LENGTH = 5
 
   def call(query, account = nil, options = {})
-    @acct_hint = query&.start_with?('@')
-    @query     = query&.strip&.gsub(/\A@/, '')
-    @limit     = options[:limit].to_i
-    @offset    = options[:offset].to_i
-    @options   = options
-    @account   = account
+    @query   = query&.strip&.gsub(/\A@/, '')
+    @limit   = options[:limit].to_i
+    @offset  = options[:offset].to_i
+    @options = options
+    @account = account
 
     search_service_results.compact.uniq
   end


@@ 72,8 71,8 @@ class AccountSearchService < BaseService
   end
 
   def from_elasticsearch
-    must_clauses   = [{ multi_match: { query: terms_for_query, fields: likely_acct? ? %w(acct.edge_ngram acct) : %w(acct.edge_ngram acct display_name.edge_ngram display_name), type: 'most_fields', operator: 'and' } }]
-    should_clauses = []
+    must_clauses   = must_clause
+    should_clauses = should_clause
 
     if account
       return [] if options[:following] && following_ids.empty?


@@ 88,7 87,7 @@ class AccountSearchService < BaseService
     query     = { bool: { must: must_clauses, should: should_clauses } }
     functions = [reputation_score_function, followers_score_function, time_distance_function]
 
-    records = AccountsIndex.query(function_score: { query: query, functions: functions, boost_mode: 'multiply', score_mode: 'avg' })
+    records = AccountsIndex.query(function_score: { query: query, functions: functions })
                            .limit(limit_for_non_exact_results)
                            .offset(offset)
                            .objects


@@ 133,6 132,36 @@ class AccountSearchService < BaseService
     }
   end
 
+  def must_clause
+    fields = %w(username username.* display_name display_name.*)
+    fields << 'text' << 'text.*' if options[:use_searchable_text]
+
+    [
+      {
+        multi_match: {
+          query: terms_for_query,
+          fields: fields,
+          type: 'best_fields',
+          operator: 'or',
+        },
+      },
+    ]
+  end
+
+  def should_clause
+    [
+      {
+        multi_match: {
+          query: terms_for_query,
+          fields: %w(username username.* display_name display_name.*),
+          type: 'best_fields',
+          operator: 'and',
+          boost: 10,
+        },
+      },
+    ]
+  end
+
   def following_ids
     @following_ids ||= account.active_relationships.pluck(:target_account_id) + [account.id]
   end


@@ 182,8 211,4 @@ class AccountSearchService < BaseService
   def username_complete?
     query.include?('@') && "@#{query}".match?(MENTION_ONLY_RE)
   end
-
-  def likely_acct?
-    @acct_hint || username_complete?
-  end
 end

@@ 89,13 89,28 @@ class ResolveURLService < BaseService
   def process_local_url
     recognized_params = Rails.application.routes.recognize_path(@url)
 
-    return unless recognized_params[:action] == 'show'
+    case recognized_params[:controller]
+    when 'statuses'
+      return unless recognized_params[:action] == 'show'
 
-    if recognized_params[:controller] == 'statuses'
       status = Status.find_by(id: recognized_params[:id])
       check_local_status(status)
-    elsif recognized_params[:controller] == 'accounts'
+    when 'accounts'
+      return unless recognized_params[:action] == 'show'
+
       Account.find_local(recognized_params[:username])
+    when 'home'
+      return unless recognized_params[:action] == 'index' && recognized_params[:username_with_domain].present?
+
+      if recognized_params[:any]&.match?(/\A[0-9]+\Z/)
+        status = Status.find_by(id: recognized_params[:any])
+        check_local_status(status)
+      elsif recognized_params[:any].blank?
+        username, domain = recognized_params[:username_with_domain].gsub(/\A@/, '').split('@')
+        return unless username.present? && domain.present?
+
+        Account.find_remote(username, domain)
+      end
     end
   end
 

@@ 30,7 30,8 @@ class SearchService < BaseService
       @account,
       limit: @limit,
       resolve: @resolve,
-      offset: @offset
+      offset: @offset,
+      use_searchable_text: true
     )
   end
 

@@ 104,8 104,6 @@ Rails.application.routes.draw do
 
     resources :followers, only: [:index], controller: :follower_accounts
     resources :following, only: [:index], controller: :following_accounts
-    resource :follow, only: [:create], controller: :account_follow
-    resource :unfollow, only: [:create], controller: :account_unfollow
 
     resource :outbox, only: [:show], module: :activitypub
     resource :inbox, only: [:create], module: :activitypub


@@ 165,7 163,7 @@ Rails.application.routes.draw do
   get '/backups/:id/download', to: 'backups#download', as: :download_backup, format: false
 
   resource :authorize_interaction, only: [:show, :create]
-  resource :share, only: [:show, :create]
+  resource :share, only: [:show]
 
   draw(:admin)
 

@@ 3,7 3,7 @@
 namespace :admin do
   get '/dashboard', to: 'dashboard#index'
 
-  resources :domain_allows, only: [:new, :create, :show, :destroy]
+  resources :domain_allows, only: [:new, :create, :destroy]
   resources :domain_blocks, only: [:new, :create, :destroy, :update, :edit] do
     collection do
       post :batch


@@ 31,7 31,7 @@ namespace :admin do
   end
 
   resources :action_logs, only: [:index]
-  resources :warning_presets, except: [:new]
+  resources :warning_presets, except: [:new, :show]
 
   resources :announcements, except: [:show] do
     member do


@@ 76,7 76,7 @@ namespace :admin do
     end
   end
 
-  resources :rules
+  resources :rules, only: [:index, :create, :edit, :update, :destroy]
 
   resources :webhooks do
     member do

@@ 1,6 1,5 @@
+skip_untranslated_strings: 1
 commit_message: '[ci skip]'
-skip_untranslated_strings: true
-
 files:
   - source: /app/javascript/mastodon/locales/en.json
     translation: /app/javascript/mastodon/locales/%two_letters_code%.json

@@ 0,0 1,9 @@
+# frozen_string_literal: true
+
+class AddIndexBackupsOnUserId < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :backups, :user_id, algorithm: :concurrently
+  end
+end

@@ 10,7 10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2023_06_05_085711) do
+ActiveRecord::Schema.define(version: 2023_06_30_145300) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"


@@ 273,6 273,7 @@ ActiveRecord::Schema.define(version: 2023_06_05_085711) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.bigint "dump_file_size"
+    t.index ["user_id"], name: "index_backups_on_user_id"
   end
 
   create_table "blocks", force: :cascade do |t|

@@ 5,19 5,124 @@ require 'rails_helper'
 describe Api::V1::DirectoriesController do
   render_views
 
-  let(:user)    { Fabricate(:user) }
+  let(:user)    { Fabricate(:user, confirmed_at: nil) }
   let(:token)   { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:follows') }
-  let(:account) { Fabricate(:account) }
 
   before do
     allow(controller).to receive(:doorkeeper_token) { token }
   end
 
   describe 'GET #show' do
-    it 'returns http success' do
-      get :show
+    context 'with no params' do
+      before do
+        _local_unconfirmed_account = Fabricate(
+          :account,
+          domain: nil,
+          user: Fabricate(:user, confirmed_at: nil, approved: true),
+          username: 'local_unconfirmed'
+        )
 
-      expect(response).to have_http_status(200)
+        local_unapproved_account = Fabricate(
+          :account,
+          domain: nil,
+          user: Fabricate(:user, confirmed_at: 10.days.ago),
+          username: 'local_unapproved'
+        )
+        local_unapproved_account.user.update(approved: false)
+
+        _local_undiscoverable_account = Fabricate(
+          :account,
+          domain: nil,
+          user: Fabricate(:user, confirmed_at: 10.days.ago, approved: true),
+          discoverable: false,
+          username: 'local_undiscoverable'
+        )
+
+        excluded_from_timeline_account = Fabricate(
+          :account,
+          domain: 'host.example',
+          discoverable: true,
+          username: 'remote_excluded_from_timeline'
+        )
+        Fabricate(:block, account: user.account, target_account: excluded_from_timeline_account)
+
+        _domain_blocked_account = Fabricate(
+          :account,
+          domain: 'test.example',
+          discoverable: true,
+          username: 'remote_domain_blocked'
+        )
+        Fabricate(:account_domain_block, account: user.account, domain: 'test.example')
+      end
+
+      it 'returns only the local discoverable account' do
+        local_discoverable_account = Fabricate(
+          :account,
+          domain: nil,
+          user: Fabricate(:user, confirmed_at: 10.days.ago, approved: true),
+          discoverable: true,
+          username: 'local_discoverable'
+        )
+
+        eligible_remote_account = Fabricate(
+          :account,
+          domain: 'host.example',
+          discoverable: true,
+          username: 'eligible_remote'
+        )
+
+        get :show
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json.size).to eq(2)
+        expect(body_as_json.first[:id]).to include(eligible_remote_account.id.to_s)
+        expect(body_as_json.second[:id]).to include(local_discoverable_account.id.to_s)
+      end
+    end
+
+    context 'when asking for local accounts only' do
+      it 'returns only the local accounts' do
+        user = Fabricate(:user, confirmed_at: 10.days.ago, approved: true)
+        local_account = Fabricate(:account, domain: nil, user: user)
+        remote_account = Fabricate(:account, domain: 'host.example')
+
+        get :show, params: { local: '1' }
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json.size).to eq(1)
+        expect(body_as_json.first[:id]).to include(local_account.id.to_s)
+        expect(response.body).to_not include(remote_account.id.to_s)
+      end
+    end
+
+    context 'when ordered by active' do
+      it 'returns accounts in order of most recent status activity' do
+        status_old = Fabricate(:status)
+        travel_to 10.seconds.from_now
+        status_new = Fabricate(:status)
+
+        get :show, params: { order: 'active' }
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json.size).to eq(2)
+        expect(body_as_json.first[:id]).to include(status_new.account.id.to_s)
+        expect(body_as_json.second[:id]).to include(status_old.account.id.to_s)
+      end
+    end
+
+    context 'when ordered by new' do
+      it 'returns accounts in order of creation' do
+        account_old = Fabricate(:account)
+        travel_to 10.seconds.from_now
+        account_new = Fabricate(:account)
+
+        get :show, params: { order: 'new' }
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json.size).to eq(2)
+        expect(body_as_json.first[:id]).to include(account_new.id.to_s)
+        expect(body_as_json.second[:id]).to include(account_old.id.to_s)
+      end
     end
   end
 end

@@ 130,5 130,13 @@ RSpec.describe Api::V1::Emails::ConfirmationsController do
         end
       end
     end
+
+    context 'without an oauth token and an authentication cookie' do
+      it 'returns http unauthorized' do
+        get :check
+
+        expect(response).to have_http_status(401)
+      end
+    end
   end
 end

@@ 145,5 145,35 @@ describe ResolveURLService, type: :service do
         expect(subject.call(url, on_behalf_of: account)).to eq(status)
       end
     end
+
+    context 'when searching for a local link of a remote private status' do
+      let(:account)    { Fabricate(:account) }
+      let(:poster)     { Fabricate(:account, username: 'foo', domain: 'example.com') }
+      let(:url)        { 'https://example.com/@foo/42' }
+      let(:uri)        { 'https://example.com/users/foo/statuses/42' }
+      let!(:status)    { Fabricate(:status, url: url, uri: uri, account: poster, visibility: :private) }
+      let(:search_url) { "https://#{Rails.configuration.x.local_domain}/@foo@example.com/#{status.id}" }
+
+      before do
+        stub_request(:get, url).to_return(status: 404) if url.present?
+        stub_request(:get, uri).to_return(status: 404)
+      end
+
+      context 'when the account follows the poster' do
+        before do
+          account.follow!(poster)
+        end
+
+        it 'returns the status' do
+          expect(subject.call(search_url, on_behalf_of: account)).to eq(status)
+        end
+      end
+
+      context 'when the account does not follow the poster' do
+        it 'does not return the status' do
+          expect(subject.call(search_url, on_behalf_of: account)).to be_nil
+        end
+      end
+    end
   end
 end

@@ 68,7 68,7 @@ describe SearchService, type: :service do
           allow(AccountSearchService).to receive(:new).and_return(service)
 
           results = subject.call(query, nil, 10)
-          expect(service).to have_received(:call).with(query, nil, limit: 10, offset: 0, resolve: false)
+          expect(service).to have_received(:call).with(query, nil, limit: 10, offset: 0, resolve: false, use_searchable_text: true)
           expect(results).to eq empty_results.merge(accounts: [account])
         end
       end