<commit_before>import * as express from 'express'
import { sanitizeUrl } from '@server/helpers/core-utils'
import { doRequest } from '@server/helpers/requests'
import { CONFIG } from '@server/initializers/config'
import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos'
import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
import { getServerActor } from '@server/models/application/application'
import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
import { ResultList, Video, VideoChannel } from '@shared/models'
import { SearchTargetQuery } from '@shared/models/search/search-target-query.model'
import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils'
import { logger } from '../../helpers/logger'
import { getFormattedObjects } from '../../helpers/utils'
import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor'
import {
  asyncMiddleware,
  commonVideosFiltersValidator,
  optionalAuthenticate,
  paginationValidator,
  setDefaultPagination,
  setDefaultSearchSort,
  videoChannelsSearchSortValidator,
  videoChannelsSearchValidator,
  videosSearchSortValidator,
  videosSearchValidator
} from '../../middlewares'
import { VideoModel } from '../../models/video/video'
import { VideoChannelModel } from '../../models/video/video-channel'
import { MChannelAccountDefault, MVideoAccountLightBlacklistAllFiles } from '../../typings/models'

const searchRouter = express.Router()

searchRouter.get('/videos',
  paginationValidator,
  setDefaultPagination,
  videosSearchSortValidator,
  setDefaultSearchSort,
  optionalAuthenticate,
  commonVideosFiltersValidator,
  videosSearchValidator,
  asyncMiddleware(searchVideos)
)

searchRouter.get('/video-channels',
  paginationValidator,
  setDefaultPagination,
  videoChannelsSearchSortValidator,
  setDefaultSearchSort,
  optionalAuthenticate,
  videoChannelsSearchValidator,
  asyncMiddleware(searchVideoChannels)
)

// ---------------------------------------------------------------------------

export { searchRouter }

// ---------------------------------------------------------------------------

function searchVideoChannels (req: express.Request, res: express.Response) {
  const query: VideoChannelsSearchQuery = req.query
  const search = query.search

  const isURISearch = search.startsWith('http://') || search.startsWith('https://')

  const parts = search.split('@')

  // Handle strings like <EMAIL>
  if (parts.length === 3 && parts[0].length === 0) parts.shift()
  const isWebfingerSearch = parts.length === 2 && parts.every(p => p && !p.includes(' '))

  if (isURISearch || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res)

  // @username -> username to search in DB
  if (query.search.startsWith('@')) query.search = query.search.replace(/^@/, '')

  if (isSearchIndexSearch(query)) {
    return searchVideoChannelsIndex(query, res)
  }

  return searchVideoChannelsDB(query, res)
}

async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: express.Response) {
  logger.debug('Doing channels search on search index.')

  const result = await buildMutedForSearchIndex(res)

  const body = Object.assign(query, result)

  const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-channels'

  try {
    const searchIndexResult = await doRequest<ResultList<VideoChannel>>({ uri: url, body, json: true })

    return res.json(searchIndexResult.body)
  } catch (err) {
    logger.warn('Cannot use search index to make video channels search.', { err })

    return res.sendStatus(500)
  }
}

async function searchVideoChannelsDB (query: VideoChannelsSearchQuery, res: express.Response) {
  const serverActor = await getServerActor()

  const options = {
    actorId: serverActor.id,
    search: query.search,
    start: query.start,
    count: query.count,
    sort: query.sort
  }
  const resultList = await VideoChannelModel.searchForApi(options)

  return res.json(getFormattedObjects(resultList.data, resultList.total))
}

async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean, res: express.Response) {
  let videoChannel: MChannelAccountDefault
  let uri = search

  if (isWebfingerSearch) {
    try {
      uri = await loadActorUrlOrGetFromWebfinger(search)
    } catch (err) {
      logger.warn('Cannot load actor URL or get from webfinger.', { search, err })

      return res.json({ total: 0, data: [] })
    }
  }

  if (isUserAbleToSearchRemoteURI(res)) {
    try {
      const actor = await getOrCreateActorAndServerAndModel(uri, 'all', true, true)
      videoChannel = actor.VideoChannel
    } catch (err) {
      logger.info('Cannot search remote video channel %s.', uri, { err })
    }
  } else {
    videoChannel = await VideoChannelModel.loadByUrlAndPopulateAccount(uri)
  }

  return res.json({
    total: videoChannel ? 1 : 0,
    data: videoChannel ? [ videoChannel.toFormattedJSON() ] : []
  })
}

function searchVideos (req: express.Request, res: express.Response) {
  const query: VideosSearchQuery = req.query
  const search = query.search

  if (search && (search.startsWith('http://') || search.startsWith('https://'))) {
    return searchVideoURI(search, res)
  }

  if (isSearchIndexSearch(query)) {
    return searchVideosIndex(query, res)
  }

  return searchVideosDB(query, res)
}

async function searchVideosIndex (query: VideosSearchQuery, res: express.Response) {
  logger.debug('Doing videos search on search index.')

  const result = await buildMutedForSearchIndex(res)

  const body: VideosSearchQuery = Object.assign(query, result)

  // Use the default instance NSFW policy if not specified
  if (!body.nsfw) {
    body.nsfw = CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list'<commit_msg>Fix global search nsfw policy when logged in<commit_after>import * as express from 'express'
import { sanitizeUrl } from '@server/helpers/core-utils'
import { doRequest } from '@server/helpers/requests'
import { CONFIG } from '@server/initializers/config'
import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos'
import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
import { getServerActor } from '@server/models/application/application'
import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
import { ResultList, Video, VideoChannel } from '@shared/models'
import { SearchTargetQuery } from '@shared/models/search/search-target-query.model'
import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils'
import { logger } from '../../helpers/logger'
import { getFormattedObjects } from '../../helpers/utils'
import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor'
import {
  asyncMiddleware,
  commonVideosFiltersValidator,
  optionalAuthenticate,
  paginationValidator,
  setDefaultPagination,
  setDefaultSearchSort,
  videoChannelsSearchSortValidator,
  videoChannelsSearchValidator,
  videosSearchSortValidator,
  videosSearchValidator
} from '../../middlewares'
import { VideoModel } from '../../models/video/video'
import { VideoChannelModel } from '../../models/video/video-channel'
import { MChannelAccountDefault, MVideoAccountLightBlacklistAllFiles } from '../../typings/models'

const searchRouter = express.Router()

searchRouter.get('/videos',
  paginationValidator,
  setDefaultPagination,
  videosSearchSortValidator,
  setDefaultSearchSort,
  optionalAuthenticate,
  commonVideosFiltersValidator,
  videosSearchValidator,
  asyncMiddleware(searchVideos)
)

searchRouter.get('/video-channels',
  paginationValidator,
  setDefaultPagination,
  videoChannelsSearchSortValidator,
  setDefaultSearchSort,
  optionalAuthenticate,
  videoChannelsSearchValidator,
  asyncMiddleware(searchVideoChannels)
)

// ---------------------------------------------------------------------------

export { searchRouter }

// ---------------------------------------------------------------------------

function searchVideoChannels (req: express.Request, res: express.Response) {
  const query: VideoChannelsSearchQuery = req.query
  const search = query.search

  const isURISearch = search.startsWith('http://') || search.startsWith('https://')

  const parts = search.split('@')

  // Handle strings like <EMAIL>
  if (parts.length === 3 && parts[0].length === 0) parts.shift()
  const isWebfingerSearch = parts.length === 2 && parts.every(p => p && !p.includes(' '))

  if (isURISearch || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res)

  // @username -> username to search in DB
  if (query.search.startsWith('@')) query.search = query.search.replace(/^@/, '')

  if (isSearchIndexSearch(query)) {
    return searchVideoChannelsIndex(query, res)
  }

  return searchVideoChannelsDB(query, res)
}

async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: express.Response) {
  logger.debug('Doing channels search on search index.')

  const result = await buildMutedForSearchIndex(res)

  const body = Object.assign(query, result)

  const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-channels'

  try {
    const searchIndexResult = await doRequest<ResultList<VideoChannel>>({ uri: url, body, json: true })

    return res.json(searchIndexResult.body)
  } catch (err) {
    logger.warn('Cannot use search index to make video channels search.', { err })

    return res.sendStatus(500)
  }
}

async function searchVideoChannelsDB (query: VideoChannelsSearchQuery, res: express.Response) {
  const serverActor = await getServerActor()

  const options = {
    actorId: serverActor.id,
    search: query.search,
    start: query.start,
    count: query.count,
    sort: query.sort
  }
  const resultList = await VideoChannelModel.searchForApi(options)

  return res.json(getFormattedObjects(resultList.data, resultList.total))
}

async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean, res: express.Response) {
  let videoChannel: MChannelAccountDefault
  let uri = search

  if (isWebfingerSearch) {
    try {
      uri = await loadActorUrlOrGetFromWebfinger(search)
    } catch (err) {
      logger.warn('Cannot load actor URL or get from webfinger.', { search, err })

      return res.json({ total: 0, data: [] })
    }
  }

  if (isUserAbleToSearchRemoteURI(res)) {
    try {
      const actor = await getOrCreateActorAndServerAndModel(uri, 'all', true, true)
      videoChannel = actor.VideoChannel
    } catch (err) {
      logger.info('Cannot search remote video channel %s.', uri, { err })
    }
  } else {
    videoChannel = await VideoChannelModel.loadByUrlAndPopulateAccount(uri)
  }

  return res.json({
    total: videoChannel ? 1 : 0,
    data: videoChannel ? [ videoChannel.toFormattedJSON() ] : []
  })
}

function searchVideos (req: express.Request, res: express.Response) {
  const query: VideosSearchQuery = req.query
  const search = query.search

  if (search && (search.startsWith('http://') || search.startsWith('https://'))) {
    return searchVideoURI(search, res)
  }

  if (isSearchIndexSearch(query)) {
    return searchVideosIndex(query, res)
  }

  return searchVideosDB(query, res)
}

async function searchVideosIndex (query: VideosSearchQuery, res: express.Response) {
  logger.debug('Doing videos search on search index.')

  const result = await buildMutedForSearchIndex(res)

  const body: VideosSearchQuery = Object.assign(query, result)

  // Use the default instance NSFW policy if not specified
  if (!body.nsfw) {
    const nsfwPolicy = res.locals.oauth
      ? res.locals.oauth.token.User.nsfwPolicy
      : CONFIG.INSTANCE.DEFAULT_NSFW_POLICY

    body.nsfw = nsfwPolicy === 'do_not_list'
{'fraction_non_alphanumeric': 0.13892551087673038, 'fraction_numerical': 0.0018127883981542518, 'mean_word_length': 3.9905427631578947, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3930246', 'n_tokens_mistral': 3517, 'n_tokens_neox': 3483, 'n_words': 805}