<commit_before>import * as R from 'ramda'
import * as React from 'react'

import { HTTPError } from 'utils/APIConnector'
import * as AWS from 'utils/AWS'
import * as Config from 'utils/Config'
import * as Data from 'utils/Data'
import mkSearch from 'utils/mkSearch'
import type { S3HandleBase } from 'utils/s3paths'

import { CONTEXT, PreviewData } from '../types'
import * as summarize from './summarize'

import * as utils from './utils'

const isCsv = utils.extIs('.csv')

const isExcel = utils.extIn(['.xls', '.xlsx'])

const isJsonl = utils.extIs('.jsonl')

const isParquet = R.anyPass([
  utils.extIn(['.parquet', '.pq']),
  R.test(/.+_0$/),
  R.test(/[.-]c\d{3,5}$/gi),
])

const isTsv = utils.extIs('.tsv')

const detectBySummarizeType = summarize.detect('perspective')

const detectByExtension: (key: string) => boolean = R.pipe(
  utils.stripCompression,
  R.anyPass([isCsv, isExcel, isJsonl, isParquet, isTsv]),
)

export function detect(key: string, options: summarize.File): boolean | summarize.Type {
  return detectBySummarizeType(options) || detectByExtension(key)<commit_msg>Support .tab as .tsv (#2895)<commit_after>import * as R from 'ramda'
import * as React from 'react'

import { HTTPError } from 'utils/APIConnector'
import * as AWS from 'utils/AWS'
import * as Config from 'utils/Config'
import * as Data from 'utils/Data'
import mkSearch from 'utils/mkSearch'
import type { S3HandleBase } from 'utils/s3paths'

import { CONTEXT, PreviewData } from '../types'
import * as summarize from './summarize'

import * as utils from './utils'

const isCsv = utils.extIs('.csv')

const isExcel = utils.extIn(['.xls', '.xlsx'])

const isJsonl = utils.extIs('.jsonl')

const isParquet = R.anyPass([
  utils.extIn(['.parquet', '.pq']),
  R.test(/.+_0$/),
  R.test(/[.-]c\d{3,5}$/gi),
])

const isTsv = utils.extIn(['.tsv', '.tab'])

const detectBySummarizeType = summarize.detect('perspective')

const detectByExtension: (key: string) => boolean = R.pipe(
  utils.stripCompression,
  R.anyPass([isCsv, isExcel, isJsonl, isParquet, isTsv]),
)

export function detect(key: string, options: summarize.File): boolean | summarize.Type {
  return detectBySummarizeType(options) || detectByExtension(key)
{'fraction_non_alphanumeric': 0.16161158895427796, 'fraction_numerical': 0.006337709370755998, 'mean_word_length': 4.710594315245478, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7583450', 'n_tokens_mistral': 786, 'n_tokens_neox': 780, 'n_words': 202}