<commit_before>import { resolve, relative, dirname } from 'path'
import { readFileSync, statSync } from 'fs'
import { EOL } from 'os'
import * as ts from 'typescript'
import extend = require('xtend')
import { parseQuery, urlToRequest } from 'loader-utils'

interface WebPackLoader {
  cacheable(flag?: boolean): void
  query: string
  resourcePath: string
  context: string
  sourceMap: boolean
  loaderIndex: number
  _compiler: any
  addDependency(fileName: string): void
  clearDependencies(): void
  emitWarning(warning: string): void
  emitError(error: string): void
  callback(err: Error): void
  callback(err: void, contents: string, sourceMap?: SourceMap): void
  options: {
    context: string
  }
}

interface SourceMap {
  sources: string[]
  file: string
  sourcesContent: string[]
}

type FilesMap = ts.Map<{ version: number, text: string }>

interface LoaderInstance {
  files: FilesMap
  service: ts.LanguageService
}

let currentLoader: WebPackLoader

const loaderInstances: { [key: number]: LoaderInstance } = {}

/**
 * Support TypeScript in Webpack.
 *
 * @param {string} content
 */
function loader (content: string): void {
  const loader: WebPackLoader = this
  const fileName = this.resourcePath
  const { files, service } = getLoaderInstance(this)
  let file = files[fileName]

  this.cacheable()

  // Only set content on the first load. The watch task maintains reloads and
  // the version doesn't need to change when every dependency is re-run.
  if (!file) {
    file = files[fileName] = { version: 0, text: '' }
  }

  file.text = content
  file.version++

  currentLoader = loader
  const output = service.getEmitOutput(fileName)
  currentLoader = undefined

  service.getSyntacticDiagnostics(fileName)
    .forEach((diagnostic) => {
      loader.emitError(formatDiagnostic(diagnostic))
    })

  if (output.emitSkipped) {
    loader.callback(new Error(`${fileName}: File not found`))
    return
  }

  const result = output.outputFiles[loader.sourceMap ? 1 : 0].text
  let sourceMap: SourceMap

  if (loader.sourceMap) {
    sourceMap = JSON.parse(output.outputFiles[0].text)
    sourceMap.sources = [fileName]
    sourceMap.file = fileName
    sourceMap.sourcesContent = [content]
  }

  loader.callback(null, result, sourceMap)
}

/**
 * Create a TypeScript language service from the first instance.
 *
 * @param {FilesMap}      files
 * @param {WebPackLoader} loader
 */
function createService (files: FilesMap, loader: WebPackLoader) {
  const context = loader.options.context
  const configFile = findConfigFile(dirname(loader.resourcePath))
  let defaultFiles: string[] = [loader.resourcePath]

  let compilerOptions: any = {
    target: 'es5',
    module: 'commonjs'
  }

  if (configFile) {
    const tsconfig = ts.readConfigFile(configFile)
    const configDir = dirname(configFile)

    const files = tsconfig.files
      .map((file: string) => resolve(configDir, file))
      .filter((file: string) => file !== loader.resourcePath)

    // Include `tsconfig.json` files in default files to load.
    defaultFiles = defaultFiles.concat(files)

    // Extend default compiler options with `tsconfig.json`.
    compilerOptions = extend(compilerOptions, tsconfig.compilerOptions)
  }

  // Extend compiler options with the webpack options.
  compilerOptions = extend(compilerOptions, parseQuery(loader.query))
  compilerOptions.sourceMap = loader.sourceMap

  const config = ts.parseConfigFile({
    files: defaultFiles,
    compilerOptions
  })

  if (config.errors.length) {
    config.errors.forEach((error) => loader.emitError(formatDiagnostic(error)))
  }

  const defaultLibFileName = ts.getDefaultLibFilePath(config.options)
<commit_msg>Check `tsconfig.files` is an array before use<commit_after>import { resolve, relative, dirname } from 'path'
import { readFileSync, statSync } from 'fs'
import { EOL } from 'os'
import * as ts from 'typescript'
import extend = require('xtend')
import { parseQuery, urlToRequest } from 'loader-utils'

interface WebPackLoader {
  cacheable(flag?: boolean): void
  query: string
  resourcePath: string
  context: string
  sourceMap: boolean
  loaderIndex: number
  _compiler: any
  addDependency(fileName: string): void
  clearDependencies(): void
  emitWarning(warning: string): void
  emitError(error: string): void
  callback(err: Error): void
  callback(err: void, contents: string, sourceMap?: SourceMap): void
  options: {
    context: string
  }
}

interface SourceMap {
  sources: string[]
  file: string
  sourcesContent: string[]
}

type FilesMap = ts.Map<{ version: number, text: string }>

interface LoaderInstance {
  files: FilesMap
  service: ts.LanguageService
}

let currentLoader: WebPackLoader

const loaderInstances: { [key: number]: LoaderInstance } = {}

/**
 * Support TypeScript in Webpack.
 *
 * @param {string} content
 */
function loader (content: string): void {
  const loader: WebPackLoader = this
  const fileName = this.resourcePath
  const { files, service } = getLoaderInstance(this)
  let file = files[fileName]

  this.cacheable()

  // Only set content on the first load. The watch task maintains reloads and
  // the version doesn't need to change when every dependency is re-run.
  if (!file) {
    file = files[fileName] = { version: 0, text: '' }
  }

  file.text = content
  file.version++

  currentLoader = loader
  const output = service.getEmitOutput(fileName)
  currentLoader = undefined

  service.getSyntacticDiagnostics(fileName)
    .forEach((diagnostic) => {
      loader.emitError(formatDiagnostic(diagnostic))
    })

  if (output.emitSkipped) {
    loader.callback(new Error(`${fileName}: File not found`))
    return
  }

  const result = output.outputFiles[loader.sourceMap ? 1 : 0].text
  let sourceMap: SourceMap

  if (loader.sourceMap) {
    sourceMap = JSON.parse(output.outputFiles[0].text)
    sourceMap.sources = [fileName]
    sourceMap.file = fileName
    sourceMap.sourcesContent = [content]
  }

  loader.callback(null, result, sourceMap)
}

/**
 * Create a TypeScript language service from the first instance.
 *
 * @param {FilesMap}      files
 * @param {WebPackLoader} loader
 */
function createService (files: FilesMap, loader: WebPackLoader) {
  const context = loader.options.context
  const configFile = findConfigFile(dirname(loader.resourcePath))
  let defaultFiles: string[] = [loader.resourcePath]

  let compilerOptions: any = {
    target: 'es5',
    module: 'commonjs'
  }

  if (configFile) {
    const tsconfig = ts.readConfigFile(configFile)
    const configDir = dirname(configFile)

    if (Array.isArray(tsconfig.files)) {
      const files = tsconfig.files
        .map((file: string) => resolve(configDir, file))
        .filter((file: string) => file !== loader.resourcePath)

      // Include `tsconfig.json` files in default files to load.
      defaultFiles = defaultFiles.concat(files)
    }

    // Extend default compiler options with `tsconfig.json`.
    compilerOptions = extend(compilerOptions, tsconfig.compilerOptions)
  }

  // Extend compiler options with the webpack options.
  compilerOptions = extend(compilerOptions, parseQuery(loader.query))
  compilerOptions.sourceMap = loader.sourceMap

  const config = ts.parseConfigFile({
    files: defaultFiles,
    compilerOptions
  })

  if (config.errors.length) {
    config.errors.forEach((error) => loader.emitError(formatDiagnostic(error)))
  }

  const defaultLibFileName = ts.getDefaultLibFilePath(config.options)

{'fraction_non_alphanumeric': 0.1090543259557344, 'fraction_numerical': 0.0013413816230717639, 'mean_word_length': 4.017496635262449, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3374800', 'n_tokens_mistral': 2195, 'n_tokens_neox': 2171, 'n_words': 606}