<commit_before>import { resolve, relative } from 'path'
import { readFileSync, statSync } from 'fs'
import { EOL } from 'os'
import * as ts from 'typescript'
import extend = require('xtend')
import { parseQuery, urlToRequest } from 'loader-utils'
import { getProjectSync } from 'tsconfig'

interface WebPackLoader {
  cacheable(flag?: boolean): void
  query: string
  resourcePath: string
  context: string
  sourceMap: boolean
  loaderIndex: number
  _compiler: any
  addDependency(fileName: string): void
  clearDependencies(): void
  emitWarning(warning: string): void
  emitError(error: string): void
  callback(err: Error): void
  callback(err: void, contents: string, sourceMap?: SourceMap): void
  options: {
    context: string
  }
}

interface SourceMap {
  sources: string[]
  file: string
  sourcesContent: string[]
}

type FilesMap = ts.Map<{ version: number, text: string }>

interface LoaderInstance {
  files: FilesMap
  service: ts.LanguageService
}

var currentLoader: WebPackLoader

const loaderInstances: { [key: number]: LoaderInstance } = {}

/**
 * Support TypeScript in Webpack.
 *
 * @param {string} content
 */
function loader (content: string): void {
  let loader: WebPackLoader = this
  let fileName = this.resourcePath
  let { files, service } = getLoaderInstance(this)
  let file = files[fileName]

  this.cacheable()

  // Only set content on the first load. The watch task maintains reloads and
  // the version doesn't need to change when every dependency is re-run.
  if (!file) {
    file = files[fileName] = { version: 0, text: '' }
  }

  file.text = content
  file.version++

  currentLoader = loader
  let output = service.getEmitOutput(fileName)
  currentLoader = undefined

  service.getSyntacticDiagnostics(fileName)
    .forEach((diagnostic) => {
      loader.emitError(formatDiagnostic(diagnostic))
    })

  if (output.emitSkipped) {
    loader.callback(new Error(`${fileName}: File not found`))
    return
  }

  let result = output.outputFiles[loader.sourceMap ? 1 : 0].text
  let sourceMap: SourceMap

  if (loader.sourceMap) {
    sourceMap = JSON.parse(output.outputFiles[0].text)
    sourceMap.sources = [fileName]
    sourceMap.file = fileName
    sourceMap.sourcesContent = [content]
  }

  loader.callback(null, result, sourceMap)
}

/**
 * Create a TypeScript language service from the first instance.
 *
 * @param {FilesMap}      files
 * @param {WebPackLoader} loader
 */
function createService (files: FilesMap, loader: WebPackLoader) {
  let context = loader.options.context
  let tsconfig = getProjectSync(context)
  let defaultFiles: string[] = [loader.resourcePath]

  let compilerOptions: any = {
    target: 'es5',
    module: 'commonjs'
  }

  if (tsconfig) {
    let files = tsconfig.project.files
      .map((file) => {
        return resolve(tsconfig.projectFileDirectory, file)
      })
      .filter((file) => {
        return file !== loader.resourcePath
      })

    // Include `tsconfig.json` files in default files to load.
    defaultFiles = defaultFiles.concat(files)

    // Extend default compiler options with `tsconfig.json`.
    compilerOptions = extend(compilerOptions, tsconfig.project.compilerOptions)
  }

  // Extend compiler options with the webpack options.
  compilerOptions = extend(compilerOptions, parseQuery(loader.query))
  compilerOptions.sourceMap = loader.sourceMap

  let config = ts.parseConfigFile({
    files: defaultFiles,
    compilerOptions
  })

  if (config.errors.length) {
    config.errors.forEach((error) => loader.emitError(formatDiagnostic(error)))
    return
  }

  let defaultLibFileName = ts.getDefaultLibFilePath(config.options)

  // Add the default library to default files.
  config.fileNames.push(defaultLibFileName)

  let serviceHost: ts.LanguageServiceHost = {
    getScriptFileNames (): string[] {
      return config.fileNames
    },
    getScriptVersion (fileName) {
      return files[fileName] && files[fileName].version.toString()
    },
    getScriptSnapshot (fileName: string): ts.IScriptSnapshot {
      let file = files[fileName]
      let exists = fileExists(fileName)
<commit_msg>always return all compiled file names<commit_after>import { resolve, relative } from 'path'
import { readFileSync, statSync } from 'fs'
import { EOL } from 'os'
import * as ts from 'typescript'
import extend = require('xtend')
import { parseQuery, urlToRequest } from 'loader-utils'
import { getProjectSync } from 'tsconfig'

interface WebPackLoader {
  cacheable(flag?: boolean): void
  query: string
  resourcePath: string
  context: string
  sourceMap: boolean
  loaderIndex: number
  _compiler: any
  addDependency(fileName: string): void
  clearDependencies(): void
  emitWarning(warning: string): void
  emitError(error: string): void
  callback(err: Error): void
  callback(err: void, contents: string, sourceMap?: SourceMap): void
  options: {
    context: string
  }
}

interface SourceMap {
  sources: string[]
  file: string
  sourcesContent: string[]
}

type FilesMap = ts.Map<{ version: number, text: string }>

interface LoaderInstance {
  files: FilesMap
  service: ts.LanguageService
}

var currentLoader: WebPackLoader

const loaderInstances: { [key: number]: LoaderInstance } = {}

/**
 * Support TypeScript in Webpack.
 *
 * @param {string} content
 */
function loader (content: string): void {
  let loader: WebPackLoader = this
  let fileName = this.resourcePath
  let { files, service } = getLoaderInstance(this)
  let file = files[fileName]

  this.cacheable()

  // Only set content on the first load. The watch task maintains reloads and
  // the version doesn't need to change when every dependency is re-run.
  if (!file) {
    file = files[fileName] = { version: 0, text: '' }
  }

  file.text = content
  file.version++

  currentLoader = loader
  let output = service.getEmitOutput(fileName)
  currentLoader = undefined

  service.getSyntacticDiagnostics(fileName)
    .forEach((diagnostic) => {
      loader.emitError(formatDiagnostic(diagnostic))
    })

  if (output.emitSkipped) {
    loader.callback(new Error(`${fileName}: File not found`))
    return
  }

  let result = output.outputFiles[loader.sourceMap ? 1 : 0].text
  let sourceMap: SourceMap

  if (loader.sourceMap) {
    sourceMap = JSON.parse(output.outputFiles[0].text)
    sourceMap.sources = [fileName]
    sourceMap.file = fileName
    sourceMap.sourcesContent = [content]
  }

  loader.callback(null, result, sourceMap)
}

/**
 * Create a TypeScript language service from the first instance.
 *
 * @param {FilesMap}      files
 * @param {WebPackLoader} loader
 */
function createService (files: FilesMap, loader: WebPackLoader) {
  let context = loader.options.context
  let tsconfig = getProjectSync(context)
  let defaultFiles: string[] = [loader.resourcePath]

  let compilerOptions: any = {
    target: 'es5',
    module: 'commonjs'
  }

  if (tsconfig) {
    let files = tsconfig.project.files
      .map((file) => {
        return resolve(tsconfig.projectFileDirectory, file)
      })
      .filter((file) => {
        return file !== loader.resourcePath
      })

    // Include `tsconfig.json` files in default files to load.
    defaultFiles = defaultFiles.concat(files)

    // Extend default compiler options with `tsconfig.json`.
    compilerOptions = extend(compilerOptions, tsconfig.project.compilerOptions)
  }

  // Extend compiler options with the webpack options.
  compilerOptions = extend(compilerOptions, parseQuery(loader.query))
  compilerOptions.sourceMap = loader.sourceMap

  let config = ts.parseConfigFile({
    files: defaultFiles,
    compilerOptions
  })

  if (config.errors.length) {
    config.errors.forEach((error) => loader.emitError(formatDiagnostic(error)))
    return
  }

  let defaultLibFileName = ts.getDefaultLibFilePath(config.options)

  // Add the default library to default files.
  config.fileNames.push(defaultLibFileName)

  let serviceHost: ts.LanguageServiceHost = {
    getScriptFileNames (): string[] {
      // Return an array of all file names. We can't return just the default
      // files because webpack may have traversed through a regular JS file
      // back to a TypeScript file and if we don't have that file in the array,
      // TypeScript will give us a file not found compilation error.
      return defaultFiles.concat(Object.keys(files))
    },
    getScriptVersion (fileName) {
      return files[fileName] && files[fileName].version.toString()
    },
    getScriptSnapshot (fileName: string): ts.IScriptSnapshot {
      let file = files[fileName]
      let exists = fileExists(fileName)

{'fraction_non_alphanumeric': 0.10604824612516024, 'fraction_numerical': 0.0011653653420347278, 'mean_word_length': 3.7466814159292037, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3494880', 'n_tokens_mistral': 2489, 'n_tokens_neox': 2472, 'n_words': 708}