<commit_before>import * as fs from 'fs';
import { CompilerHost, CompilerOptions, createCompilerHost, createSourceFile, getDefaultLibFileName, ScriptTarget, SourceFile, sys, WriteFileCallback} from 'typescript';
const path = require('canonical-path');

// We need to provide our own version of CompilerHost because we want to set the
// base directory and specify what extensions to consider when trying to load a source
// file
export class CustomCompilerHost implements CompilerHost {

  constructor(private options: CompilerOptions, private baseDir: string, private extensions: string[], private log: any) {}

  getSourceFile(fileName: string, languageVersion: ScriptTarget, onError?: (message: string) => void): SourceFile {
    let text;
    let baseFilePath;
    let resolvedPath;
    let resolvedPathWithExt;

    // let's just try loading the file as-is initially
    try {
      resolvedPath = path.resolve(this.baseDir, fileName);
      text = fs.readFileSync(resolvedPath, { encoding: this.options.charset! });
      this.log.debug('found source file:', fileName);
      return createSourceFile(path.relative(this.baseDir, resolvedPath), text, languageVersion);
    } catch (e) {
      // if it is a folder then try loading the index file of that folder
      if (e.code === 'EISDIR') {
        return this.getSourceFile(fileName + '/index.ts', languageVersion, onError);
      }
      // otherwise ignore the error and move on to the strategy below...
    }

    // Special case for `/// <reference types="x">` directives.
    // Typescript does not fully resolve these before calling getSourceFile
    // so fileName ends up looking like `node_modules/@types/x/index.d.ts`.
    // In the case that node_modules is below the baseDir, we look
    // for ../node_modules/@types/x/index.d.ts, ../..node_modules/@types/x/index.d.ts, etc...
    if (fileName.startsWith(path.join('node_modules','@types')) && fileName.endsWith('index.d.ts')) {
      const baseDirDepth = this.baseDir.split(path.sep).length;
      let maybe = path.join('..', fileName);
      for (let i=0; i<baseDirDepth; ++i) {
        try {
          resolvedPath = path.resolve(this.baseDir, maybe);
          text = fs.readFileSync(resolvedPath, { encoding: this.options.charset! });
          this.log.debug('found source file:', fileName);
          return createSourceFile(path.relative(this.baseDir, resolvedPath), text, languageVersion);
        } catch (e) {
          // ignore the error and move on to the next maybe...
        }
        maybe = path.join('..', maybe);
      }
    }

    // Strip off the extension and resolve relative to the baseDir<commit_msg>fix(typescript): invalid path delimiter for triple slash type directives in windows<commit_after>import * as fs from 'fs';
import { CompilerHost, CompilerOptions, createCompilerHost, createSourceFile, getDefaultLibFileName, ScriptTarget, SourceFile, sys, WriteFileCallback} from 'typescript';
const path = require('canonical-path');

// We need to provide our own version of CompilerHost because we want to set the
// base directory and specify what extensions to consider when trying to load a source
// file
export class CustomCompilerHost implements CompilerHost {

  constructor(private options: CompilerOptions, private baseDir: string, private extensions: string[], private log: any) {}

  getSourceFile(fileName: string, languageVersion: ScriptTarget, onError?: (message: string) => void): SourceFile {
    let text;
    let baseFilePath;
    let resolvedPath;
    let resolvedPathWithExt;

    // let's just try loading the file as-is initially
    try {
      resolvedPath = path.resolve(this.baseDir, fileName);
      text = fs.readFileSync(resolvedPath, { encoding: this.options.charset! });
      this.log.debug('found source file:', fileName);
      return createSourceFile(path.relative(this.baseDir, resolvedPath), text, languageVersion);
    } catch (e) {
      // if it is a folder then try loading the index file of that folder
      if (e.code === 'EISDIR') {
        return this.getSourceFile(fileName + '/index.ts', languageVersion, onError);
      }
      // otherwise ignore the error and move on to the strategy below...
    }

    // Special case for `/// <reference types="x">` directives.
    // Typescript does not fully resolve these before calling getSourceFile
    // so fileName ends up looking like `node_modules/@types/x/index.d.ts`.
    // In the case that node_modules is below the baseDir, we look
    // for ../node_modules/@types/x/index.d.ts, ../..node_modules/@types/x/index.d.ts, etc...
    if (fileName.startsWith(path.join('node_modules', '@types')) && fileName.endsWith('index.d.ts')) {
      // The base directory path is already posix normalized through TypeScript. In order to
      // properly determine the depth of the base dir, the delimiter is always a forward slash.
      const baseDirDepth = this.baseDir.split('/').length;
      let maybe = path.join('..', fileName);
      for (let i = 0; i < baseDirDepth; ++i) {
        try {
          resolvedPath = path.resolve(this.baseDir, maybe);
          text = fs.readFileSync(resolvedPath, { encoding: this.options.charset! });
          this.log.debug('found source file:', fileName);
          return createSourceFile(path.relative(this.baseDir, resolvedPath), text, languageVersion);
        } catch (e) {
          // ignore the error and move on to the next maybe...
        }
        maybe = path.join('..', maybe);
      }
    }

    // Strip off the extension and resolve relative to the baseDir
{'fraction_non_alphanumeric': 0.11453268503511616, 'fraction_numerical': 0.0003601656762110571, 'mean_word_length': 3.2332317073170733, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4736298', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1500, 'n_words': 516}