<commit_before>export class ParseLocation {
  constructor(
      public file: ParseSourceFile, public offset: number, public line: number,
      public col: number) {}

  toString(): string {
    return this.offset ? `${this.file.url}@${this.line}:${this.col}` : this.file.url;
  }
}

export class ParseSourceFile {
  constructor(public content: string, public url: string) {}
}

export class ParseSourceSpan {
  constructor(
      public start: ParseLocation, public end: ParseLocation, public details: string = null) {}

  toString(): string {
    return this.start.file.content.substring(this.start.offset, this.end.offset);
  }
}

export enum ParseErrorLevel {
  WARNING,
  FATAL
}

export abstract class ParseError {
  constructor(
      public span: ParseSourceSpan, public msg: string,
      public level: ParseErrorLevel = ParseErrorLevel.FATAL) {}

  toString(): string {
    const source = this.span.start.file.content;
    let ctxStart = this.span.start.offset;
    let contextStr = '';
    let details = '';
    if (ctxStart) {
      if (ctxStart > source.length - 1) {
        ctxStart = source.length - 1;
      }
      const ctxEnd = ctxStart;
      let ctxLen = 0;
      let ctxLines = 0;

      while (ctxLen < 100 && ctxStart > 0) {
        ctxStart--;
        ctxLen++;
        if (source[ctxStart] === '\n') {
          if (++ctxLines === 3) {
            break;
          }
        }
      }

      ctxLen = 0;
      ctxLines = 0;
      while (ctxLen < 100 && ctxEnd < source.length - 1) {
        ctxEnd++;
        ctxLen++;
        if (source[ctxEnd] === '\n') {
          if (++ctxLines === 3) {
            break;
          }<commit_msg>Fix test with wrong var/const change<commit_after>export class ParseLocation {
  constructor(
      public file: ParseSourceFile, public offset: number, public line: number,
      public col: number) {}

  toString(): string {
    return this.offset ? `${this.file.url}@${this.line}:${this.col}` : this.file.url;
  }
}

export class ParseSourceFile {
  constructor(public content: string, public url: string) {}
}

export class ParseSourceSpan {
  constructor(
      public start: ParseLocation, public end: ParseLocation, public details: string = null) {}

  toString(): string {
    return this.start.file.content.substring(this.start.offset, this.end.offset);
  }
}

export enum ParseErrorLevel {
  WARNING,
  FATAL
}

export abstract class ParseError {
  constructor(
      public span: ParseSourceSpan, public msg: string,
      public level: ParseErrorLevel = ParseErrorLevel.FATAL) {}

  toString(): string {
    const source = this.span.start.file.content;
    let ctxStart = this.span.start.offset;
    let contextStr = '';
    let details = '';
    if (ctxStart) {
      if (ctxStart > source.length - 1) {
        ctxStart = source.length - 1;
      }
      let ctxEnd = ctxStart;
      let ctxLen = 0;
      let ctxLines = 0;

      while (ctxLen < 100 && ctxStart > 0) {
        ctxStart--;
        ctxLen++;
        if (source[ctxStart] === '\n') {
          if (++ctxLines === 3) {
            break;
          }
        }
      }

      ctxLen = 0;
      ctxLines = 0;
      while (ctxLen < 100 && ctxEnd < source.length - 1) {
        ctxEnd++;
        ctxLen++;
        if (source[ctxEnd] === '\n') {
          if (++ctxLines === 3) {
            break;
          }
{'fraction_non_alphanumeric': 0.12944693572496263, 'fraction_numerical': 0.009566517189835576, 'mean_word_length': 2.3030602171767027, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6415490', 'n_tokens_mistral': 1036, 'n_tokens_neox': 994, 'n_words': 271}