<commit_before>import {Injectable} from 'angular2/src/core/di/decorators';
import {ListWrapper, SetWrapper} from "angular2/src/core/facade/collection";
import {
  NumberWrapper,
  StringJoiner,
  StringWrapper,
  BaseException,
  isPresent
} from "angular2/src/core/facade/lang";

export enum TokenType {
  Character,
  Identifier,
  Keyword,
  String,
  Operator,
  Number
}

@Injectable()
export class Lexer {
  tokenize(text: string): any[] {
    var scanner = new _Scanner(text);
    var tokens = [];
    var token = scanner.scanToken();
    while (token != null) {
      tokens.push(token);
      token = scanner.scanToken();
    }
    return tokens;
  }
}

export class Token {
  constructor(public index: number, public type: TokenType, public numValue: number,
              public strValue: string) {}

  isCharacter(code: number): boolean {
    return (this.type == TokenType.Character && this.numValue == code);
  }

  isNumber(): boolean { return (this.type == TokenType.Number); }

  isString(): boolean { return (this.type == TokenType.String); }

  isOperator(operater: string): boolean {
    return (this.type == TokenType.Operator && this.strValue == operater);
  }

  isIdentifier(): boolean { return (this.type == TokenType.Identifier); }

  isKeyword(): boolean { return (this.type == TokenType.Keyword); }

  isKeywordVar(): boolean { return (this.type == TokenType.Keyword && this.strValue == "var"); }

  isKeywordNull(): boolean { return (this.type == TokenType.Keyword && this.strValue == "null"); }

  isKeywordUndefined(): boolean {
    return (this.type == TokenType.Keyword && this.strValue == "undefined");
  }

  isKeywordTrue(): boolean { return (this.type == TokenType.Keyword && this.strValue == "true"); }

  isKeywordIf(): boolean { return (this.type == TokenType.Keyword && this.strValue == "if"); }

  isKeywordElse(): boolean { return (this.type == TokenType.Keyword && this.strValue == "else"); }

  isKeywordFalse(): boolean { return (this.type == TokenType.Keyword && this.strValue == "false"); }

  toNumber(): number {
    // -1 instead of NULL ok?
    return (this.type == TokenType.Number) ? this.numValue : -1;
  }

  toString(): string {
    switch (this.type) {
      case TokenType.Character:
      case TokenType.String:
      case TokenType.Identifier:
      case TokenType.Keyword:
        return this.strValue;
      case TokenType.Number:
        return this.numValue.toString();
      default:
        return null;
    }
  }
}

function newCharacterToken(index: number, code: number): Token {
  return new Token(index, TokenType.Character, code, StringWrapper.fromCharCode(code));
}
<commit_msg>fix(compiler): Implement Token#toString for Operator<commit_after>import {Injectable} from 'angular2/src/core/di/decorators';
import {ListWrapper, SetWrapper} from "angular2/src/core/facade/collection";
import {
  NumberWrapper,
  StringJoiner,
  StringWrapper,
  BaseException,
  isPresent
} from "angular2/src/core/facade/lang";

export enum TokenType {
  Character,
  Identifier,
  Keyword,
  String,
  Operator,
  Number
}

@Injectable()
export class Lexer {
  tokenize(text: string): any[] {
    var scanner = new _Scanner(text);
    var tokens = [];
    var token = scanner.scanToken();
    while (token != null) {
      tokens.push(token);
      token = scanner.scanToken();
    }
    return tokens;
  }
}

export class Token {
  constructor(public index: number, public type: TokenType, public numValue: number,
              public strValue: string) {}

  isCharacter(code: number): boolean {
    return (this.type == TokenType.Character && this.numValue == code);
  }

  isNumber(): boolean { return (this.type == TokenType.Number); }

  isString(): boolean { return (this.type == TokenType.String); }

  isOperator(operater: string): boolean {
    return (this.type == TokenType.Operator && this.strValue == operater);
  }

  isIdentifier(): boolean { return (this.type == TokenType.Identifier); }

  isKeyword(): boolean { return (this.type == TokenType.Keyword); }

  isKeywordVar(): boolean { return (this.type == TokenType.Keyword && this.strValue == "var"); }

  isKeywordNull(): boolean { return (this.type == TokenType.Keyword && this.strValue == "null"); }

  isKeywordUndefined(): boolean {
    return (this.type == TokenType.Keyword && this.strValue == "undefined");
  }

  isKeywordTrue(): boolean { return (this.type == TokenType.Keyword && this.strValue == "true"); }

  isKeywordIf(): boolean { return (this.type == TokenType.Keyword && this.strValue == "if"); }

  isKeywordElse(): boolean { return (this.type == TokenType.Keyword && this.strValue == "else"); }

  isKeywordFalse(): boolean { return (this.type == TokenType.Keyword && this.strValue == "false"); }

  toNumber(): number {
    // -1 instead of NULL ok?
    return (this.type == TokenType.Number) ? this.numValue : -1;
  }

  toString(): string {
    switch (this.type) {
      case TokenType.Character:
      case TokenType.Identifier:
      case TokenType.Keyword:
      case TokenType.Operator:
      case TokenType.String:
        return this.strValue;
      case TokenType.Number:
        return this.numValue.toString();
      default:
        return null;
    }
  }
}

function newCharacterToken(index: number, code: number): Token {
  return new Token(index, TokenType.Character, code, StringWrapper.fromCharCode(code));
}

{'fraction_non_alphanumeric': 0.14296351451973194, 'fraction_numerical': 0.0018615040953090098, 'mean_word_length': 3.5923076923076924, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '371323', 'n_tokens_mistral': 1615, 'n_tokens_neox': 1610, 'n_words': 396}