<commit_before>#!/usr/local/bin/python2.7
import fileinput
import glob
import os
import re
import sys
import datetime


class JackAnalyzer:
    def __init__(self, source, dest):
        self.dest = dest
        self.tokenizer = JackTokenizer(source)
        return

    def xml(self):
        self.dest.write('<tokens>\n')
        for token in self.tokenizer.tokens:
            if token.type == Token.KEYWORD:
                self.dest.write('<keyword> ' + token.keyWord + ' </keyword>')
            elif token.type == Token.SYMBOL:
                self.dest.write('<symbol> ' + escapeSymbol(token.symbol) + ' </symbol>')
            elif token.type == Token.IDENTIFIER:
                self.dest.write('<identifier> ' + token.identifier + ' </identifier>')
            elif token.type == Token.INT_CONST:
                self.dest.write('<integerConstant> ' + str(token.intVal) + ' </integerConstant>')
            elif token.type == Token.STRING_CONST:
                self.dest.write('<stringConstant> ' + token.stringVal + ' </stringConstant>')
            else:
                sys.stderr.write("I know not this token: \"" + token.type + "\"\n")
                sys.exit(3)
            self.dest.write('\n')
        self.dest.write('</tokens>\n')


class JackTokenizer:
    keywords = 'class', 'constructor', 'function', 'method', 'field', \
               'static', 'var', 'int', 'char', 'boolean', 'void', 'true', \
               'false', 'null', 'this', 'let', 'do', 'if', 'else', 'while', \
               'return'
    symbols = '{', '}', '(', ')', '[', ']', '.', ',', ';', '+', '-', '*', \
              '/', '&', ',', '<', '>', '=', '~'
    reSymbols = re.compile(r'([{\}()\[\],.;+\-*/&<>=~])')
    reIdentifier = re.compile(r'^[a-zA-Z]{1}[a-zA-Z0-9]*$')
    reIntegerConstant = re.compile(r'^\d+$')
    reStringConstant = re.compile(r'^".*"$')

    def __init__(self, source):
        self.source = source
        self.token = None
        self.tokens = self.readAllTokens()
        return

    def hasMoreTokens(self):
        if len(self.tokens) > 0:
            return True
        else:
            return False

    def advance(self):
        self.token = self.tokens.pop(0)

    def tokenType(self):
        return self.token.type

    def symbol(self):
        return self.token.symbol

    def identifier(self):
        return self.token.identifier

    def intVal(self):
        return self.token.intVal

    def stringVal(self):
        return self.token.stringVal

    def readAllTokens(self):
        tokens = []
        input = self.source.read()
        input = re.sub(r'//.*$', '', input, flags=re.MULTILINE)  # strip comments `//`
        input = re.sub(r'/\*.*?\*/', '', input, flags=re.MULTILINE)  # strip comments `/* */`
        bigFields = input.split()
        for bigField in bigFields:
            fields = JackTokenizer.reSymbols.split(bigField)
            for field in fields:
                # sys.stderr.write("field: \"" + field + "\"\n")
                # Lame coding alert: I shouldn't have empty strings
                # and I shouldn't skip them by using `pass`. This
                # code is double-lame, but I'll never fix it, sorry.<commit_msg>🐞 Multi-line `/* */` are properly excised<commit_after>#!/usr/local/bin/python2.7
import fileinput
import glob
import os
import re
import sys
import datetime


class JackAnalyzer:
    def __init__(self, source, dest):
        self.dest = dest
        self.tokenizer = JackTokenizer(source)
        return

    def xml(self):
        self.dest.write('<tokens>\n')
        for token in self.tokenizer.tokens:
            if token.type == Token.KEYWORD:
                self.dest.write('<keyword> ' + token.keyWord + ' </keyword>')
            elif token.type == Token.SYMBOL:
                self.dest.write('<symbol> ' + escapeSymbol(token.symbol) + ' </symbol>')
            elif token.type == Token.IDENTIFIER:
                self.dest.write('<identifier> ' + token.identifier + ' </identifier>')
            elif token.type == Token.INT_CONST:
                self.dest.write('<integerConstant> ' + str(token.intVal) + ' </integerConstant>')
            elif token.type == Token.STRING_CONST:
                self.dest.write('<stringConstant> ' + token.stringVal + ' </stringConstant>')
            else:
                sys.stderr.write("I know not this token: \"" + token.type + "\"\n")
                sys.exit(3)
            self.dest.write('\n')
        self.dest.write('</tokens>\n')


class JackTokenizer:
    keywords = 'class', 'constructor', 'function', 'method', 'field', \
               'static', 'var', 'int', 'char', 'boolean', 'void', 'true', \
               'false', 'null', 'this', 'let', 'do', 'if', 'else', 'while', \
               'return'
    symbols = '{', '}', '(', ')', '[', ']', '.', ',', ';', '+', '-', '*', \
              '/', '&', ',', '<', '>', '=', '~'
    reSymbols = re.compile(r'([{\}()\[\],.;+\-*/&<>=~])')
    reIdentifier = re.compile(r'^[a-zA-Z]{1}[a-zA-Z0-9]*$')
    reIntegerConstant = re.compile(r'^\d+$')
    reStringConstant = re.compile(r'^".*"$')

    def __init__(self, source):
        self.source = source
        self.token = None
        self.tokens = self.readAllTokens()
        return

    def hasMoreTokens(self):
        if len(self.tokens) > 0:
            return True
        else:
            return False

    def advance(self):
        self.token = self.tokens.pop(0)

    def tokenType(self):
        return self.token.type

    def symbol(self):
        return self.token.symbol

    def identifier(self):
        return self.token.identifier

    def intVal(self):
        return self.token.intVal

    def stringVal(self):
        return self.token.stringVal

    def readAllTokens(self):
        tokens = []
        input = self.source.read()
        input = re.sub(r'//.*$', '', input, flags=re.MULTILINE)  # strip comments `//`
        input = re.sub(r'/\*.*?\*/', '', input, flags=re.MULTILINE | re.DOTALL)  # strip comments `/* */`
        bigFields = input.split()
        for bigField in bigFields:
            fields = JackTokenizer.reSymbols.split(bigField)
            for field in fields:
                # sys.stderr.write("field: \"" + field + "\"\n") # used for debugging
                # Lame coding alert: I shouldn't have empty strings
                # and I shouldn't skip them by using `pass`. This
                # code is double-lame, but I'll never fix it, sorry.
{'fraction_non_alphanumeric': 0.1761425251742835, 'fraction_numerical': 0.0024786986831913244, 'mean_word_length': 2.015413358243811, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '296322', 'n_tokens_mistral': 2010, 'n_tokens_neox': 1941, 'n_words': 421}