<commit_before>import string

class Tokenizer(object):

    # I don't use these two
    special_chars = ' =+-*/\\()[]{},.:;!"%&~<>?\'`|$#@'     # Table 3.1
    lexical_tokens = '=+-*/()[],.:;%&<>'                    # Meaningful?

    # I only use this one
    punctuation = '=+-*/\\()[]{},:;%&~<>?`|$#@'    # Unhandled Table 3.1 tokens

    # Token pairs (syntax and operators)
    # TODO: (/ and /) are currently removed, for reasons discussed below.
    pairs = ('::', '=>', '**', '//', '==', '/=', '<=', '>=')

    def __init__(self):
        self.characters = None
        self.prior_char = None
        self.char = None
        self.idx = None
        self.whitespace = string.whitespace.replace('\n', '')

        self.prior_delim = None

    def parse(self, line, macros={}):
        """Tokenize a line of Fortran source."""

        tokens = []

        self.idx = -1   # Bogus value to ensure idx = 0 after first iteration
        self.characters = iter(line)
        self.update_chars()

        while self.char != '\n':
            word = ''
            if self.char in self.whitespace:
                while self.char in self.whitespace:
                    word += self.char
                    self.update_chars()

            elif self.char in '"\'' or self.prior_delim:
                word = self.parse_string()
                if (self.prior_char, self.char) == ('&', '\n'):
                    tokens.append(word)
                    word = self.prior_char

            elif self.char.isalpha():
                while self.char.isalnum() or self.char in  '\'"_':
                    word += self.char
                    self.update_chars()

            elif self.char.isdigit() or self.char == '-':
                word = self.parse_numeric()

            elif self.char in ('!', '#'):
                # Abort the iteration and build the comment token
                word = line[self.idx:-1]
                self.char = '\n'

            elif self.char == '.':
                self.update_chars()
                if self.char.isdigit():
                    frac = self.parse_numeric()
                    word = '.' + frac
                else:
                    word = '.'
                    while self.char.isalpha():
                        word += self.char
                        self.update_chars()
                    if self.char == '.':
                        word += self.char
                        self.update_chars()

            elif self.char in Tokenizer.punctuation:
                word = self.char
                self.update_chars()

                # NOTE: The following check does not work for (/ and /) because
                # it produces false tokens inside `operator (/)` declarations.
                # One potential solution is to check for the `operator` token
                # inside of `tokens`, but it's a little more complicated...
                # For now, I just omit (/ and /).

                if self.prior_char + self.char in self.pairs:
                    word = self.prior_char + self.char
                    tokens.append(word)
                    self.update_chars()
                    continue

            else:
                # This should never happen
                raise ValueError

            # Modify token if needed
            if word in macros:
                # TODO: Multiword substitutions are not tokenized!
                print('replacing {} with {}'.format(word, macros[word]))
                word = macros[word]<commit_msg>Macro support removed from tokenizer<commit_after>import string

class Tokenizer(object):

    # I don't use these two
    special_chars = ' =+-*/\\()[]{},.:;!"%&~<>?\'`|$#@'     # Table 3.1
    lexical_tokens = '=+-*/()[],.:;%&<>'                    # Meaningful?

    # I only use this one
    punctuation = '=+-*/\\()[]{},:;%&~<>?`|$#@'    # Unhandled Table 3.1 tokens

    # Token pairs (syntax and operators)
    # TODO: (/ and /) are currently removed, for reasons discussed below.
    pairs = ('::', '=>', '**', '//', '==', '/=', '<=', '>=')

    def __init__(self):
        self.characters = None
        self.prior_char = None
        self.char = None
        self.idx = None
        self.whitespace = string.whitespace.replace('\n', '')

        self.prior_delim = None

    def parse(self, line):
        """Tokenize a line of Fortran source."""

        tokens = []

        self.idx = -1   # Bogus value to ensure idx = 0 after first iteration
        self.characters = iter(line)
        self.update_chars()

        while self.char != '\n':
            word = ''
            if self.char in self.whitespace:
                while self.char in self.whitespace:
                    word += self.char
                    self.update_chars()

            elif self.char in '"\'' or self.prior_delim:
                word = self.parse_string()
                if (self.prior_char, self.char) == ('&', '\n'):
                    tokens.append(word)
                    word = self.prior_char

            elif self.char.isalpha():
                while self.char.isalnum() or self.char in  '\'"_':
                    word += self.char
                    self.update_chars()

            elif self.char.isdigit() or self.char == '-':
                word = self.parse_numeric()

            elif self.char in ('!', '#'):
                # Abort the iteration and build the comment token
                word = line[self.idx:-1]
                self.char = '\n'

            elif self.char == '.':
                self.update_chars()
                if self.char.isdigit():
                    frac = self.parse_numeric()
                    word = '.' + frac
                else:
                    word = '.'
                    while self.char.isalpha():
                        word += self.char
                        self.update_chars()
                    if self.char == '.':
                        word += self.char
                        self.update_chars()

            elif self.char in Tokenizer.punctuation:
                word = self.char
                self.update_chars()

                # NOTE: The following check does not work for (/ and /) because
                # it produces false tokens inside `operator (/)` declarations.
                # One potential solution is to check for the `operator` token
                # inside of `tokens`, but it's a little more complicated...
                # For now, I just omit (/ and /).

                if self.prior_char + self.char in self.pairs:
                    word = self.prior_char + self.char
                    tokens.append(word)
                    self.update_chars()
                    continue

            else:
                # This should never happen
                raise ValueError
{'fraction_non_alphanumeric': 0.12975194481139, 'fraction_numerical': 0.0020548950535740494, 'mean_word_length': 1.2850435949027499, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3581358', 'n_tokens_mistral': 1913, 'n_tokens_neox': 1737, 'n_words': 474}