<commit_before>import os

from .character import Character

class Scanner(object):
    """docstring for Scanner."""
    ENDMARK = '\0'

    def __init__(self, source_text):
        super(Scanner, self).__init__()

        self.source_text = source_text
        self.last_index = len(source_text) - 1
        self.source_index = -1
        self.line_index = 0
        self.col_index = -1

    def get_next(self):
        self.source_index += 1

        if self.source_index > 0:
            if self.source_text[self.source_index-1] == os.linesep:  # the previous character was a newline
                self.line_index += 1
                self.col_index = -1

        self.col_index += 1

        if self.source_index > self.last_index: # end of source_text
            value = self.ENDMARK
        else:
            value = self.source_text[self.source_index]

        return Character(
            value,
            self.line_index,
            self.col_index,
            self.source_index
        )

    def peek_next(self):
        pass<commit_msg>fix peek_next method and create lookahead method<commit_after>import os

from .character import Character

class Scanner(object):
    """docstring for Scanner."""
    ENDMARK = '\0'

    def __init__(self, source_text):
        super(Scanner, self).__init__()

        self.source_text = source_text
        self.last_index = len(source_text) - 1
        self.source_index = -1
        self.line_index = 0
        self.col_index = -1

    def get_next(self):
        self.source_index += 1

        if self.source_index > 0:
            if self.source_text[self.source_index-1] == os.linesep:  # the previous character was a newline
                self.line_index += 1
                self.col_index = -1

        self.col_index += 1

        if self.source_index > self.last_index: # end of source_text
            value = self.ENDMARK
        else:
            value = self.source_text[self.source_index]

        return Character(
            value,
            self.line_index,
            self.col_index,
            self.source_index
        )

    def peek_next(self, offset=1):
        source_index = self.source_index
        line_index = self.line_index
        col_index = self.col_index

        source_index += 1

        if source_index > 0:
            if self.source_text[source_index-1] == os.linesep:  # the previous character was a newline
                line_index += 1
                col_index = -1

        col_index += 1

        if source_index > self.last_index: # end of source_text
            value = self.ENDMARK
        else:
            value = self.source_text[source_index]

        return Character(
            value,
            line_index,
            col_index,
            source_index
        )

    def lookahead(self, offset=1):
        index = self.source_index + offset

        if index > self.last_index:
            return self.ENDMARK

        return self.source_text[index]
{'fraction_non_alphanumeric': 0.08324941255454851, 'fraction_numerical': 0.010070493454179255, 'mean_word_length': 1.7389705882352942, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5070501', 'n_tokens_mistral': 905, 'n_tokens_neox': 853, 'n_words': 203}