<commit_before>class GherkinLine:
    def __init__(self, line_text, line_number):
        self._line_text = line_text
        self._line_number = line_number
        self._trimmed_line_text = line_text.lstrip()
        self.indent = len(line_text) - len(self._trimmed_line_text)

    def get_rest_trimmed(self, length):
        return self._trimmed_line_text[length:].strip()

    def get_line_text(self, indent_to_remove=-1):
        if indent_to_remove < 0 or indent_to_remove > self.indent:
            return self._trimmed_line_text
        else:
            return self._line_text[indent_to_remove:]

    def is_empty(self):
        return not self._trimmed_line_text

    def startswith(self, prefix):
        return self._trimmed_line_text.startswith(prefix)

    def startswith_title_keyword(self, keyword):
        return self._trimmed_line_text.startswith(keyword + ':')

    def table_cells(self):
        column = self.indent + 1
        items = self._trimmed_line_text.strip().split('|')
        cells = []
        for item in items[1:-1]:
            cell_indent = len(item) - len(item.lstrip()) + 1
            cells.append({'column': column + cell_indent, 'text': item.strip()})
            column += len(item) + 1
        return cells

    def tags(self):<commit_msg>Python: handle escaping in table cells<commit_after>def table_cells(row):
    """
    An iterator returning all the table cells in a row, accounting for
    escaping.
    """

    row = iter(row)
    cell = ''
    while True:
        char = next(row, None)
        if char == '|':
            yield cell
            cell = ''
        elif char == '\\':
            char = next(row)
            if char == 'n':
                cell += '\n'
            else:
                cell += char
        elif char:
            cell += char
        else:
            break
    yield cell


class GherkinLine:
    def __init__(self, line_text, line_number):
        self._line_text = line_text
        self._line_number = line_number
        self._trimmed_line_text = line_text.lstrip()
        self.indent = len(line_text) - len(self._trimmed_line_text)

    def get_rest_trimmed(self, length):
        return self._trimmed_line_text[length:].strip()

    def get_line_text(self, indent_to_remove=-1):
        if indent_to_remove < 0 or indent_to_remove > self.indent:
            return self._trimmed_line_text
        else:
            return self._line_text[indent_to_remove:]

    def is_empty(self):
        return not self._trimmed_line_text

    def startswith(self, prefix):
        return self._trimmed_line_text.startswith(prefix)

    def startswith_title_keyword(self, keyword):
        return self._trimmed_line_text.startswith(keyword + ':')

    def table_cells(self):
        column = self.indent + 1
        items = list(table_cells(self._trimmed_line_text.strip()))
        cells = []
        for item in items[1:-1]:
            cell_indent = len(item) - len(item.lstrip()) + 1
            cells.append({'column': column + cell_indent, 'text': item.strip()})
            column += len(item) + 1
        return cells

    def tags(self):
{'fraction_non_alphanumeric': 0.10517902813299233, 'fraction_numerical': 0.004475703324808184, 'mean_word_length': 2.297154899894626, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4298819', 'n_tokens_mistral': 980, 'n_tokens_neox': 943, 'n_words': 206}