<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2015-2016, Exa Analytics Development Team
# Distributed under the terms of the Apache License 2.0
"""
Editor
####################################
The :class:`~exa.management.editor.Editor` class is a way for programmatic text-
editor-like manipulation of files on disk. It does not strive to be a full
featured text editor. A large number of Pythonic operations can be performed on
editors:

.. code-block:: Python

    editor = Editor(file.txt)       # Create an editor from a file
    editor = Editor(file.txt.gz)    # Automatically decompress
    editor = Editor(file.txt.bz2)
    for line in editor:             # Iterate over lines in the file
        pass
    if 'text' in editor:            # True if "text" appears on any line
        True

Text lines are stored in memory; file handles are only open during reading and
writing. For large repetitive files, memoization can reduce the memory footprint
(see the **as_interned** kwarg).
"""
import os
import re
import sys
import gzip
import bz2
import pandas as pd
from copy import copy, deepcopy
from operator import add
from collections import Counter
from io import StringIO, TextIOWrapper


class Editor:
    """
    An in memory copy of a file on disk that can be programmatically manipulated.

    Editor line numbers use a 0 base index. To increase the number of lines<commit_msg>Need systematic way of dealing with string in py2 and 3<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2015-2016, Exa Analytics Development Team
# Distributed under the terms of the Apache License 2.0
"""
Editor
####################################
The :class:`~exa.management.editor.Editor` class is a way for programmatic text-
editor-like manipulation of files on disk. It does not strive to be a full
featured text editor. A large number of Pythonic operations can be performed on
editors:

.. code-block:: Python

    editor = Editor(file.txt)       # Create an editor from a file
    editor = Editor(file.txt.gz)    # Automatically decompress
    editor = Editor(file.txt.bz2)
    for line in editor:             # Iterate over lines in the file
        pass
    if 'text' in editor:            # True if "text" appears on any line
        True

Text lines are stored in memory; file handles are only open during reading and
writing. For large repetitive files, memoization can reduce the memory footprint
(see the **as_interned** kwarg).
"""
import os
import re
import sys
import gzip
import bz2
import pandas as pd
from builtins import bytes
from copy import copy, deepcopy
from operator import add
from collections import Counter
from io import StringIO, TextIOWrapper


class Editor:
    """
    An in memory copy of a file on disk that can be programmatically manipulated.

    Editor line numbers use a 0 base index. To increase the number of lines
{'fraction_non_alphanumeric': 0.08452672022354174, 'fraction_numerical': 0.010478519035976248, 'mean_word_length': 3.454121306376361, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '740480', 'n_tokens_mistral': 775, 'n_tokens_neox': 729, 'n_words': 387}