<commit_before>#!/usr/bin/env python
"""
Morfessor 2.0 - Python implementation of the Morfessor method
"""

__all__ = ['MorfessorIO', 'BaselineModel', 'Encoding', 'CorpusEncoding',
           'AnnotatedCorpusEncoding', 'LexiconEncoding']

__version__ = '2.0.0pre1'
__author__ = '<NAME>, <NAME>'
__author_email__ = "<EMAIL>"

import codecs
import collections
import datetime
import gzip
import locale
import logging
import math
import random
import re
import sys
import time
import types

try:
    # In Python2 import cPickle for better performance
    import cPickle as pickle
except ImportError:
    import pickle

try:
    from functools import reduce
except ImportError:
    pass

_logger = logging.getLogger(__name__)

show_progress_bar = True


def _progress(iter_func):
    """Decorator/function for displaying a progress bar when iterating
    through a list.

    iter_func can be both a function providing a iterator (for decorator
    style use) or an iterator itself.

    No progressbar is displayed when stderr is redirected to a file

    If the progressbar module is available a fancy percentage style
    progressbar is displayed. Otherwise 20 dots are printed as indicator.

    """

    if not show_progress_bar:
        return iter_func

    #Try to see or the progressbar module is available, else fabricate our own
    try:
        from progressbar import ProgressBar
    except ImportError:
        class SimpleProgressBar:
            NUM_DOTS = 60

            def __call__(self, it):
                self.it = iter(it)
                self.i = 0

                # Dot frequency is determined as ceil(len(it) / NUM_DOTS)
                self.dotfreq = len(it) + self.NUM_DOTS - 1 // self.NUM_DOTS
                if self.dotfreq < 1:
                    self.dotfreq = 1

                return self

            def __iter__(self):
                return self

            def next(self):
                self.i += 1
                if self.i % self.dotfreq == 0:
                    sys.stderr.write('.')
                try:
                    return self.it.next()
                except StopIteration:
                    sys.stderr.write('\n')<commit_msg>Fix (Simple) progressbar bug that didn't display dots<commit_after>#!/usr/bin/env python
"""
Morfessor 2.0 - Python implementation of the Morfessor method
"""

__all__ = ['MorfessorIO', 'BaselineModel', 'Encoding', 'CorpusEncoding',
           'AnnotatedCorpusEncoding', 'LexiconEncoding']

__version__ = '2.0.0pre1'
__author__ = '<NAME>, <NAME>'
__author_email__ = "<EMAIL>"

import codecs
import collections
import datetime
import gzip
import locale
import logging
import math
import random
import re
import sys
import time
import types

try:
    # In Python2 import cPickle for better performance
    import cPickle as pickle
except ImportError:
    import pickle

try:
    from functools import reduce
except ImportError:
    pass

_logger = logging.getLogger(__name__)

show_progress_bar = True


def _progress(iter_func):
    """Decorator/function for displaying a progress bar when iterating
    through a list.

    iter_func can be both a function providing a iterator (for decorator
    style use) or an iterator itself.

    No progressbar is displayed when stderr is redirected to a file

    If the progressbar module is available a fancy percentage style
    progressbar is displayed. Otherwise 20 dots are printed as indicator.

    """

    if not show_progress_bar:
        return iter_func

    #Try to see or the progressbar module is available, else fabricate our own
    try:
        from progressbar import ProgressBar
    except ImportError:
        class SimpleProgressBar:
            NUM_DOTS = 60

            def __call__(self, it):
                self.it = iter(it)
                self.i = 0

                # Dot frequency is determined as ceil(len(it) / NUM_DOTS)
                self.dotfreq = (len(it) + self.NUM_DOTS - 1) // self.NUM_DOTS
                if self.dotfreq < 1:
                    self.dotfreq = 1

                return self

            def __iter__(self):
                return self

            def next(self):
                self.i += 1
                if self.i % self.dotfreq == 0:
                    sys.stderr.write('.')
                try:
                    return self.it.next()
                except StopIteration:
                    sys.stderr.write('\n')
{'fraction_non_alphanumeric': 0.07042572463768115, 'fraction_numerical': 0.007699275362318841, 'mean_word_length': 2.1504992867332384, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '857205', 'n_tokens_mistral': 1274, 'n_tokens_neox': 1173, 'n_words': 428}