<commit_before>from __future__ import unicode_literals

import collections
import datetime
import logging
import re
import sys

import morfessor

from . import get_version
from .categorizationscheme import get_categories, CategorizedMorph
from .exception import InvalidCategoryError
from .utils import _generator_progress

PY3 = sys.version_info.major == 3

_logger = logging.getLogger(__name__)


class FlatcatIO(morfessor.MorfessorIO):
    """Definition for all input and output files. Also handles all
    encoding issues.

    The only state this class has is the separators used in the data.
    Therefore, the same class instance can be used for initializing multiple
    files.

    Extends Morfessor Baseline data file formats to include category tags.
    """

    def __init__(self,
                 encoding=None,
                 construction_separator=' + ',
                 comment_start='#',
                 compound_separator='\s+',
                 analysis_separator=',',
                 category_separator='/',
                 strict=True):
        # FIXME: temporary fix for old-style class in released baseline
        #super(FlatcatIO, self).__init__(
        morfessor.MorfessorIO.__init__(self,
            encoding=encoding,
            construction_separator=construction_separator,
            comment_start=comment_start,
            compound_separator=compound_separator,
            atom_separator=None)
        self.analysis_separator = analysis_separator
        self.category_separator = category_separator
        self._strict = strict

    def write_segmentation_file(self, file_name, segmentations,
                                construction_sep=None,
                                output_tags=True,
                                comment_string=''):
        """Write segmentation file.

        File format (single line, wrapped only for pep8):
        <count> <construction1><cat_sep><category1><cons_sep>...
                <constructionN><cat_sep><categoryN>
        """
        construction_sep = (construction_sep if construction_sep
                            else self.construction_separator)

        _logger.info("Saving analysis to '%s'..." % file_name)
        output_morph = _make_morph_formatter(
            self.category_separator, output_tags)
        with self._open_text_file_write(file_name) as file_obj:
            d = datetime.datetime.now().replace(microsecond=0)<commit_msg>Revert "Quick fix for old-style class in released baseline"<commit_after>from __future__ import unicode_literals

import collections
import datetime
import logging
import re
import sys

import morfessor

from . import get_version
from .categorizationscheme import get_categories, CategorizedMorph
from .exception import InvalidCategoryError
from .utils import _generator_progress

PY3 = sys.version_info.major == 3

_logger = logging.getLogger(__name__)


class FlatcatIO(morfessor.MorfessorIO):
    """Definition for all input and output files. Also handles all
    encoding issues.

    The only state this class has is the separators used in the data.
    Therefore, the same class instance can be used for initializing multiple
    files.

    Extends Morfessor Baseline data file formats to include category tags.
    """

    def __init__(self,
                 encoding=None,
                 construction_separator=' + ',
                 comment_start='#',
                 compound_separator='\s+',
                 analysis_separator=',',
                 category_separator='/',
                 strict=True):
        super(FlatcatIO, self).__init__(
            encoding=encoding,
            construction_separator=construction_separator,
            comment_start=comment_start,
            compound_separator=compound_separator,
            atom_separator=None)
        self.analysis_separator = analysis_separator
        self.category_separator = category_separator
        self._strict = strict

    def write_segmentation_file(self, file_name, segmentations,
                                construction_sep=None,
                                output_tags=True,
                                comment_string=''):
        """Write segmentation file.

        File format (single line, wrapped only for pep8):
        <count> <construction1><cat_sep><category1><cons_sep>...
                <constructionN><cat_sep><categoryN>
        """
        construction_sep = (construction_sep if construction_sep
                            else self.construction_separator)

        _logger.info("Saving analysis to '%s'..." % file_name)
        output_morph = _make_morph_formatter(
            self.category_separator, output_tags)
        with self._open_text_file_write(file_name) as file_obj:
            d = datetime.datetime.now().replace(microsecond=0)
{'fraction_non_alphanumeric': 0.07036114570361146, 'fraction_numerical': 0.0024906600249066002, 'mean_word_length': 2.417730496453901, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '173694', 'n_tokens_mistral': 1246, 'n_tokens_neox': 1167, 'n_words': 317}