<commit_before>#! -*- coding: utf-8 -*-
# package modules
from JapaneseTokenizer.object_models import WrapperBase
from JapaneseTokenizer.common import text_preprocess, juman_utils
from JapaneseTokenizer import init_logger
from JapaneseTokenizer.datamodels import FilteredObject, TokenizedSenetence
from JapaneseTokenizer.common.sever_handler import JumanppHnadler  # todo
# pyknp modules
from pyknp.juman.mlist import MList
# else
from typing import List, Union, TypeVar, Tuple, Callable
from six import text_type
import logging
import sys
import socket
import six
import os
import re
__author__ = 'kensuke-mi'

logger = init_logger.init_logger(logging.getLogger(init_logger.LOGGER_NAME))
python_version = sys.version_info
ContentsTypes = TypeVar('T')

try:
    import pyknp
except ImportError:
    logger.warning(msg='pyknp is not ready to use. Install first if you would like to use pyknp wrapper.')


class MonkeyPatchSocket(object):
    """* Class for overwriting pyknp.Socket because it is only for python2.x"""
    def __init__(self, hostname, port, option=None):
        try:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.connect((hostname, port))
        except:
            raise
        if option is not None:
            self.sock.send(option)
        data = b""
        while b"OK" not in data:
            #while isinstance(data, bytes) and b"OK" not in data:
            data = self.sock.recv(1024)

    def __del__(self):
        if self.sock:
            self.sock.close()

    def query(self, sentence, pattern):
        assert(isinstance(sentence, six.text_type))
        self.sock.sendall(b"%s\n" % sentence.encode('utf-8').strip())
        data = self.sock.recv(1024)
        assert isinstance(data, bytes)
        recv = data
        while not re.search(pattern, recv):
            data = self.sock.recv(1024)
            recv = b"%s%s" % (recv, data)
        return recv.strip().decode('utf-8')


class JumanWrapper(WrapperBase):
    def __init__(self,
                 command='juman',
                 server=None,
                 port=32000,
                 timeout=30,
                 rcfile=None,
                 option='-e2 -B',
                 pattern='EOS',
                 is_use_pyknp=False,
                 **args):
        """* Class to call Juman tokenizer
        """
        # type: (text_type,Union[str,None],int,int,text_type,Union[bytes,str],Union[bytes,str],bool)->None

        self.timeout = timeout
        self.pattern = pattern
        self.option = option
        self.command = command
        if not rcfile is None and not os.path.exists(rcfile):
            raise FileExistsError('rcfile does not exist at {}'.format(rcfile))
        if not server is None:
            ### It converts from str into bytes only for sever mode ###
            self.option = self.option.encode('utf-8')
            self.pattern = self.pattern.encode('utf-8')
        else:
            pass

        # check os #
        if os.name == 'nt':
            if not is_use_pyknp:
                logger.warning(msg='It forces is_use_pyknp = True on Windows.')
            else:
                pass
            self.is_use_pyknp = True
        else:
            pass


        if is_use_pyknp:
            self.juman = pyknp.Juman(command=command, server=server, port=port,
                                     timeout=self.timeout, rcfile=rcfile, option=option,
                                     pattern=pattern, **args)
            ### It overwrites juman_lines() method ###
            self.juman.juman_lines = self.__monkey_patch_juman_lines
        else:
            self.juman = JumanppHnadler(jumanpp_command=command,<commit_msg>fixed a bug in juman server mode for python3<commit_after>#! -*- coding: utf-8 -*-
# package modules
from JapaneseTokenizer.object_models import WrapperBase
from JapaneseTokenizer.common import text_preprocess, juman_utils
from JapaneseTokenizer import init_logger
from JapaneseTokenizer.datamodels import FilteredObject, TokenizedSenetence
from JapaneseTokenizer.common.sever_handler import JumanppHnadler  # todo
# pyknp modules
from pyknp.juman.mlist import MList
# else
from typing import List, Union, TypeVar, Tuple, Callable
from six import text_type
import logging
import sys
import socket
import six
import os
import re
__author__ = 'kensuke-mi'

logger = init_logger.init_logger(logging.getLogger(init_logger.LOGGER_NAME))
python_version = sys.version_info
ContentsTypes = TypeVar('T')

try:
    import pyknp
except ImportError:
    logger.warning(msg='pyknp is not ready to use. Install first if you would like to use pyknp wrapper.')


class MonkeyPatchSocket(object):
    """* Class for overwriting pyknp.Socket because it is only for python2.x"""
    def __init__(self, hostname, port, option=None):
        try:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.connect((hostname, port))
        except:
            raise
        if option is not None:
            self.sock.send(option)
        data = b""
        while b"OK" not in data:
            #while isinstance(data, bytes) and b"OK" not in data:
            data = self.sock.recv(1024)

    def __del__(self):
        if self.sock:
            self.sock.close()

    def query(self, sentence, pattern):
        """"""
        # type: (str,str)->str
        assert(isinstance(sentence, six.text_type))
        sentence_bytes = sentence.encode('utf-8').strip()
        pattern_bytes = pattern.encode('utf-8')

        self.sock.sendall(sentence_bytes + b"\n")
        data = self.sock.recv(1024)
        assert isinstance(data, bytes)
        recv = data
        while not re.search(pattern_bytes, recv):
            data = self.sock.recv(1024)
            recv = recv + data
        return recv.strip().decode('utf-8')


class JumanWrapper(WrapperBase):
    def __init__(self,
                 command='juman',
                 server=None,
                 port=32000,
                 timeout=30,
                 rcfile=None,
                 option='-e2 -B',
                 pattern='EOS',
                 is_use_pyknp=False,
                 **args):
        """* Class to call Juman tokenizer
        """
        # type: (text_type,Union[str,None],int,int,text_type,Union[bytes,str],Union[bytes,str],bool)->None

        self.timeout = timeout
        self.pattern = pattern
        self.option = option
        self.command = command
        if not rcfile is None and not os.path.exists(rcfile):
            raise FileExistsError('rcfile does not exist at {}'.format(rcfile))
        if not server is None:
            ### It converts from str into bytes only for sever mode ###
            self.option = self.option.encode('utf-8')
            self.pattern = self.pattern.encode('utf-8')
        else:
            pass

        # check os #
        if os.name == 'nt':
            if not is_use_pyknp:
                logger.warning(msg='It forces is_use_pyknp = True on Windows.')
            else:
                pass
            self.is_use_pyknp = True
        else:
            pass


        if is_use_pyknp or not server is None:
            self.juman = pyknp.Juman(command=command, server=server, port=port,
                                     timeout=self.timeout, rcfile=rcfile, option=option,
                                     pattern=pattern, **args)
            ### It overwrites juman_lines() method ###
            self.juman.juman_lines = self.__monkey_patch_juman_lines
        else:
            self.juman = JumanppHnadler(jumanpp_command=command,
{'fraction_non_alphanumeric': 0.09751935949599685, 'fraction_numerical': 0.007087544297151857, 'mean_word_length': 2.1500620090946674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5008660', 'n_tokens_mistral': 2236, 'n_tokens_neox': 2174, 'n_words': 563}