<commit_before>#    Copyright 2015 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License attached#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
#    License for the specific language governing permissions and limitations
#    under the License.

from gevent import monkey
monkey.patch_all()


from gevent.server import StreamServer
import socket

import msgpack
import struct
import errno
import sys
import traceback
import pwd
import os

from types import GeneratorType
from solard.logger import logger
from solard.core import SolardContext, SolardIface
from solard.tcp_core import *


SERVER_BUFF = 4096


class SolardTCPException(Exception):
    pass


class ReadFailure(SolardTCPException):
    pass


class SolardTCPHandler(object):

    def __init__(self, sock, address):
        self.sock = sock
        self.address = address
        self.ctx = SolardContext()
        self.auth = None
        self._wrote = False
        self.forked = False

    def _read(self):
        # TODO: client closed connection
        try:
            size = struct.unpack(HDR, self.sock.recv(HDR_SIZE))[0]
        except:
            raise ReadFailure("Can't read header data")
        d = []
        while True:
            b = min(size, SERVER_BUFF)
            curr = self.sock.recv(b)
            if not curr:
                raise ReadFailure("No data")
            d.append(curr)
            size -= len(curr)
            assert size >= 0
            if not size:
                break
        self._wrote = False
        try:
            return msgpack.unpackb(''.join(d))
        except:
            print '--------------------'
            print repr(d)
            print '--------------------'
            raise

    def _read_stream(self, size=None):
        if size is None:
            try:
                size = struct.unpack(HDR, self.sock.recv(HDR_SIZE))[0]
            except:
                raise ReadFailure("Can't read header data")
        while True:
            b = min(size, SERVER_BUFF)
            curr = self.sock.recv(b)
            if not curr:
                if size > 0:
                    raise ReadFailure("Expected more data")
            size -= len(curr)
            assert size >= 0
            yield curr
            if not size:
                break

    def _write(self, **kwargs):
        assert self._wrote is False
        _data = msgpack.packb(kwargs)
        size = len(_data)
        hdr = struct.pack(HDR, size)
        self.sock.sendall(hdr + _data)
        self._wrote = True

    def _write_ok(self, res):<commit_msg>Removed some debug statement<commit_after>#    Copyright 2015 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License attached#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
#    License for the specific language governing permissions and limitations
#    under the License.

from gevent import monkey
monkey.patch_all()


from gevent.server import StreamServer
import socket

import msgpack
import struct
import errno
import sys
import traceback
import pwd
import os

from types import GeneratorType
from solard.logger import logger
from solard.core import SolardContext, SolardIface
from solard.tcp_core import *


SERVER_BUFF = 4096


class SolardTCPException(Exception):
    pass


class ReadFailure(SolardTCPException):
    pass


class SolardTCPHandler(object):

    def __init__(self, sock, address):
        self.sock = sock
        self.address = address
        self.ctx = SolardContext()
        self.auth = None
        self._wrote = False
        self.forked = False

    def _read(self):
        # TODO: client closed connection
        try:
            size = struct.unpack(HDR, self.sock.recv(HDR_SIZE))[0]
        except:
            raise ReadFailure("Can't read header data")
        d = []
        while True:
            b = min(size, SERVER_BUFF)
            curr = self.sock.recv(b)
            if not curr:
                raise ReadFailure("No data")
            d.append(curr)
            size -= len(curr)
            assert size >= 0
            if not size:
                break
        self._wrote = False
        try:
            return msgpack.unpackb(''.join(d))
        except:
            raise

    def _read_stream(self, size=None):
        if size is None:
            try:
                size = struct.unpack(HDR, self.sock.recv(HDR_SIZE))[0]
            except:
                raise ReadFailure("Can't read header data")
        while True:
            b = min(size, SERVER_BUFF)
            curr = self.sock.recv(b)
            if not curr:
                if size > 0:
                    raise ReadFailure("Expected more data")
            size -= len(curr)
            assert size >= 0
            yield curr
            if not size:
                break

    def _write(self, **kwargs):
        assert self._wrote is False
        _data = msgpack.packb(kwargs)
        size = len(_data)
        hdr = struct.pack(HDR, size)
        self.sock.sendall(hdr + _data)
        self._wrote = True

    def _write_ok(self, res):
{'fraction_non_alphanumeric': 0.08421052631578947, 'fraction_numerical': 0.005867126833477136, 'mean_word_length': 1.889332003988036, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1753306', 'n_tokens_mistral': 1631, 'n_tokens_neox': 1561, 'n_words': 532}