<commit_before>import sys
import time
from datetime import date, datetime
from decimal import Decimal
from .otypes import OrientRecordLink, OrientRecord, OrientBinaryObject
from .exceptions import PyOrientBadMethodCallException
try:
    import pyorient_native
    binary_support=True
except:
    binary_support=False

class OrientSerializationBinary(object):
    def __init__(self, props):
        self.className = None
        self.data = {}
        self.type = OrientSerialization.Binary
        self.props = props
        
    def decode(self, content):
        if not binary_support:
            raise NotImplementedError
        clsname, data = pyorient_native.deserialize(content,
                                    content.__sizeof__(), self.props)
        rels = [k for k in data.keys() if ('in_' in k or 'out_' in k
                                       or k=='in' or k=='out')] 
        for k in rels:
            if isinstance(data[k],list):
                for i in range(len(data[k])):
                    data[k][i] = OrientRecordLink(str(data[k][i][1]) + ':' +
                                                  str(data[k][i][2]))
            elif isinstance(data[k],tuple):
                data[k] = OrientRecordLink(str(data[k][1]) + ':' +
                                                  str(data[k][2]))
        return [clsname, data]

    def encode(self, record):
        raise NotImplementedError


class OrientSerializationCSV(object):
    def __init__(self):
        self.className = None
        self.data = {}
        self.type = OrientSerialization.CSV

    def decode(self, content ):
        """
         Deserialize a record.
           :param content str The input to un-serialize.
           :return: (class_name, dict)
        """

        if not content:
            return self.className, self.data

        if not isinstance(content, str):
            content = content.decode()

        content = content.strip()

        chunk = self._parse_first_key( content )

        if chunk[2]:
            # this is actually a class name.<commit_msg>Add support for binary encoding<commit_after>import sys
import time
from datetime import date, datetime
from decimal import Decimal
from .otypes import OrientRecordLink, OrientRecord, OrientBinaryObject
from .exceptions import PyOrientBadMethodCallException
try:
    import pyorient_native
    binary_support=True
except:
    binary_support=False

class OrientSerializationBinary(object):
    def __init__(self, props):
        self.className = None
        self.data = {}
        self.type = OrientSerialization.Binary
        self.props = props
        self._writer = None
        
    def decode(self, content):
        if not binary_support:
            raise Exception("To support Binary Serialization,\
                            pyorient_native must be installed")
        clsname, data = pyorient_native.deserialize(content,
                                    content.__sizeof__(), self.props)
        rels = [k for k in data.keys() if ('in_' in k or 'out_' in k
                                       or k=='in' or k=='out')] 
        for k in rels:
            if isinstance(data[k],list):
                for i in range(len(data[k])):
                    data[k][i] = OrientRecordLink(str(data[k][i][1]) + ':' +
                                                  str(data[k][i][2]))
            elif isinstance(data[k],tuple):
                data[k] = OrientRecordLink(str(data[k][1]) + ':' +
                                                  str(data[k][2]))
        return [clsname, data]

    def encode(self, record):
        if not binary_support:
            raise Exception("To support Binary Serialization,\
                            pyorient_native must be installed")
        if record:
            return pyorient_native.serialize(record)
        else:
            return None
            
            
class OrientSerializationCSV(object):
    def __init__(self):
        self.className = None
        self.data = {}
        self.type = OrientSerialization.CSV

    def decode(self, content ):
        """
         Deserialize a record.
           :param content str The input to un-serialize.
           :return: (class_name, dict)
        """

        if not content:
            return self.className, self.data

        if not isinstance(content, str):
            content = content.decode()

        content = content.strip()

        chunk = self._parse_first_key( content )

        if chunk[2]:
            # this is actually a class name.
{'fraction_non_alphanumeric': 0.09308452250274424, 'fraction_numerical': 0.0021953896816684962, 'mean_word_length': 1.751207729468599, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4302412', 'n_tokens_mistral': 1186, 'n_tokens_neox': 1142, 'n_words': 325}