<commit_before>#!/usr/bin/env python
"""edited session.py to work with streams, and move msg_type to the header
"""


import os
import sys
import traceback
import pprint
import uuid
from datetime import datetime

import zmq
from zmq.utils import jsonapi
from zmq.eventloop.zmqstream import ZMQStream

from IPython.zmq.pickleutil import can, uncan, canSequence, uncanSequence
from IPython.zmq.newserialized import serialize, unserialize

try:
    import cPickle
    pickle = cPickle
except:
    cPickle = None
    import pickle

# packer priority: jsonlib[2], cPickle, simplejson/json, pickle
json_name = '' if not jsonapi.jsonmod else jsonapi.jsonmod.__name__
if json_name in ('jsonlib', 'jsonlib2'):
    use_json = True
elif json_name:
    if cPickle is None:
        use_json = True
    else:
        use_json = False
else:
    use_json = False

def squash_unicode(obj):
    if isinstance(obj,dict):
        for key in obj.keys():
            obj[key] = squash_unicode(obj[key])
            if isinstance(key, unicode):
                obj[squash_unicode(key)] = obj.pop(key)
    elif isinstance(obj, list):
        for i,v in enumerate(obj):
            obj[i] = squash_unicode(v)
    elif isinstance(obj, unicode):
        obj = obj.encode('utf8')
    return obj

if use_json:
    default_packer = jsonapi.dumps
    default_unpacker = lambda s: squash_unicode(jsonapi.loads(s))
else:
    default_packer = lambda o: pickle.dumps(o,-1)
    default_unpacker = pickle.loads


DELIM="<IDS|MSG>"

def wrap_exception():
    etype, evalue, tb = sys.exc_info()
    tb = traceback.format_exception(etype, evalue, tb)
    exc_content = {
        u'status' : u'error',
        u'traceback' : tb,
        u'etype' : unicode(etype),
        u'evalue' : unicode(evalue)
    }
    return exc_content

class KernelError(Exception):<commit_msg>removed unicode from error dict<commit_after>#!/usr/bin/env python
"""edited session.py to work with streams, and move msg_type to the header
"""


import os
import sys
import traceback
import pprint
import uuid
from datetime import datetime

import zmq
from zmq.utils import jsonapi
from zmq.eventloop.zmqstream import ZMQStream

from IPython.zmq.pickleutil import can, uncan, canSequence, uncanSequence
from IPython.zmq.newserialized import serialize, unserialize

try:
    import cPickle
    pickle = cPickle
except:
    cPickle = None
    import pickle

# packer priority: jsonlib[2], cPickle, simplejson/json, pickle
json_name = '' if not jsonapi.jsonmod else jsonapi.jsonmod.__name__
if json_name in ('jsonlib', 'jsonlib2'):
    use_json = True
elif json_name:
    if cPickle is None:
        use_json = True
    else:
        use_json = False
else:
    use_json = False

def squash_unicode(obj):
    if isinstance(obj,dict):
        for key in obj.keys():
            obj[key] = squash_unicode(obj[key])
            if isinstance(key, unicode):
                obj[squash_unicode(key)] = obj.pop(key)
    elif isinstance(obj, list):
        for i,v in enumerate(obj):
            obj[i] = squash_unicode(v)
    elif isinstance(obj, unicode):
        obj = obj.encode('utf8')
    return obj

if use_json:
    default_packer = jsonapi.dumps
    default_unpacker = lambda s: squash_unicode(jsonapi.loads(s))
else:
    default_packer = lambda o: pickle.dumps(o,-1)
    default_unpacker = pickle.loads


DELIM="<IDS|MSG>"

def wrap_exception():
    etype, evalue, tb = sys.exc_info()
    tb = traceback.format_exception(etype, evalue, tb)
    exc_content = {
        'status' : 'error',
        'traceback' : str(tb),
        'etype' : str(etype),
        'evalue' : str(evalue)
    }
    return exc_content

class KernelError(Exception):
{'fraction_non_alphanumeric': 0.0997275204359673, 'fraction_numerical': 0.0021798365122615805, 'mean_word_length': 3.20504009163803, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5984851', 'n_tokens_mistral': 1257, 'n_tokens_neox': 1204, 'n_words': 329}