<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import with_statement

try:
    # For Python >= 2.6
    import json
except ImportError:
    # For Python < 2.6 or people using a newer version of simplejson
    import simplejson as json

import random
from datetime import date, datetime
from urllib import urlencode
from urlparse import urlunsplit
import base64
import time
from decimal import Decimal
from urllib import quote
import threading
import copy

try:
    from .connection import connect as thrift_connect
    from .pyesthrift.ttypes import Method, RestRequest

    thrift_enable = True
except ImportError:
    from .fakettypes import Method, RestRequest

    thrift_enable = False

from .connection_http import connect as http_connect
from . import logger
from .mappings import Mapper

from .convert_errors import raise_if_error
from .exceptions import (ElasticSearchException, IndexAlreadyExistsException,
                         IndexMissingException, InvalidQuery,
                         ReduceSearchPhaseException, VersionConflictEngineException,
                         BulkOperationException)

__all__ = ['ES', 'file_to_attachment', 'decode_json']

#
# models
#

class DotDict(dict):
    def __getattr__(self, attr):
        return self.get(attr, None)

    __setattr__ = dict.__setitem__

    __delattr__ = dict.__delitem__

    def __deepcopy__(self, memo):
        return DotDict([(copy.deepcopy(k, memo), copy.deepcopy(v, memo)) for k, v in self.items()])

class ElasticSearchModel(DotDict):
    def __init__(self, *args, **kwargs):<commit_msg>Fixed pickling DotDict. closes #183<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import with_statement

try:
    # For Python >= 2.6
    import json
except ImportError:
    # For Python < 2.6 or people using a newer version of simplejson
    import simplejson as json

import random
from datetime import date, datetime
from urllib import urlencode
from urlparse import urlunsplit
import base64
import time
from decimal import Decimal
from urllib import quote
import threading
import copy

try:
    from .connection import connect as thrift_connect
    from .pyesthrift.ttypes import Method, RestRequest

    thrift_enable = True
except ImportError:
    from .fakettypes import Method, RestRequest

    thrift_enable = False

from .connection_http import connect as http_connect
from . import logger
from .mappings import Mapper

from .convert_errors import raise_if_error
from .exceptions import (ElasticSearchException, IndexAlreadyExistsException,
                         IndexMissingException, InvalidQuery,
                         ReduceSearchPhaseException, VersionConflictEngineException,
                         BulkOperationException)

__all__ = ['ES', 'file_to_attachment', 'decode_json']

#
# models
#

class DotDict(dict):
    def __getattr__(self, attr):
        if attr.startswith('__'):
            raise AttributeError
        return self.get(attr, None)

    __setattr__ = dict.__setitem__

    __delattr__ = dict.__delitem__

    def __deepcopy__(self, memo):
        return DotDict([(copy.deepcopy(k, memo), copy.deepcopy(v, memo)) for k, v in self.items()])

class ElasticSearchModel(DotDict):
    def __init__(self, *args, **kwargs):
{'fraction_non_alphanumeric': 0.06962213626896757, 'fraction_numerical': 0.0050580184468908065, 'mean_word_length': 3.5617367706919945, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3452901', 'n_tokens_mistral': 957, 'n_tokens_neox': 922, 'n_words': 311}