<commit_before>import UserDict, requests, uuid
from copy import deepcopy
from datetime import datetime, timedelta
import time
import re

from portality.core import app
import urllib2
import json

import esprit


# All models in models.py should inherit this DomainObject to know how to save themselves in the index and so on.
# You can overwrite and add to the DomainObject functions as required. See models.py for some examples.


ES_MAPPING_MISSING_REGEX = re.compile(r'.*No mapping found for \[[a-zA-Z0-9-_]+?\] in order to sort on.*', re.DOTALL)


class ElasticSearchWriteException(Exception):
    pass


class DomainObject(UserDict.IterableUserDict, object):
    __type__ = None                                                       # set the type on the model that inherits this

    def __init__(self, **kwargs):
        # if self.data is already set, don't do anything here
        try:
            object.__getattribute__(self, "data")
        except:
            if '_source' in kwargs:
                self.data = dict(kwargs['_source'])
                self.meta = dict(kwargs)
                del self.meta['_source']
            else:
                self.data = dict(kwargs)
        # FIXME: calling super() breaks everything, even thought this is the correct thing to do
        # this is because the DomainObject incorrectly overrides properties of the super class
        # super(DomainObject, self).__init__()

    @classmethod
    def target_whole_index(cls):
        t = str(app.config['ELASTIC_SEARCH_HOST']).rstrip('/') + '/'
        t += app.config['ELASTIC_SEARCH_DB'] + '/'
        return t
            
    @classmethod
    def target(cls):
        t = cls.target_whole_index()
        t += cls.__type__ + '/'
        return t
    
    @classmethod
    def makeid(cls):
        """Create a new id for data object overwrite this in specific model types if required"""
        return uuid.uuid4().hex

    @property
    def id(self):
        return self.data.get('id', None)
    
    def set_id(self, id=None):
        if id is None:
            id = self.makeid()
        self.data["id"] = id
    
    @property
    def version(self):
        return self.meta.get('_version', None)

    @property
    def json(self):
        return json.dumps(self.data)
    
    def set_created(self, date=None):
        if date is None:
            self.data['created_date'] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
        else:
            self.data['created_date'] = date

    @property
    def created_date(self):
        return self.data.get("created_date")

    @property
    def created_timestamp(self):
        return datetime.strptime(self.data.get("created_date"), "%Y-%m-%dT%H:%M:%SZ")
    
    @property
    def last_updated(self):
        return self.data.get("last_updated")

    def save(self, retries=0, back_off_factor=1, differentiate=False, blocking=False):

        if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
            app.logger.warn("System is in READ-ONLY mode, save command cannot run")
            return

        if 'id' not in self.data:
            self.data['id'] = self.makeid()

        now = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
        if (blocking or differentiate) and "last_updated" in self.data:
            diff = datetime.now() - datetime.strptime(self.data["last_updated"], "%Y-%m-%dT%H:%M:%SZ")

            # we need the new last_updated time to be later than the new one
            if diff.total_seconds() < 1:
                soon = datetime.utcnow() + timedelta(seconds=1)
                now = soon.strftime("%Y-%m-%dT%H:%M:%SZ")

        self.data['last_updated'] = now

        if 'created_date' not in self.data:
            self.data['created_date'] = now

        attempt = 0
        url = self.target() + self.data['id']
        d = json.dumps(self.data)
        r = None
        while attempt <= retries:
            try:
                r = requests.post(url, data=d)
                if r.status_code > 400:
                    raise ElasticSearchWriteException(u"Error on ES save. Response code {0}".format(r.status_code))
                else:
                    break  # everything is OK, so r should now be assigned to the result

            except requests.exceptions.ConnectionError:
                app.logger.exception(u"Failed to connect to ES")
                attempt += 1
            except ElasticSearchWriteException:
                try:
                    error_details = r.json()
                except (ValueError, AttributeError):
                    error_details = None

                # Retries depend on which end the error lies.
                if 400 <= r.status_code < 500:
                    # Bad request, do not retry as it won't work
                    app.logger.exception(u"Bad Request to ES, save failed. Details: {0}".format(error_details))
                    break
                elif r.status_code >= 500:
                    # Server error, this could be temporary so we may want to retry
                    app.logger.exception(u"Server Error from ES, retrying. Details: {0}".format(error_details))
                    attempt += 1

            # wait before retrying
            time.sleep((2**attempt) * back_off_factor)

        if attempt > retries:
            raise DAOSaveExceptionMaxRetriesReached(
                u"After the max {attempts} attempts the record with "<commit_msg>Fallback to r.text if r.json() isn't available.<commit_after>import UserDict, requests, uuid
from copy import deepcopy
from datetime import datetime, timedelta
import time
import re

from portality.core import app
import urllib2
import json

import esprit


# All models in models.py should inherit this DomainObject to know how to save themselves in the index and so on.
# You can overwrite and add to the DomainObject functions as required. See models.py for some examples.


ES_MAPPING_MISSING_REGEX = re.compile(r'.*No mapping found for \[[a-zA-Z0-9-_]+?\] in order to sort on.*', re.DOTALL)


class ElasticSearchWriteException(Exception):
    pass


class DomainObject(UserDict.IterableUserDict, object):
    __type__ = None                                                       # set the type on the model that inherits this

    def __init__(self, **kwargs):
        # if self.data is already set, don't do anything here
        try:
            object.__getattribute__(self, "data")
        except:
            if '_source' in kwargs:
                self.data = dict(kwargs['_source'])
                self.meta = dict(kwargs)
                del self.meta['_source']
            else:
                self.data = dict(kwargs)
        # FIXME: calling super() breaks everything, even thought this is the correct thing to do
        # this is because the DomainObject incorrectly overrides properties of the super class
        # super(DomainObject, self).__init__()

    @classmethod
    def target_whole_index(cls):
        t = str(app.config['ELASTIC_SEARCH_HOST']).rstrip('/') + '/'
        t += app.config['ELASTIC_SEARCH_DB'] + '/'
        return t
            
    @classmethod
    def target(cls):
        t = cls.target_whole_index()
        t += cls.__type__ + '/'
        return t
    
    @classmethod
    def makeid(cls):
        """Create a new id for data object overwrite this in specific model types if required"""
        return uuid.uuid4().hex

    @property
    def id(self):
        return self.data.get('id', None)
    
    def set_id(self, id=None):
        if id is None:
            id = self.makeid()
        self.data["id"] = id
    
    @property
    def version(self):
        return self.meta.get('_version', None)

    @property
    def json(self):
        return json.dumps(self.data)
    
    def set_created(self, date=None):
        if date is None:
            self.data['created_date'] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
        else:
            self.data['created_date'] = date

    @property
    def created_date(self):
        return self.data.get("created_date")

    @property
    def created_timestamp(self):
        return datetime.strptime(self.data.get("created_date"), "%Y-%m-%dT%H:%M:%SZ")
    
    @property
    def last_updated(self):
        return self.data.get("last_updated")

    def save(self, retries=0, back_off_factor=1, differentiate=False, blocking=False):

        if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
            app.logger.warn("System is in READ-ONLY mode, save command cannot run")
            return

        if 'id' not in self.data:
            self.data['id'] = self.makeid()

        now = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
        if (blocking or differentiate) and "last_updated" in self.data:
            diff = datetime.now() - datetime.strptime(self.data["last_updated"], "%Y-%m-%dT%H:%M:%SZ")

            # we need the new last_updated time to be later than the new one
            if diff.total_seconds() < 1:
                soon = datetime.utcnow() + timedelta(seconds=1)
                now = soon.strftime("%Y-%m-%dT%H:%M:%SZ")

        self.data['last_updated'] = now

        if 'created_date' not in self.data:
            self.data['created_date'] = now

        attempt = 0
        url = self.target() + self.data['id']
        d = json.dumps(self.data)
        r = None
        while attempt <= retries:
            try:
                r = requests.post(url, data=d)
                if r.status_code > 400:
                    raise ElasticSearchWriteException(u"Error on ES save. Response code {0}".format(r.status_code))
                else:
                    break  # everything is OK, so r should now be assigned to the result

            except requests.exceptions.ConnectionError:
                app.logger.exception(u"Failed to connect to ES")
                attempt += 1
            except ElasticSearchWriteException:
                try:
                    error_details = r.json()
                except (ValueError, AttributeError):
                    error_details = r.text

                # Retries depend on which end the error lies.
                if 400 <= r.status_code < 500:
                    # Bad request, do not retry as it won't work
                    app.logger.exception(u"Bad Request to ES, save failed. Details: {0}".format(error_details))
                    break
                elif r.status_code >= 500:
                    # Server error, this could be temporary so we may want to retry
                    app.logger.exception(u"Server Error from ES, retrying. Details: {0}".format(error_details))
                    attempt += 1

            # wait before retrying
            time.sleep((2**attempt) * back_off_factor)

        if attempt > retries:
            raise DAOSaveExceptionMaxRetriesReached(
                u"After the max {attempts} attempts the record with "
{'fraction_non_alphanumeric': 0.10385945749135263, 'fraction_numerical': 0.004915346805024577, 'mean_word_length': 2.001092597650915, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1628263', 'n_tokens_mistral': 3194, 'n_tokens_neox': 3018, 'n_words': 904}