<commit_before>
import babel.core
import datetime
import hashlib
import os
import re
import time

from adhocracy import model


def _set_optional(o, data, key, export_prefix=''):
    assert o
    external_key = export_prefix + key
    if external_key in data:
        setattr(o, key, data[external_key])


def encode_locale(locale):
    if locale is None:
        return None
    return '_'.join(filter(None, (locale.language, locale.script,
                                  locale.territory, locale.variant)))


def decode_locale(ldata):
    return ldata


def encode_time(dt):
    return dt.isoformat()


def decode_time(s):
    strptime = datetime.datetime.strptime
    try:
        return strptime(s, '%Y-%m-%dT%H:%M:%S.%f')
    except ValueError:
        try:
            return strptime(s, '%Y-%m-%dT%H:%M:%SZ')
        except ValueError:
            return strptime(s, '%Y-%m-%dT%H:%M:%S')


class _Transform(object):
    """ Every transform must define:
    * _ID_KEY: The name of the property to be used as key
    * _export(self, obj): A method to export an object to JSON
    * import_all(self, data): The method to import an object from data
    Instead of import_all, the class can also use the default implementation and define instead:
    * _create(data) - Create a new object with the minimum set of properties
    * _modify(obj, data) - Modify an object to have the specified properties
    """

    def __init__(self, options):
        self._options = options
        self.name = re.match(r'^(.*)Transform$', type(self).__name__).group(1)
        self.public_name = self.name.lower()
        self._model_class = getattr(model, self.name)

    def _get_by_key(self, k):
        """ Returns None if the object cannot be found,
        and the local object in database otherwise. """
        if k is None:
            return None
        findm = getattr(self._model_class, 'find_by_' + self._ID_KEY)
        res = findm(k)
        if res is not None:
            assert getattr(res, self._ID_KEY) == k
        return res

    def _compute_key(self, o):
        return unicode(getattr(o, self._ID_KEY))

    def export_all(self):
        return dict((self._compute_key(o), self._export(o))
                    for o in self._get_all())

    def _get_all(self):
        return self._model_class.all()

    def import_all(self, odict):
        return dict((k, self._import(data)) for k, data in odict.items())

    def _import(self, data):
        obj = self._get_by_key(data.get(self._ID_KEY))
        if obj:
            doUpdate = self._replacement_strategy == 'update'
        else:
            obj = self._create(data)
            doUpdate = True

        if doUpdate:
            self._modify(obj, data)
        model.meta.Session.add(obj)
        return obj

    @property
    def _replacement_strategy(self):
        return self._options.get('replacement_strategy', 'update')


class _ExportOnlyTransform(_Transform):
    def import_(self, odict, replacement_strategy):
        raise NotImplementedError()


class BadgeTransform(_Transform):
    _ID_KEY = 'title'

    def _get_by_key(self, k):
        return self._model_class.find(k)

    def _create(self, data):
        btype = data.get('adhocracy_badge_type', 'user')
        if btype == 'user':
            return model.UserBadge.create(
                title=data['title'],
                color=data['color'],
                visible=data['visible'],
                description=data['description'])
        else:
            raise NotImplementedError()

    def _modify(self, obj, data):
        obj.title = data['title']
        obj.color = data['color']
        obj.visible = data['visible']
        obj.description = data['description']

    def _export(self, obj):
        return {
            'title': obj.title,
            'color': obj.color,
            'visible': obj.visible,<commit_msg>Import: Make color and description optional fields<commit_after>
import babel.core
import datetime
import hashlib
import os
import re
import time

from adhocracy import model


def _set_optional(o, data, key, export_prefix=''):
    assert o
    external_key = export_prefix + key
    if external_key in data:
        setattr(o, key, data[external_key])


def encode_locale(locale):
    if locale is None:
        return None
    return '_'.join(filter(None, (locale.language, locale.script,
                                  locale.territory, locale.variant)))


def decode_locale(ldata):
    return ldata


def encode_time(dt):
    return dt.isoformat()


def decode_time(s):
    strptime = datetime.datetime.strptime
    try:
        return strptime(s, '%Y-%m-%dT%H:%M:%S.%f')
    except ValueError:
        try:
            return strptime(s, '%Y-%m-%dT%H:%M:%SZ')
        except ValueError:
            return strptime(s, '%Y-%m-%dT%H:%M:%S')


class _Transform(object):
    """ Every transform must define:
    * _ID_KEY: The name of the property to be used as key
    * _export(self, obj): A method to export an object to JSON
    * import_all(self, data): The method to import an object from data
    Instead of import_all, the class can also use the default implementation and define instead:
    * _create(data) - Create a new object with the minimum set of properties
    * _modify(obj, data) - Modify an object to have the specified properties
    """

    def __init__(self, options):
        self._options = options
        self.name = re.match(r'^(.*)Transform$', type(self).__name__).group(1)
        self.public_name = self.name.lower()
        self._model_class = getattr(model, self.name)

    def _get_by_key(self, k):
        """ Returns None if the object cannot be found,
        and the local object in database otherwise. """
        if k is None:
            return None
        findm = getattr(self._model_class, 'find_by_' + self._ID_KEY)
        res = findm(k)
        if res is not None:
            assert getattr(res, self._ID_KEY) == k
        return res

    def _compute_key(self, o):
        return unicode(getattr(o, self._ID_KEY))

    def export_all(self):
        return dict((self._compute_key(o), self._export(o))
                    for o in self._get_all())

    def _get_all(self):
        return self._model_class.all()

    def import_all(self, odict):
        return dict((k, self._import(data)) for k, data in odict.items())

    def _import(self, data):
        obj = self._get_by_key(data.get(self._ID_KEY))
        if obj:
            doUpdate = self._replacement_strategy == 'update'
        else:
            obj = self._create(data)
            doUpdate = True

        if doUpdate:
            self._modify(obj, data)
        model.meta.Session.add(obj)
        return obj

    @property
    def _replacement_strategy(self):
        return self._options.get('replacement_strategy', 'update')


class _ExportOnlyTransform(_Transform):
    def import_(self, odict, replacement_strategy):
        raise NotImplementedError()


class BadgeTransform(_Transform):
    _ID_KEY = 'title'

    def _get_by_key(self, k):
        return self._model_class.find(k)

    def _create(self, data):
        btype = data.get('adhocracy_badge_type', 'user')
        if btype == 'user':
            return model.UserBadge.create(
                title=data['title'],
                color=data.get('color', u''),
                visible=data['visible'],
                description=data.get('description', u''))
        else:
            raise NotImplementedError()

    def _modify(self, obj, data):
        obj.title = data['title']
        _set_optional(obj, data, 'color')
        obj.visible = data['visible']
        _set_optional(obj, data, 'description')

    def _export(self, obj):
        return {
            'title': obj.title,
            'color': obj.color,
            'visible': obj.visible,
{'fraction_non_alphanumeric': 0.11136683288582022, 'fraction_numerical': 0.0002557217747091165, 'mean_word_length': 2.3993915688830945, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6429713', 'n_tokens_mistral': 2386, 'n_tokens_neox': 2317, 'n_words': 643}