<commit_before>from ...common.decorators import *
from ...common.errors import *
from .data_type import DataType
from decimal import *
import json
import dateutil.parser

class DynaText:
    def __init__(self, text="", targetaction="", dictionary="", wordcorpus=""):
        self.text = text
        self.targetaction = targetaction
        self.dictionary = dictionary
        self.wordcorpus = wordcorpus


    @staticmethod
    def from_dict(dct):
        retval = None
        try:
            retval = DynaText(dct.get('text', ""),
                                   dct.get('target_action', ""),
                                   dct.get('dictionary', ""),
                                   dct.get('wordcorpus', ""))
        except Exception as e:
            raise DeserializatinoError(DynaText, 'dict', dct) from e
        return retval


    @staticmethod
    def from_json(json_string):
        dynatext = None
        try:
            data = json.loads(json_string)
            dynatext = DynaText.from_dict(data)
        except Exception as e:
            raise DeserializationError(DynaText, 'json', data) from e
        return dynatext


    def to_dict(self):
        dct = None
        try:
            dct = {
                'text': self.text,
                'target_action': self.targetaction,
                'dictionary': self.dictionary,
                'wordcorpus': self.wordcorpus
            }
        except Exception as e:
            raise SerializationError(DynaText, 'dict') from e
        return dct


    def to_json(self):
        json_string = ""
        try:
            dct = self.to_dict()
            json_string = json.dumps(dct)
        except Exception as e:
            raise SerializationError(DynaText, 'json') from e
        return json_string




@valid_field_filters('id', 'value', 'datatype')
class DataElement:
    def __init__(self, id=None, value=None, datatype=None):
        self.id = id
        self.value = DataElement._format_input_value(datatype, value)
        self.datatype = DataType.VOID if datatype is None else datatype


    @staticmethod
    def _format_input_value(datatype, value):
        if datatype == DataType.INTEGER:
            return int(value)
        if datatype == DataType.STRING:
            return str(value)
        if datatype == DataType.BOOLEAN:
            return bool(value)
        if datatype == DataType.DECIMAL:
            return Decimal(value)
        if datatype == DataType.TIMESTAMP:
            if type(value).__name__ == 'str':
                return dateutil.parser.parse(value)
            if type(value).__name__ == 'date' :
                return datetime.datetime(value.year, value.month, value.day)
            if type(value).__name__ == 'time':
                td = datetime.today()
                return datetime.datetime(td.year, td.month, td.day, value.hour, value.minute, value.second, value.microsecond)
            if type(value).__name__ == 'datetime':
                #return value
                # Remove timezone for now since not supported
                return value.replace(tzinfo=None)
            return datetime.datetime.min
        if datatype == DataType.URI:
            return str(value)
        if datatype == DataType.DYNATEXT:
            return value
        return None


    @staticmethod
    def _convertfrom_json(datatype, value):
        if datatype == DataType.INTEGER:
            return int(value)
        if datatype == DataType.STRING:
            return str(value)
        if datatype == DataType.BOOLEAN:
            return value
        if datatype == DataType.DECIMAL:
            return Decimal(value)
        if datatype == DataType.TIMESTAMP:
            return dateutil.parser.parse(value)
        if datatype == DataType.URI:
            return str(value)
        if datatype == DataType.DYNATEXT:
            return DynaText.from_json(value)
        return None

    @staticmethod
    def _convert_to_json(datatype, value):
        if datatype == DataType.INTEGER:
            return int(value)
        if datatype == DataType.STRING:
            return str(value)
        if datatype == DataType.BOOLEAN:
            return 'true' if value else 'false'
        if datatype == DataType.DECIMAL:
            return str(value)
        if datatype == DataType.TIMESTAMP:
            return str(value.isoformat())
        if datatype == DataType.URI:
            return str(value)
        if datatype == DataType.DYNATEXT:
            return value.to_json()
        return 'Void'

    @staticmethod
    def from_dict(dct):<commit_msg>write boolean values as bools in the json<commit_after>from ...common.decorators import *
from ...common.errors import *
from .data_type import DataType
from decimal import *
import json
import dateutil.parser

class DynaText:
    def __init__(self, text="", targetaction="", dictionary="", wordcorpus=""):
        self.text = text
        self.targetaction = targetaction
        self.dictionary = dictionary
        self.wordcorpus = wordcorpus


    @staticmethod
    def from_dict(dct):
        retval = None
        try:
            retval = DynaText(dct.get('text', ""),
                                   dct.get('target_action', ""),
                                   dct.get('dictionary', ""),
                                   dct.get('wordcorpus', ""))
        except Exception as e:
            raise DeserializatinoError(DynaText, 'dict', dct) from e
        return retval


    @staticmethod
    def from_json(json_string):
        dynatext = None
        try:
            data = json.loads(json_string)
            dynatext = DynaText.from_dict(data)
        except Exception as e:
            raise DeserializationError(DynaText, 'json', data) from e
        return dynatext


    def to_dict(self):
        dct = None
        try:
            dct = {
                'text': self.text,
                'target_action': self.targetaction,
                'dictionary': self.dictionary,
                'wordcorpus': self.wordcorpus
            }
        except Exception as e:
            raise SerializationError(DynaText, 'dict') from e
        return dct


    def to_json(self):
        json_string = ""
        try:
            dct = self.to_dict()
            json_string = json.dumps(dct)
        except Exception as e:
            raise SerializationError(DynaText, 'json') from e
        return json_string




@valid_field_filters('id', 'value', 'datatype')
class DataElement:
    def __init__(self, id=None, value=None, datatype=None):
        self.id = id
        self.value = DataElement._format_input_value(datatype, value)
        self.datatype = DataType.VOID if datatype is None else datatype


    @staticmethod
    def _format_input_value(datatype, value):
        if datatype == DataType.INTEGER:
            return int(value)
        if datatype == DataType.STRING:
            return str(value)
        if datatype == DataType.BOOLEAN:
            return bool(value)
        if datatype == DataType.DECIMAL:
            return Decimal(value)
        if datatype == DataType.TIMESTAMP:
            if type(value).__name__ == 'str':
                return dateutil.parser.parse(value)
            if type(value).__name__ == 'date' :
                return datetime.datetime(value.year, value.month, value.day)
            if type(value).__name__ == 'time':
                td = datetime.today()
                return datetime.datetime(td.year, td.month, td.day, value.hour, value.minute, value.second, value.microsecond)
            if type(value).__name__ == 'datetime':
                #return value
                # Remove timezone for now since not supported
                return value.replace(tzinfo=None)
            return datetime.datetime.min
        if datatype == DataType.URI:
            return str(value)
        if datatype == DataType.DYNATEXT:
            return value
        return None


    @staticmethod
    def _convertfrom_json(datatype, value):
        if datatype == DataType.INTEGER:
            return int(value)
        if datatype == DataType.STRING:
            return str(value)
        if datatype == DataType.BOOLEAN:
            return value
        if datatype == DataType.DECIMAL:
            return Decimal(value)
        if datatype == DataType.TIMESTAMP:
            return dateutil.parser.parse(value)
        if datatype == DataType.URI:
            return str(value)
        if datatype == DataType.DYNATEXT:
            return DynaText.from_json(value)
        return None

    @staticmethod
    def _convert_to_json(datatype, value):
        if datatype == DataType.INTEGER:
            return int(value)
        if datatype == DataType.STRING:
            return str(value)
        if datatype == DataType.BOOLEAN:
            #return 'true' if value else 'false'
            return value
        if datatype == DataType.DECIMAL:
            return str(value)
        if datatype == DataType.TIMESTAMP:
            return str(value.isoformat())
        if datatype == DataType.URI:
            return str(value)
        if datatype == DataType.DYNATEXT:
            return value.to_json()
        return 'Void'

    @staticmethod
    def from_dict(dct):
{'fraction_non_alphanumeric': 0.09200260078023408, 'fraction_numerical': 0.0, 'mean_word_length': 1.8309815950920245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3609550', 'n_tokens_mistral': 2586, 'n_tokens_neox': 2517, 'n_words': 636}