<commit_before>import re

import pytz
from django.core.exceptions import ValidationError
from django.db.models.fields import DateTimeField
from django.utils import timezone

# Encode with a Z at the end to ensure that Django parses it as
# UTC.
date_time_format = "%Y-%m-%d %H:%M:%S.%fZ"
tz_format = "({tz})"
tz_regex = re.compile("\(([^\)]+)\)")
db_storage_string = "{date_time_string}{tz_string}"

class DateTimeTzField(DateTimeField):

    def get_internal_type(self):
        return "CharField"

    def to_python(self, value):
        # Take advantage of the pre-existing checking code to handle
        # values that are returned already as datetimes and None.
        # May also allow us to let Postgres timezone aware datetime objects
        # through without worrying about them.
        try:
            return super(DateTimeTzField, self).to_python(value)
        except ValidationError:
            try:
                tz = tz_regex.search(value).groups()[0]
            except IndexError:
                tz = timezone.get_default_timezone()
            utc_value = tz_regex.sub('', value)
            datetime = super(DateTimeTzField, self).to_python(utc_value)
            return datetime.astimezone(pytz.timezone(tz))

    def get_db_prep_value(self, value, connection, prepared=False):
        # Casts datetimes into the format expected by the backend
        if not prepared:
            tz = value.tzname
            value = self.get_prep_value(value)
            date_time_string = value.strptime(date_time_format)
            tz_string = tz_format.format(tz=tz)
            value = db_storage_string.format(date_time_string=date_time_string, tz_string=tz_string)
        return value<commit_msg>Working first pass implementation of DateTimeTzField, inheriting only from Field.<commit_after>import datetime
import re

import pytz
from django.db.models.fields import Field
from django.utils import timezone

# Encode with a Z at the end to ensure that Django parses it as
# UTC.
date_time_format = "%Y-%m-%d %H:%M:%S.%fZ"
tz_format = "({tz})"
tz_regex = re.compile("\(([^\)]+)\)")
db_storage_string = "{date_time_string}{tz_string}"

def parse_timezonestamp(value):
    try:
        tz = pytz.timezone(tz_regex.search(value).groups()[0])
    except IndexError:
        tz = timezone.get_default_timezone()
    utc_value = tz_regex.sub('', value)
    value = datetime.datetime.strptime(utc_value, date_time_format)
    value = timezone.make_aware(value, pytz.utc)
    return value.astimezone(tz)

class DateTimeTzField(Field):

    def db_type(self, connection):
        return "char"

    def from_db_value(self, value, expression, connection, context):
        if value is None:
            return value
        return parse_timezonestamp(value)

    def to_python(self, value):
        if isinstance(value, datetime.datetime):
            return value

        if value is None:
            return value

        return parse_timezonestamp(value)

    def get_prep_value(self, value):
        # Casts datetimes into the format expected by the backend
        # value = super(DateTimeTzField, self).get_prep_value(value)
        tz = value.tzinfo.zone
        date_time_string = value.strftime(date_time_format)
        tz_string = tz_format.format(tz=tz)
        value = db_storage_string.format(date_time_string=date_time_string, tz_string=tz_string)
        return value

    def get_db_prep_value(self, value, connection, prepared=False):
        if not prepared:
            value = self.get_prep_value(value)
        return value
{'fraction_non_alphanumeric': 0.09249367444475681, 'fraction_numerical': 0.000562271577171774, 'mean_word_length': 3.0616438356164384, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6236801', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1050, 'n_words': 275}