<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
Literal.py

Created by <NAME> on 2008-02-09.
Copyright (c) 2008 Openvest. All rights reserved.
"""
from rdflib import Namespace, Literal
from rdflib.Literal import bind as bindLiteral
from rdflib.Literal import _PythonToXSD
import re

XSD = Namespace(u'http://www.w3.org/2001/XMLSchema#')

################################################################################
## Let's make toPython return a Decimal if an XSD.decimal in in the triplestore
try:
    from decimal import Decimal
    bindLiteral(XSD.decimal,Decimal)
    _PythonToXSD.update(dict([(Decimal,(str,XSD.decimal))]))
except:
    pass
    
################################################################################
## Default behavior returns untyped literals as literals
## this brings untyped literals back as unicode strings
bindLiteral(None,unicode)

################################################################################
## Default behavior returns string literals as literals
## this brings  string literals back as unicode strings
bindLiteral(XSD.string,unicode)
        
################################################################################
## Let's make toPython return a datetime if the literal has fractional seconds
## Note: dateparser adapted from http://www.mnot.net/python/isodate.py
## modified to: handle fractional seconds beyond tenths 
##              and to allow pseudo iso i.e. "2001-12-15 22:43:46" 
##                                        vs "2001-12-15T22:43:46"
import datetime

date_parser = re.compile(r"""^
    (?P<year>\d{4})
    (?:-
        (?P<month>\d{1,2})
        (?:-
            (?P<day>\d{1,2})
            (?:[T ]
                (?P<hour>\d{1,2})
                :
                (?P<minute>\d{1,2})
                (?::
                    (?P<second>\d{1,2})
                    (?P<dec_second>\.\d+)?
                )?                    
                (?:Z|(?:
                        (?P<tz_sign>[+-])
                        (?P<tz_hour>\d{1,2})
                        :
                        (?P<tz_min>\d{2,2})
                     )
                )?
            )?
        )?
    )?
$""", re.VERBOSE)

def _strToDateTime(s):
        """ parse a string and return a datetime object. """
        assert isinstance(s, basestring)
        r = date_parser.search(s)
        try:
            a = r.groupdict('0')
        except:
            raise ValueError, 'invalid date string format'
            
        dt = datetime.datetime(int(a['year']),
                               int(a['month']) or 1,
                               int(a['day']) or 1,
                               # If not given these will default to 00:00:00.0
                               int(a['hour']),<commit_msg>fix to allow parsing of dateTime to be more permissive<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
Literal.py

Created by <NAME> on 2008-02-09.
Copyright (c) 2008 Openvest. All rights reserved.
"""
from rdflib import Namespace, Literal
from rdflib.Literal import bind as bindLiteral
from rdflib.Literal import _PythonToXSD
import re

XSD = Namespace(u'http://www.w3.org/2001/XMLSchema#')

################################################################################
## Let's make toPython return a Decimal if an XSD.decimal in in the triplestore
try:
    from decimal import Decimal
    bindLiteral(XSD.decimal,Decimal)
    _PythonToXSD.update(dict([(Decimal,(str,XSD.decimal))]))
except:
    pass
    
################################################################################
## Default behavior returns untyped literals as literals
## this brings untyped literals back as unicode strings
bindLiteral(None,unicode)

################################################################################
## Default behavior returns string literals as literals
## this brings  string literals back as unicode strings
bindLiteral(XSD.string,unicode)
        
################################################################################
## Let's make toPython return a datetime if the literal has fractional seconds
## Note: dateparser adapted from http://www.mnot.net/python/isodate.py
## modified to: handle fractional seconds beyond tenths 
##              and to allow pseudo iso i.e. "2001-12-15 22:43:46" 
##                                        vs "2001-12-15T22:43:46"
import datetime

date_parser = re.compile(r"""^
    (?P<year>\d{4})
    (?:-
        (?P<month>\d{1,2})
        (?:-
            (?P<day>\d{1,2})
            (?:[T ]
                (?P<hour>\d{1,2})
                :
                (?P<minute>\d{1,2})
                (?::
                    (?P<second>\d{1,2})
                    (?P<dec_second>\.\d+)?
                )?                    
                (?:Z|(?:
                        (?P<tz_sign>[+-])
                        (?P<tz_hour>\d{1,2})
                        :?
                        (?P<tz_min>\d{2,2})
                     )
                )?
            )?
        )?
    )?
$""", re.VERBOSE)

def _strToDateTime(s):
        """ parse a string and return a datetime object. """
        assert isinstance(s, basestring)
        r = date_parser.search(s)
        try:
            a = r.groupdict('0')
        except:
            raise ValueError, 'invalid date string format'
            
        dt = datetime.datetime(int(a['year']),
                               int(a['month']) or 1,
                               int(a['day']) or 1,
                               # If not given these will default to 00:00:00.0
                               int(a['hour']),
{'fraction_non_alphanumeric': 0.2274822695035461, 'fraction_numerical': 0.02517730496453901, 'mean_word_length': 1.9909862142099681, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6162031', 'n_tokens_mistral': 1675, 'n_tokens_neox': 1480, 'n_words': 374}