<commit_before>"""
Validation code for the Humanitarian Exchange Language (HXL) v1.0
<NAME>
Started October 2014

License: Public Domain
Documentation: https://github.com/HXLStandard/libhxl-python/wiki
"""

import sys
import re
import os
from copy import copy
from email.utils import parseaddr

from hxl import HXLException
from hxl.model import TagPattern
from hxl.io import StreamInput, HXLReader
from hxl.taxonomy import readTaxonomy

if sys.version_info[0] > 2:
    from urllib.parse import urlparse
else:
    from urlparse import urlparse


class HXLValidationException(HXLException):
    """
    Data structure to hold a HXL validation error.
    """

    def __init__(self, message, rule=None, value=None, row=None, column=None, severity="error"):
        """Construct a new exception."""
        super(HXLValidationException, self).__init__(message)
        self.rule = rule
        self.value = value
        self.row = row
        self.column = column
        self.severity = severity

    def __str__(self):
        """Get a string rendition of this error."""
        s = ''

        if self.rule.tag_pattern:
            if self.value:
                s += '{}={} '.format(str(self.rule.tag_pattern), str(self.value))
            else:
                s += '{} '.format(str(self.rule.tag_pattern))

        if self.message:
            s += '- {}'.format(self.message)

        return s



        return s
        
        value = "<no value>"
        if self.value:
            value = str(re.sub('\s+', ' ', self.value))
            value = '"' + value[:10] + (value[10:] and '...') + '"'
        source_row_number = "?"
        if self.row:
            source_row_number = str(self.row.source_row_number + 1)
        source_column_number = "?"
        if self.column:
            source_column_number = str(self.column.source_column_number + 1)

        return "E " + "(" + source_row_number + "," + source_column_number + ") " + str(self.rule.tag_pattern) + " " + value + ": " + self.message

class SchemaRule(object):
    """
    Validation rule for a single HXL hashtag.
    """

    # allow datatypes (others ignored)
    DATATYPES = ['text', 'number', 'url', 'email', 'phone']

    def __init__(self, tag, minOccur=None, maxOccur=None, dataType=None, minValue=None, maxValue=None,
                 valuePattern=None, valueEnumeration=None, caseSensitive=True, taxonomy=None, taxonomyLevel=None,
                 callback=None, severity="error", description=None):
        if type(tag) is TagPattern:
            self.tag_pattern = tag
        else:
            self.tag_pattern = TagPattern.parse(tag)
        self.minOccur = minOccur
        self.maxOccur = maxOccur
        self.dataType = dataType
        self.minValue = minValue
        self.maxValue = maxValue
        self.valuePattern = valuePattern<commit_msg>Validate that SchemaRule gets a valid datatype.<commit_after>"""
Validation code for the Humanitarian Exchange Language (HXL) v1.0
<NAME>
Started October 2014

License: Public Domain
Documentation: https://github.com/HXLStandard/libhxl-python/wiki
"""

import sys
import re
import os
from copy import copy
from email.utils import parseaddr

from hxl import HXLException
from hxl.model import TagPattern
from hxl.io import StreamInput, HXLReader
from hxl.taxonomy import readTaxonomy

if sys.version_info[0] > 2:
    from urllib.parse import urlparse
else:
    from urlparse import urlparse


class HXLValidationException(HXLException):
    """
    Data structure to hold a HXL validation error.
    """

    def __init__(self, message, rule=None, value=None, row=None, column=None, severity="error"):
        """Construct a new exception."""
        super(HXLValidationException, self).__init__(message)
        self.rule = rule
        self.value = value
        self.row = row
        self.column = column
        self.severity = severity

    def __str__(self):
        """Get a string rendition of this error."""
        s = ''

        if self.rule.tag_pattern:
            if self.value:
                s += '{}={} '.format(str(self.rule.tag_pattern), str(self.value))
            else:
                s += '{} '.format(str(self.rule.tag_pattern))

        if self.message:
            s += '- {}'.format(self.message)

        return s



        return s
        
        value = "<no value>"
        if self.value:
            value = str(re.sub('\s+', ' ', self.value))
            value = '"' + value[:10] + (value[10:] and '...') + '"'
        source_row_number = "?"
        if self.row:
            source_row_number = str(self.row.source_row_number + 1)
        source_column_number = "?"
        if self.column:
            source_column_number = str(self.column.source_column_number + 1)

        return "E " + "(" + source_row_number + "," + source_column_number + ") " + str(self.rule.tag_pattern) + " " + value + ": " + self.message

class SchemaRule(object):
    """
    Validation rule for a single HXL hashtag.
    """

    # allow datatypes (others ignored)
    DATATYPES = ['text', 'number', 'url', 'email', 'phone']

    def __init__(self, tag, minOccur=None, maxOccur=None, dataType=None, minValue=None, maxValue=None,
                 valuePattern=None, valueEnumeration=None, caseSensitive=True, taxonomy=None, taxonomyLevel=None,
                 callback=None, severity="error", description=None):
        if type(tag) is TagPattern:
            self.tag_pattern = tag
        else:
            self.tag_pattern = TagPattern.parse(tag)
        self.minOccur = minOccur
        self.maxOccur = maxOccur
        if dataType is None or dataType in self.DATATYPES:
            self.dataType = dataType
        else:
            raise HXLException('Unknown data type: {}'.format(dataType))
        self.minValue = minValue
        self.maxValue = maxValue
        self.valuePattern = valuePattern
{'fraction_non_alphanumeric': 0.11664392905866303, 'fraction_numerical': 0.0047748976807639835, 'mean_word_length': 2.470414201183432, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6364916', 'n_tokens_mistral': 1792, 'n_tokens_neox': 1711, 'n_words': 441}