<commit_before>import os
import logging
from lxml.etree import XMLSyntaxError, XPath, tostring
import re
from urllib2 import HTTPError

from django.conf import settings
from django.core.urlresolvers import reverse

from eulcore.xmlmap.core import load_xmlobject_from_file, load_xmlobject_from_string
from eulcore.xmlmap.eadmap import EAD_NAMESPACE
from pidservices.djangowrapper.shortcuts import DjangoPidmanRestClient
from pidservices.clients import is_ark, parse_ark

from findingaids.fa.models import FindingAid, ID_DELIMITER
from findingaids.fa.urls import EADID_URL_REGEX, TITLE_LETTERS

# pre-compile an xpath to easily get node names without EAD namespace
local_name = XPath('local-name()')

# init logger for this module
logger = logging.getLogger(__name__)

def check_ead(filename, dbpath, xml=None):
    """
    Sanity check an EAD file before it is loaded to the configured database.

    Checks the following:
     - EAD schema valid
     - eadid matches expected pattern-- filename without .xml
     - check that eadid is unique within the database (only present once, in the file that will be updated)
     - additional checks done by :meth:`check_eadxml`

    :param filename: full path to the EAD file to be checked
    :param dbpath: full path within eXist where the document will be saved
    :returns: list of all errors found
    :rtype: list
    """
    errors = []
    if xml is not None:
        load_xml = load_xmlobject_from_string
        content = xml
    else:
        load_xml = load_xmlobject_from_file
        content = filename

    try:
        ead = load_xml(content, FindingAid)
    except XMLSyntaxError, e:
        # if this fails, document is not well-formed xml
        # can't do any further processing, so return
        errors.append(e)
        return errors

    # schema validation
    if not ead.schema_valid():
        # if not valid, report all schema validation errors
        # - simplify error message: report line & columen #s, and text of the error
        errors.extend('Line %d, column %d: %s' % (err.line, err.column, err.message)
                        for err in ead.validation_errors())
    
    # eadid is expected to match filename without .xml extension
    expected_eadid = os.path.basename(filename).replace('.xml', '')
    if ead.eadid.value != expected_eadid :
        errors.append("eadid '%s' does not match expected value of '%s'" % (ead.eadid.value, expected_eadid))
    else:   # if eadid is acceptable, check for uniqueness in configured database
        fa = FindingAid.objects.filter(eadid=ead.eadid.value).only("document_name", "collection_name")
        if fa.count() > 1:
            errors.append("Database already contains %s instances of eadid '%s'! (%s)"
                    % (fa.count(), ead.eadid.value, ", ".join([f.document_name for f in fa])))
        elif fa.count() == 1:
            # some inconsistency in when /db is included on exist collection names
            path = fa[0].collection_name.replace('/db', '') + "/" + fa[0].document_name
            if path != dbpath:
                errors.append("Database contains eadid '%s' in a different document (%s)"
                    % (ead.eadid.value, fa[0].document_name))

    errors.extend(check_eadxml(ead))

    return errors

def check_eadxml(ead):
    """Sanity checks specific to the EAD xml, independent of file or eXist.

    Checks the following:
     - series and index ids are present
     - fields used for search/browse title match code expectations:
        - at most one top-level origination
        - no leading whitespace in list-title (origination or unittitle)
        - alphabetical first letter (for first-letter browse)
     - eadid matches site URL regex

    :param ead: :class:`~findingaids.fa.models.FindingAid` ead instance to be checked
    :returns: list of all errors found
    :rtype: list
    """
    # NOTE: throughout, be sure to use unicode instead of string
    errors = []

    # check that series ids are set
    if ead.dsc and ead.dsc.hasSeries():
        for series in ead.dsc.c:
            errors.extend(check_series_ids(series))

    # check that any index ids are set
    for index in ead.archdesc.index:
        if not index.id:
            errors.append("%(node)s id attribute is not set for %(label)s"
                % { 'node' : local_name(index.node), 'label' : unicode(index.head) })

    # eadid matches appropriate site URL regex
    if not re.match('^%s$' % EADID_URL_REGEX, ead.eadid.value):   # entire eadid should match regex
        errors.append("eadid '%s' does not match site URL regular expression" \
                      % ead.eadid.value)

    # multiple tests to ensure xml used for search/browse list-title matches what code expects
    # -- since list title is pulled from multiple places, give enough context so it can be found & corrected
    list_title_path = "%s/%s" % (local_name(ead.list_title.node.getparent()), 
                                 local_name(ead.list_title.node))
    # - check for at most one top-level origination
    origination_count = ead.node.xpath('count(e:archdesc/e:did/e:origination)',
                                       namespaces={'e': EAD_NAMESPACE})
    if int(origination_count)  > 1:
        errors.append("Site expects only one archdesc/did/origination; found %d" \
                        % origination_count)

    # container list formatting (based on encoding practice) expects only 2 containers per did
    # - dids with more than 2 containers
    containers = ead.node.xpath('//e:did[count(e:container) > 2]',
                                namespaces={'e': EAD_NAMESPACE})
    if len(containers):
        errors.append("Site expects maximum of 2 containers per did; found %d did(s) with more than 2" \
                        % len(containers))
        errors.append(['Line %d: %s' % (c.sourceline, tostring(c)) for c in containers])
    # - dids with only one container
    containers = ead.node.xpath('//e:did[count(e:container) = 1]',
                                namespaces={'e': EAD_NAMESPACE})
    if len(containers):
        errors.append("Site expects 2 containers per did; found %d did(s) with only 1" \
                        % len(containers))
        errors.append(['Line %d: %s' % (c.sourceline, tostring(c)) for c in containers])

    # - no leading whitespace in list title
    title_node = ead.node.xpath("%s/text()" % ead.list_title_xpath,
                                namespaces={'e': EAD_NAMESPACE})
    if hasattr(title_node[0], 'text'):
        title_text = title_node[0].text
    else:
        title_text = unicode(title_node)
    if re.match('\s+', title_text):
        # using node.text because unicode() normalizes, which obscures whitespace problems
        errors.append("Found leading whitespace in list title field (%s): '%s'" % \
                        (list_title_path, ead.list_title.node.text) )
        # report with enough context that they can find the appropriate element to fix
        
    # - first letter of title matches regex   -- only check if whitespace test fails
    elif not re.match(TITLE_LETTERS, ead.first_letter):
        errors.append("First letter ('%s') of list title field %s does not match browse letter URL regex '%s'" % \
                      (ead.first_letter, list_title_path, TITLE_LETTERS) )

    # leading whitespace in control access fields (if any)
    if ead.archdesc.controlaccess and ead.archdesc.controlaccess.controlaccess:
        for ca in ead.archdesc.controlaccess.controlaccess:
            for term in ca.terms:
                if re.match('\s+', term.value):
                    errors.append("Found leading whitespace in controlaccess term '%s' (%s)" \
                                 % (term.value, local_name(term.node)))

    # eadid url should contain resolvable ARK
    if ead.eadid.url is None or not is_ark(ead.eadid.url):
        errors.append("eadid url is either not set or not an ARK. " +
            "To correct, run the prep process again.")

    # eadid identifier should contain short-form ARK
    if ead.eadid.identifier is None or not is_ark(ead.eadid.identifier):
        errors.append("eadid identifier is either not set or not an ARK" +
            "To correct, run the prep process again.")

    # short- and long-form ARKs should match each other
    if ead.eadid.url is not None and ead.eadid.identifier is not None and \
        not ead.eadid.url.endswith(ead.eadid.identifier):
        errors.append("eadid url and identifier do not match: url '%s' should end with identifier '%s'" \
                     % (ead.eadid.url, ead.eadid.identifier))


    return errors
   
def check_series_ids(series):
    """Recursive function to check that series and subseries ids are present.
    
    :param series: :class:`findingaids.fa.models.Series`
    :returns: list of errors, if any
    """
    errors = []
    if not series.id:<commit_msg>update check_eadxml for normalized controlaccess field change to eadmap in eulcore<commit_after>import os
import logging
from lxml.etree import XMLSyntaxError, XPath, tostring
import re
from urllib2 import HTTPError

from django.conf import settings
from django.core.urlresolvers import reverse

from eulcore.xmlmap.core import load_xmlobject_from_file, load_xmlobject_from_string
from eulcore.xmlmap.eadmap import EAD_NAMESPACE
from pidservices.djangowrapper.shortcuts import DjangoPidmanRestClient
from pidservices.clients import is_ark, parse_ark

from findingaids.fa.models import FindingAid, ID_DELIMITER
from findingaids.fa.urls import EADID_URL_REGEX, TITLE_LETTERS

# pre-compile an xpath to easily get node names without EAD namespace
local_name = XPath('local-name()')

# init logger for this module
logger = logging.getLogger(__name__)

def check_ead(filename, dbpath, xml=None):
    """
    Sanity check an EAD file before it is loaded to the configured database.

    Checks the following:
     - EAD schema valid
     - eadid matches expected pattern-- filename without .xml
     - check that eadid is unique within the database (only present once, in the file that will be updated)
     - additional checks done by :meth:`check_eadxml`

    :param filename: full path to the EAD file to be checked
    :param dbpath: full path within eXist where the document will be saved
    :returns: list of all errors found
    :rtype: list
    """
    errors = []
    if xml is not None:
        load_xml = load_xmlobject_from_string
        content = xml
    else:
        load_xml = load_xmlobject_from_file
        content = filename

    try:
        ead = load_xml(content, FindingAid)
    except XMLSyntaxError, e:
        # if this fails, document is not well-formed xml
        # can't do any further processing, so return
        errors.append(e)
        return errors

    # schema validation
    if not ead.schema_valid():
        # if not valid, report all schema validation errors
        # - simplify error message: report line & columen #s, and text of the error
        errors.extend('Line %d, column %d: %s' % (err.line, err.column, err.message)
                        for err in ead.validation_errors())
    
    # eadid is expected to match filename without .xml extension
    expected_eadid = os.path.basename(filename).replace('.xml', '')
    if ead.eadid.value != expected_eadid :
        errors.append("eadid '%s' does not match expected value of '%s'" % (ead.eadid.value, expected_eadid))
    else:   # if eadid is acceptable, check for uniqueness in configured database
        fa = FindingAid.objects.filter(eadid=ead.eadid.value).only("document_name", "collection_name")
        if fa.count() > 1:
            errors.append("Database already contains %s instances of eadid '%s'! (%s)"
                    % (fa.count(), ead.eadid.value, ", ".join([f.document_name for f in fa])))
        elif fa.count() == 1:
            # some inconsistency in when /db is included on exist collection names
            path = fa[0].collection_name.replace('/db', '') + "/" + fa[0].document_name
            if path != dbpath:
                errors.append("Database contains eadid '%s' in a different document (%s)"
                    % (ead.eadid.value, fa[0].document_name))

    errors.extend(check_eadxml(ead))

    return errors

def check_eadxml(ead):
    """Sanity checks specific to the EAD xml, independent of file or eXist.

    Checks the following:
     - series and index ids are present
     - fields used for search/browse title match code expectations:
        - at most one top-level origination
        - no leading whitespace in list-title (origination or unittitle)
        - alphabetical first letter (for first-letter browse)
     - eadid matches site URL regex

    :param ead: :class:`~findingaids.fa.models.FindingAid` ead instance to be checked
    :returns: list of all errors found
    :rtype: list
    """
    # NOTE: throughout, be sure to use unicode instead of string
    errors = []

    # check that series ids are set
    if ead.dsc and ead.dsc.hasSeries():
        for series in ead.dsc.c:
            errors.extend(check_series_ids(series))

    # check that any index ids are set
    for index in ead.archdesc.index:
        if not index.id:
            errors.append("%(node)s id attribute is not set for %(label)s"
                % { 'node' : local_name(index.node), 'label' : unicode(index.head) })

    # eadid matches appropriate site URL regex
    if not re.match('^%s$' % EADID_URL_REGEX, ead.eadid.value):   # entire eadid should match regex
        errors.append("eadid '%s' does not match site URL regular expression" \
                      % ead.eadid.value)

    # multiple tests to ensure xml used for search/browse list-title matches what code expects
    # -- since list title is pulled from multiple places, give enough context so it can be found & corrected
    list_title_path = "%s/%s" % (local_name(ead.list_title.node.getparent()), 
                                 local_name(ead.list_title.node))
    # - check for at most one top-level origination
    origination_count = ead.node.xpath('count(e:archdesc/e:did/e:origination)',
                                       namespaces={'e': EAD_NAMESPACE})
    if int(origination_count)  > 1:
        errors.append("Site expects only one archdesc/did/origination; found %d" \
                        % origination_count)

    # container list formatting (based on encoding practice) expects only 2 containers per did
    # - dids with more than 2 containers
    containers = ead.node.xpath('//e:did[count(e:container) > 2]',
                                namespaces={'e': EAD_NAMESPACE})
    if len(containers):
        errors.append("Site expects maximum of 2 containers per did; found %d did(s) with more than 2" \
                        % len(containers))
        errors.append(['Line %d: %s' % (c.sourceline, tostring(c)) for c in containers])
    # - dids with only one container
    containers = ead.node.xpath('//e:did[count(e:container) = 1]',
                                namespaces={'e': EAD_NAMESPACE})
    if len(containers):
        errors.append("Site expects 2 containers per did; found %d did(s) with only 1" \
                        % len(containers))
        errors.append(['Line %d: %s' % (c.sourceline, tostring(c)) for c in containers])

    # - no leading whitespace in list title
    title_node = ead.node.xpath("%s/text()" % ead.list_title_xpath,
                                namespaces={'e': EAD_NAMESPACE})
    if hasattr(title_node[0], 'text'):
        title_text = title_node[0].text
    else:
        title_text = unicode(title_node)
    if re.match('\s+', title_text):
        # using node.text because unicode() normalizes, which obscures whitespace problems
        errors.append("Found leading whitespace in list title field (%s): '%s'" % \
                        (list_title_path, ead.list_title.node.text) )
        # report with enough context that they can find the appropriate element to fix
        
    # - first letter of title matches regex   -- only check if whitespace test fails
    elif not re.match(TITLE_LETTERS, ead.first_letter):
        errors.append("First letter ('%s') of list title field %s does not match browse letter URL regex '%s'" % \
                      (ead.first_letter, list_title_path, TITLE_LETTERS) )

    # leading whitespace in control access fields (if any)
    if ead.archdesc.controlaccess and ead.archdesc.controlaccess.controlaccess:
        for ca in ead.archdesc.controlaccess.controlaccess:
            for term in ca.terms:
                # NOTE: using node text because term.value is now normalized
                if re.match('\s+', unicode(term.node.text)):
                    errors.append("Found leading whitespace in controlaccess term '%s' (%s)" \
                                 % (term.node.text, local_name(term.node)))

    # eadid url should contain resolvable ARK
    if ead.eadid.url is None or not is_ark(ead.eadid.url):
        errors.append("eadid url is either not set or not an ARK. " +
            "To correct, run the prep process again.")

    # eadid identifier should contain short-form ARK
    if ead.eadid.identifier is None or not is_ark(ead.eadid.identifier):
        errors.append("eadid identifier is either not set or not an ARK" +
            "To correct, run the prep process again.")

    # short- and long-form ARKs should match each other
    if ead.eadid.url is not None and ead.eadid.identifier is not None and \
        not ead.eadid.url.endswith(ead.eadid.identifier):
        errors.append("eadid url and identifier do not match: url '%s' should end with identifier '%s'" \
                     % (ead.eadid.url, ead.eadid.identifier))


    return errors
   
def check_series_ids(series):
    """Recursive function to check that series and subseries ids are present.
    
    :param series: :class:`findingaids.fa.models.Series`
    :returns: list of errors, if any
    """
    errors = []
    if not series.id:
{'fraction_non_alphanumeric': 0.09436690207418118, 'fraction_numerical': 0.0018906745259411667, 'mean_word_length': 2.5939248601119105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 45}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2948755', 'n_tokens_mistral': 5268, 'n_tokens_neox': 5129, 'n_words': 1744}