<commit_before>## This file is part of Invenio.
## Copyright (C) 2011, 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

""" Invenio Authorlist Data Conversion Engine. """
import time
try:
    import json
except ImportError:
    import simplejson as json
from xml.dom import minidom

try:
    from xml.etree import ElementTree as ET
except ImportError:
    import elementtree.ElementTree as ET

from invenio.webuser import page_not_authorized
from invenio.access_control_engine import acc_authorize_action
import invenio.authorlist_config as cfg
from invenio.search_engine import perform_request_search
from invenio.search_engine_utils import get_fieldvalues
from invenio.bibedit_utils import get_record
# from lxml import etree
from invenio.authorlist_dblayer import get_owner
from invenio.textutils import escape_latex

# default name that will be used, when affiliation name is missing
UNKNOWN_AFFILIATION = 'Unknown Affiliation'
# Namespaces used in the xml file
NAMESPACES = {'cal': 'http://www.slac.stanford.edu/spires/hepnames/authors_xml/',
              'foaf': 'http://xmlns.com/foaf/0.1/',
              }


def retrieve_data_from_record(recid):
    """
    Extract data from a record id in order to import it to the Author list
    interface
    """
    output = {}
    # Save the affiliatons variable, the default value for "Affiliation" column
    # will be always first value from type_of_affiliation table
    type_of_affiliation = cfg.OPTIONS.AUTHOR_AFFILIATION_TYPE[0]
    # Save the default identifier - first element from the list of identifiers
    default_identifier = cfg.OPTIONS.IDENTIFIERS_LIST[0]
    # Save identifiers mapping
    identifiers_mapping = cfg.OPTIONS.IDENTIFIERS_MAPPING

    bibrecord = get_record(recid)
    if not bibrecord:
        # record probably doesn't exist
        return output
    # Extract paper title, collaboration name, experiment number
    paper_title = get_fieldvalues(int(recid), '245__a') and get_fieldvalues(int(recid), '245__a')[0] or ''
    collaboration_name = get_fieldvalues(int(recid), '710__g') and get_fieldvalues(int(recid), '710__g')[0] or ''
    experiment_number = get_fieldvalues(int(recid), '693__e') and get_fieldvalues(int(recid), '693__e')[0] or ''

    # Extract authors
    try:
        record_authors = bibrecord.get('100', [])
        record_authors.extend(bibrecord.get('700', []))
    except KeyError:
        return output
    # Extract affiliations
    record_affiliations = get_fieldvalues(int(recid), '100__u')
    record_affiliations.extend(get_fieldvalues(int(recid), '700__u'))

    # Generate all the author related information
    author_list = []
    for i in xrange(len(record_authors)):
        identifiers = []
        no_affiliation = True
        for field in record_authors[i][0][1:]:
        # check for affiliations and ID's
            if field[0] == 'u':
                no_affiliation = False
            if field[0] == 'i' and field[1] in identifiers_mapping:
                    identifiers.append([field[1], identifiers_mapping[field[1]]])
        if not identifiers:
            # if there are no identifiers, assign the default value
            identifiers.append(['', default_identifier])

        if no_affiliation:
            # add this UNKNOWN_AFFILIATION to the record, because later
            # we will iterate all affiliations
            record_authors[i][0].append(('u', UNKNOWN_AFFILIATION))
            # add UNKNOWN_AFFILIATION as affiliation, so catalogers can notice
            # that something is wrong
            record_affiliations.append(UNKNOWN_AFFILIATION)
        author_info = [long(i+1),  # Row number
                       '',
                       record_authors[i][0][0][1].split(',')[0],  # Family name
                       record_authors[i][0][0][1].split(',')[1].lstrip(),  # Given name
                       record_authors[i][0][0][1],  # Name on paper
                       '',  # Status
                       [[x[1], type_of_affiliation] for x in record_authors[i][0][1:] if x[0] == 'u'],
                       identifiers]
        author_list.append(author_info)

    output.update({'authors': author_list})

    # Generate all the affiliation related information
    unique_affiliations = list(set(record_affiliations))
    affiliation_list = []
    for i in xrange(len(unique_affiliations)):
        institution = perform_request_search(c="Institutions", p='110__u:"' + unique_affiliations[i] + '"')
        full_name = unique_affiliations[i]
        if len(institution) == 1:
            full_name_110_a = get_fieldvalues(institution[0], '110__a')
            if full_name_110_a:
                full_name = str(full_name_110_a[0])
            full_name_110_b = get_fieldvalues(institution[0], '110__b')
            if full_name_110_b:
                full_name += ', ' + str(full_name_110_b[0])
        affiliation = [long(i+1),
                       '',
                       unique_affiliations[i],
                       '',
                       full_name,
                       '',
                       True,
                       '']
        affiliation_list.append(affiliation)
    output.update({'affiliations': affiliation_list})
    # Add generic information about the paper
    output.update({'paper_title': paper_title,
                   'collaboration': collaboration_name,
                   'experiment_number': experiment_number,
                   'last_modified': int(time.time()),
                   'reference_ids': [],
                   'paper_id': '1'})

    return output


def retrieve_data_from_xml(xml):
    """
    Extract data from an XML file to import it to the Author list
    interface
    """

    def get_element_value_helper(element, tag):
        """
        Helper that takes an element and returns text from the first node
        of that element
        """
        text = ''
        elements_list = element.getElementsByTagName(tag)
        if elements_list:
            child = elements_list[0].firstChild
            if child:
                text = child.nodeValue

        return text
<commit_msg>WebAuthorList: fix import from record id<commit_after>## This file is part of Invenio.
## Copyright (C) 2011, 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

""" Invenio Authorlist Data Conversion Engine. """
import time
try:
    import json
except ImportError:
    import simplejson as json
from xml.dom import minidom

try:
    from xml.etree import ElementTree as ET
except ImportError:
    import elementtree.ElementTree as ET

from invenio.webuser import page_not_authorized
from invenio.access_control_engine import acc_authorize_action
import invenio.authorlist_config as cfg
from invenio.search_engine import perform_request_search, record_exists
from invenio.search_engine_utils import get_fieldvalues
from invenio.bibedit_utils import get_record
# from lxml import etree
from invenio.authorlist_dblayer import get_owner
from invenio.textutils import escape_latex

# default name that will be used, when affiliation name is missing
UNKNOWN_AFFILIATION = 'Unknown Affiliation'
# Namespaces used in the xml file
NAMESPACES = {'cal': 'http://www.slac.stanford.edu/spires/hepnames/authors_xml/',
              'foaf': 'http://xmlns.com/foaf/0.1/',
              }


def retrieve_data_from_record(recid):
    """
    Extract data from a record id in order to import it to the Author list
    interface
    """
    if not record_exists(recid):
        return

    output = {}

    DEFAULT_AFFILIATION_TYPE = cfg.OPTIONS.AUTHOR_AFFILIATION_TYPE[0]
    DEFAULT_IDENTIFIER = cfg.OPTIONS.IDENTIFIERS_LIST[0]
    IDENTIFIERS_MAPPING = cfg.OPTIONS.IDENTIFIERS_MAPPING

    bibrecord = get_record(recid)

    try:
        paper_title = get_fieldvalues(recid, '245__a')[0]
    except IndexError:
        paper_title = ""
    try:
        collaboration_name = get_fieldvalues(recid, '710__g')
    except IndexError:
        collaboration_name = ""
    try:
        experiment_number = get_fieldvalues(recid, '693__e')
    except IndexError:
        experiment_number = ""

    record_authors = bibrecord.get('100', [])
    record_authors.extend(bibrecord.get('700', []))

    author_list = []
    unique_affiliations = []

    for i, field_instance in enumerate(record_authors, 1):
        family_name = ""
        given_name = ""
        name_on_paper = ""
        status = ""
        affiliations = []
        identifiers = []
        field = field_instance[0]
        for subfield_code, subfield_value in field:
            if subfield_code == "a":
                try:
                    family_name = subfield_value.split(',')[0]
                    given_name = subfield_value.split(',')[1].lstrip()
                except:
                    pass
                name_on_paper = subfield_value
            elif subfield_code == "u":
                affiliations.append([subfield_value, DEFAULT_AFFILIATION_TYPE])
                unique_affiliations.append(subfield_value)
            elif subfield_code == "i":
                # FIXME This will currently work only with INSPIRE IDs
                id_prefix = subfield_value.split("-")[0]
                if id_prefix in IDENTIFIERS_MAPPING:
                    identifiers.append([subfield_value, IDENTIFIERS_MAPPING[id_prefix]])
                else:
                    identifiers.append(['', DEFAULT_IDENTIFIER])
        if not affiliations:
            affiliations.append([UNKNOWN_AFFILIATION, DEFAULT_AFFILIATION_TYPE])
            unique_affiliations.append(UNKNOWN_AFFILIATION)
        author_list.append([
            i,              # Row number
            '',             # Place holder for the web interface
            family_name,
            given_name,
            name_on_paper,
            status,
            affiliations,
            identifiers
        ])

    unique_affiliations = list(set(unique_affiliations))

    output.update({'authors': author_list})

    # Generate all the affiliation related information
    affiliation_list = []
    for i, affiliation in enumerate(unique_affiliations, 1):
        institution = perform_request_search(c="Institutions", p='110__u:"' + affiliation + '"')
        full_name = affiliation
        if len(institution) == 1:
            full_name_110_a = get_fieldvalues(institution[0], '110__a')
            if full_name_110_a:
                full_name = str(full_name_110_a[0])
            full_name_110_b = get_fieldvalues(institution[0], '110__b')
            if full_name_110_b:
                full_name += ', ' + str(full_name_110_b[0])
        affiliation = [i,
                       '',
                       affiliation,
                       '',
                       full_name,
                       '',
                       True,
                       '']
        affiliation_list.append(affiliation)

    output.update({'affiliations': affiliation_list})
    output.update({'paper_title': paper_title,
                   'collaboration': collaboration_name,
                   'experiment_number': experiment_number,
                   'last_modified': int(time.time()),
                   'reference_ids': [],
                   'paper_id': '1'})

    return output


def retrieve_data_from_xml(xml):
    """
    Extract data from an XML file to import it to the Author list
    interface
    """

    def get_element_value_helper(element, tag):
        """
        Helper that takes an element and returns text from the first node
        of that element
        """
        text = ''
        elements_list = element.getElementsByTagName(tag)
        if elements_list:
            child = elements_list[0].firstChild
            if child:
                text = child.nodeValue

        return text

{'fraction_non_alphanumeric': 0.08812523864070256, 'fraction_numerical': 0.016036655211912942, 'mean_word_length': 2.496929238985314, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2265492', 'n_tokens_mistral': 3872, 'n_tokens_neox': 3644, 'n_words': 1019}