<commit_before># coding=utf-8
import logging
import re
from itertools import tee, chain

import bs4
import lxml.etree
import pointfree as pf
from fn.uniform import zip, filter, filterfalse
from functionally import first
import json

from oaipmh import common as oc
from ckanext.oaipmh import importcore
import ckanext.kata.utils
import utils
from ckanext.kata.utils import label_list_yso
from urlparse import urlparse

xml_reader = importcore.generic_xml_metadata_reader
log = logging.getLogger(__name__)

NS = {
    'dct': 'http://purl.org/dc/terms/',
    'dc': 'http://purl.org/dc/elements/1.1/',
    'cscida': "http://etsin.avointiede.fi/cscida/",
}

# TODO: Change this file to class structure to allow harvester to set values also with OAI-PMH verb 'Identify'.


def dc_metadata_reader(harvest_type):
    """ Get correct reader for given harvest_type. Currently supports 'ida' or 'default'. """
    def method(xml):
        reader_class = {u'ida': IdaDcMetadataReader}.get(unicode(harvest_type).lower(), DefaultDcMetadataReader)
        reader = reader_class(xml)
        return reader.read()
    return method


class DcMetadataReader():
    def __init__(self, xml):
        """ Create new instanse from given XML element. """
        self.xml = xml
        self.bs = bs4.BeautifulSoup(lxml.etree.tostring(self.xml), 'xml')
        self.dc = self.bs.metadata.dc

    def read(self):
        """ Parse metadata and return metadata (oaipmh.common.Metadata) with unified dictionty. """
        unified = self._read()
        result = xml_reader(self.xml).getMap()
        result['unified'] = unified
        return oc.Metadata(result)

    def _skip_note(self, note):
        return False

    def _read_notes(self):

        desc = '\r\n\r\n'.join(sorted([a.string for a in self.dc(_filter_tag_name_namespace('description', NS['dc']),
                                                                 recursive=False) if not self._skip_note(a.string)])) or ''

        return json.dumps({ "und" : desc })

    def _get_maintainer_stuff(self):
        for a in self.dc(_filter_tag_name_namespace(name='publisher', namespace=NS['dct']), recursive=False):
            for b in a(recursive=False):
                n = b.find('name').string if b.find('name') else ''
                m = b.mbox.get('resource', '') if b.mbox else ''
                p = b.phone.get('resource', '') if b.phone else ''
                h = b.get('about', '')
                yield n, m, p, h

    def _get_availability(self):
        """ Get fallback availability. By default does not return any data. """
        return []

    def _get_uploader(self):
        return ''

    def _get_version_pids(self):
        """ Get version pid. By default does not return any data. """
        return []

    def _resolve_tags(self, tag):
        try:
            if urlparse(tag).scheme in ('http', 'https'):
                resolved = label_list_yso(tag)
                if resolved:
                    return resolved
        except:
            pass
        return [tag]

    def _get_mime_type(self):
        return first([a.string for a in self.dc('format', text=re.compile('/'), recursive=False)]) or ''

    def _read(self):
        project_funder, project_funding, project_name, project_homepage = _get_project_stuff(self.dc) or ('', '', '', '')

        # Todo! This needs to be improved to use also simple-dc
        # dc(filter_tag_name_namespace('publisher', ns['dc']), recursive=False)
        availability, license_id, license_url, access_application_url = _get_rights(self.dc) or ('', '', '', '')
        if not availability:
            availability = first(self._get_availability())

        uploader = self._get_uploader()

        data_pids = list(_get_data_pids(self.dc))

        tags = []
        for tag in sorted([a.string for a in self.dc('subject', recursive=False)]):
            tags.extend(self._resolve_tags(tag))

        transl_json = {}
        for title in self.dc('title', recursive=False):
            lang = utils.convert_language(title.get('xml:lang', '').strip())
            transl_json[lang] = title.string.strip()

        title = json.dumps(transl_json)

        # Create a unified internal harvester format dict
        unified = dict(
            # ?=dc('source', recursive=False),
            # ?=dc('relation', recursive=False),
            # ?=dc('type', recursive=False),

            access_application_URL=access_application_url or '',

            # Todo! Implement
            access_request_URL='',

            algorithm=first(_get_algorithm(self.dc)) or '',

            # TODO: Handle availabilities better
            availability=availability or 'through_provider' if first(_get_download(self.dc)) else '',

            checksum=_get_checksum(self.dc) or '',

            direct_download_URL=first(_get_download(self.dc)) or '',

            # Todo! Implement
            discipline='',

            # Todo! Should be possible to implement with QDC, but not with OAI_DC
            # evdescr=[],
            # evtype=[],
            # evwhen=[],
            # evwho=[],

            # Todo! Implement
            geographic_coverage='',

            #langtitle=[dict(lang=a.get('xml:lang', ''), value=a.string) for a in self.dc('title', recursive=False)],

            title=title,

            language=','.join(sorted([a.string for a in self.dc('language', recursive=False)])),

            license_URL=license_url or '',
            license_id=license_id or 'notspecified',

            # Todo! Using only the first entry, for now
            contact=[dict(name=name or "", email=email or "", URL=url or "", phone=phone or "")
                     for name, email, phone, url in self._get_maintainer_stuff()],

            # Todo! IDA currently doesn't produce this, maybe in future
            # dc('hasFormat', recursive=False)
            mimetype=self._get_mime_type(),

            name=ckanext.kata.utils.datapid_to_name(first(data_pids) or ''),
            # name=first(map(pf.partial(urllib.quote_plus, safe=':'), get_data_pids(dc))) or '',

            notes=self._read_notes(),

            # Todo! Using only the first entry, for now
            # owner=first([a.get('resource') for a in dc('rightsHolder', recursive=False)]) or '',

            pids=[dict(id=pid, provider=_get_provider(self.bs), type='data') for pid in data_pids] +
                 [dict(id=pid, provider=_get_provider(self.bs), type='version') for pid in self._get_version_pids()] +
                 [dict(id=pid, provider=_get_provider(self.bs), type='metadata') for pid in _get_metadata_pid(self.dc)],

            agent=[dict(role='author', name=orgauth.get('value', ''), id='', organisation=orgauth.get('org', ''), URL='', fundingid='') for orgauth in _get_org_auth(self.dc)] +
                  [dict(role='funder', name=first(project_name) or '', id=first(project_name) or '', organisation=first(project_funder) or "", URL=first(project_homepage) or '', fundingid=first(project_funding) or '',)] +
                  [dict(role='owner', name=first([a.get('resource') for a in self.dc('rightsHolder', recursive=False)]) or '', id='', organisation='', URL='', fundingid='')],

            tag_string=','.join(tags) or '',<commit_msg>Quick fix to set ida pid to primary.<commit_after># coding=utf-8
import logging
import re
from itertools import tee, chain

import bs4
import lxml.etree
import pointfree as pf
from fn.uniform import zip, filter, filterfalse
from functionally import first
import json

from oaipmh import common as oc
from ckanext.oaipmh import importcore
import ckanext.kata.utils
import utils
from ckanext.kata.utils import label_list_yso
from urlparse import urlparse

xml_reader = importcore.generic_xml_metadata_reader
log = logging.getLogger(__name__)

NS = {
    'dct': 'http://purl.org/dc/terms/',
    'dc': 'http://purl.org/dc/elements/1.1/',
    'cscida': "http://etsin.avointiede.fi/cscida/",
}

# TODO: Change this file to class structure to allow harvester to set values also with OAI-PMH verb 'Identify'.


def dc_metadata_reader(harvest_type):
    """ Get correct reader for given harvest_type. Currently supports 'ida' or 'default'. """
    def method(xml):
        reader_class = {u'ida': IdaDcMetadataReader}.get(unicode(harvest_type).lower(), DefaultDcMetadataReader)
        reader = reader_class(xml)
        return reader.read()
    return method


class DcMetadataReader():
    def __init__(self, xml):
        """ Create new instanse from given XML element. """
        self.xml = xml
        self.bs = bs4.BeautifulSoup(lxml.etree.tostring(self.xml), 'xml')
        self.dc = self.bs.metadata.dc

    def read(self):
        """ Parse metadata and return metadata (oaipmh.common.Metadata) with unified dictionty. """
        unified = self._read()
        result = xml_reader(self.xml).getMap()
        result['unified'] = unified
        return oc.Metadata(result)

    def _skip_note(self, note):
        return False

    def _read_notes(self):

        desc = '\r\n\r\n'.join(sorted([a.string for a in self.dc(_filter_tag_name_namespace('description', NS['dc']),
                                                                 recursive=False) if not self._skip_note(a.string)])) or ''

        return json.dumps({ "und" : desc })

    def _get_maintainer_stuff(self):
        for a in self.dc(_filter_tag_name_namespace(name='publisher', namespace=NS['dct']), recursive=False):
            for b in a(recursive=False):
                n = b.find('name').string if b.find('name') else ''
                m = b.mbox.get('resource', '') if b.mbox else ''
                p = b.phone.get('resource', '') if b.phone else ''
                h = b.get('about', '')
                yield n, m, p, h

    def _get_availability(self):
        """ Get fallback availability. By default does not return any data. """
        return []

    def _get_uploader(self):
        return ''

    def _get_version_pids(self):
        """ Get version pid. By default does not return any data. """
        return []

    def _resolve_tags(self, tag):
        try:
            if urlparse(tag).scheme in ('http', 'https'):
                resolved = label_list_yso(tag)
                if resolved:
                    return resolved
        except:
            pass
        return [tag]

    def _get_mime_type(self):
        return first([a.string for a in self.dc('format', text=re.compile('/'), recursive=False)]) or ''

    def _read(self):
        project_funder, project_funding, project_name, project_homepage = _get_project_stuff(self.dc) or ('', '', '', '')

        # Todo! This needs to be improved to use also simple-dc
        # dc(filter_tag_name_namespace('publisher', ns['dc']), recursive=False)
        availability, license_id, license_url, access_application_url = _get_rights(self.dc) or ('', '', '', '')
        if not availability:
            availability = first(self._get_availability())

        uploader = self._get_uploader()

        data_pids = list(_get_data_pids(self.dc))

        tags = []
        for tag in sorted([a.string for a in self.dc('subject', recursive=False)]):
            tags.extend(self._resolve_tags(tag))

        transl_json = {}
        for title in self.dc('title', recursive=False):
            lang = utils.convert_language(title.get('xml:lang', '').strip())
            transl_json[lang] = title.string.strip()

        title = json.dumps(transl_json)

        def _get_primary_data_pid(data_pids):
            for dpid in data_pids:
                if dpid.startswith('urn:nbn:fi:csc-ida'):
                    data_pids.remove(dpid)
                    return [dpid]
            return []

        # Create a unified internal harvester format dict
        unified = dict(
            # ?=dc('source', recursive=False),
            # ?=dc('relation', recursive=False),
            # ?=dc('type', recursive=False),

            access_application_URL=access_application_url or '',

            # Todo! Implement
            access_request_URL='',

            algorithm=first(_get_algorithm(self.dc)) or '',

            # TODO: Handle availabilities better
            availability=availability or 'through_provider' if first(_get_download(self.dc)) else '',

            checksum=_get_checksum(self.dc) or '',

            direct_download_URL=first(_get_download(self.dc)) or '',

            # Todo! Implement
            discipline='',

            # Todo! Should be possible to implement with QDC, but not with OAI_DC
            # evdescr=[],
            # evtype=[],
            # evwhen=[],
            # evwho=[],

            # Todo! Implement
            geographic_coverage='',

            #langtitle=[dict(lang=a.get('xml:lang', ''), value=a.string) for a in self.dc('title', recursive=False)],

            title=title,

            language=','.join(sorted([a.string for a in self.dc('language', recursive=False)])),

            license_URL=license_url or '',
            license_id=license_id or 'notspecified',

            # Todo! Using only the first entry, for now
            contact=[dict(name=name or "", email=email or "", URL=url or "", phone=phone or "")
                     for name, email, phone, url in self._get_maintainer_stuff()],

            # Todo! IDA currently doesn't produce this, maybe in future
            # dc('hasFormat', recursive=False)
            mimetype=self._get_mime_type(),

            name=ckanext.kata.utils.datapid_to_name(first(data_pids) or ''),
            # name=first(map(pf.partial(urllib.quote_plus, safe=':'), get_data_pids(dc))) or '',

            notes=self._read_notes(),

            # Todo! Using only the first entry, for now
            # owner=first([a.get('resource') for a in dc('rightsHolder', recursive=False)]) or '',

            pids=[dict(id=pid, provider=_get_provider(self.bs), type='data', primary='true')
                  for pid in _get_primary_data_pid(data_pids)] +
                 [dict(id=pid, provider=_get_provider(self.bs), type='data') for pid in data_pids] +
                 [dict(id=pid, provider=_get_provider(self.bs), type='version') for pid in self._get_version_pids()] +
                 [dict(id=pid, provider=_get_provider(self.bs), type='metadata') for pid in _get_metadata_pid(self.dc)],

            agent=[dict(role='author', name=orgauth.get('value', ''), id='', organisation=orgauth.get('org', ''), URL='', fundingid='') for orgauth in _get_org_auth(self.dc)] +
                  [dict(role='funder', name=first(project_name) or '', id=first(project_name) or '', organisation=first(project_funder) or "", URL=first(project_homepage) or '', fundingid=first(project_funding) or '',)] +
                  [dict(role='owner', name=first([a.get('resource') for a in self.dc('rightsHolder', recursive=False)]) or '', id='', organisation='', URL='', fundingid='')],

            tag_string=','.join(tags) or '',
{'fraction_non_alphanumeric': 0.13533985788979755, 'fraction_numerical': 0.000670331143584931, 'mean_word_length': 2.5682850992585506, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '101717', 'n_tokens_mistral': 4526, 'n_tokens_neox': 4345, 'n_words': 1049}