<commit_before>"""API harvester for DataOne - for the SHARE project

Example query: https://cn.dataone.org/cn/v1/query/solr/?q=dateModified:[NOW-5DAY%20TO%20*]&rows=10
"""


## harvester for DataONE SOLR search API
from __future__ import unicode_literals

import re

import logging
from datetime import datetime
from datetime import timedelta

from lxml import etree
from dateutil.parser import *
from xml.etree import ElementTree

from nameparser import HumanName

from scrapi import requests
from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

logger = logging.getLogger(__name__)

NAME = "dataone"

DEFAULT_ENCODING = 'UTF-8'
DATAONE_SOLR_ENDPOINT = 'https://cn.dataone.org/cn/v1/query/solr/'

record_encoding = None


def copy_to_unicode(element):
    encoding = record_encoding or DEFAULT_ENCODING
    element = ''.join(element)
    if isinstance(element, unicode):
        return element
    else:
        return unicode(element, encoding=encoding)


def harvest(days_back=5):
    records = get_records(days_back)

    xml_list = []
    for record in records:
        doc_id = record.xpath("str[@name='id']")[0].text
        record = ElementTree.tostring(record, encoding=record_encoding)
        xml_list.append(RawDocument({
            'doc': record,
            'source': NAME,
            'docID': copy_to_unicode(doc_id),
            'filetype': 'xml'
        }))

    return xml_list


def get_records(days_back):
    ''' helper function to get a response from the DataONE
    API, with the specified number of rows.
    Returns an etree element with results '''
    to_date = datetime.utcnow()
    from_date = (datetime.utcnow() - timedelta(days=days_back))

    to_date = to_date.replace(hour=0, minute=0, second=0, microsecond=0)
    from_date = from_date.replace(hour=0, minute=0, second=0, microsecond=0)

    query = 'dateModified:[{}Z TO {}Z]'.format(from_date.isoformat(), to_date.isoformat())
    doc = requests.get(DATAONE_SOLR_ENDPOINT, params={
        'q': query,
        'start': 0,
        'rows': 1
    }).content
    doc = etree.XML(doc)
    rows = int(doc.xpath("//result/@numFound")[0])

    n = 0
    while n < rows:
        data = requests.get(DATAONE_SOLR_ENDPOINT, params={
            'q': query,
            'start': n,
            'rows': 1000
        })
        docs = etree.XML(data.content).xpath('//doc')
        for doc in docs:
            yield doc
        n += 1000

def get_properties(doc):
    properties = {
        'author': (doc.xpath("str[@name='author']/node()") or [''])[0],
        'authorGivenName': (doc.xpath("str[@name='authorGivenName']/node()") or [''])[0],
        'authorSurName': (doc.xpath("str[@name='authorSurName']/node()") or [''])[0],
        'authoritativeMN': (doc.xpath("str[@name='authoritativeMN']/node()") or [''])[0],
        'checksum': (doc.xpath("str[@name='checksum']/node()") or [''])[0],
        'checksumAlgorithm': (doc.xpath("str[@name='checksumAlgorithm']/node()") or [''])[0],
        'dataUrl': (doc.xpath("str[@name='dataUrl']/node()") or [''])[0],
        'datasource': (doc.xpath("str[@name='datasource']/node()") or [''])[0],
        'documents': doc.xpath("arr[@name='documents']/str/node()"),
        'dateModified': (doc.xpath("date[@name='dateModified']/node()") or [''])[0],
        'datePublished': (doc.xpath("date[@name='datePublished']/node()") or [''])[0],
        'dateUploaded': (doc.xpath("date[@name='dateUploaded']/node()") or [''])[0],
        'pubDate': (doc.xpath("date[@name='pubDate']/node()") or [''])[0],<commit_msg>flake8<commit_after>"""API harvester for DataOne - for the SHARE project

Example query: https://cn.dataone.org/cn/v1/query/solr/?q=dateModified:[NOW-5DAY%20TO%20*]&rows=10
"""


## harvester for DataONE SOLR search API
from __future__ import unicode_literals

import re

import logging
from datetime import datetime
from datetime import timedelta

from lxml import etree
from dateutil.parser import *
from xml.etree import ElementTree

from nameparser import HumanName

from scrapi import requests
from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

logger = logging.getLogger(__name__)

NAME = "dataone"

DEFAULT_ENCODING = 'UTF-8'
DATAONE_SOLR_ENDPOINT = 'https://cn.dataone.org/cn/v1/query/solr/'

record_encoding = None


def copy_to_unicode(element):
    encoding = record_encoding or DEFAULT_ENCODING
    element = ''.join(element)
    if isinstance(element, unicode):
        return element
    else:
        return unicode(element, encoding=encoding)


def harvest(days_back=5):
    records = get_records(days_back)

    xml_list = []
    for record in records:
        doc_id = record.xpath("str[@name='id']")[0].text
        record = ElementTree.tostring(record, encoding=record_encoding)
        xml_list.append(RawDocument({
            'doc': record,
            'source': NAME,
            'docID': copy_to_unicode(doc_id),
            'filetype': 'xml'
        }))

    return xml_list


def get_records(days_back):
    ''' helper function to get a response from the DataONE
    API, with the specified number of rows.
    Returns an etree element with results '''
    to_date = datetime.utcnow()
    from_date = (datetime.utcnow() - timedelta(days=days_back))

    to_date = to_date.replace(hour=0, minute=0, second=0, microsecond=0)
    from_date = from_date.replace(hour=0, minute=0, second=0, microsecond=0)

    query = 'dateModified:[{}Z TO {}Z]'.format(from_date.isoformat(), to_date.isoformat())
    doc = requests.get(DATAONE_SOLR_ENDPOINT, params={
        'q': query,
        'start': 0,
        'rows': 1
    }).content
    doc = etree.XML(doc)
    rows = int(doc.xpath("//result/@numFound")[0])

    n = 0
    while n < rows:
        data = requests.get(DATAONE_SOLR_ENDPOINT, params={
            'q': query,
            'start': n,
            'rows': 1000
        })
        docs = etree.XML(data.content).xpath('//doc')
        for doc in docs:
            yield doc
        n += 1000


def get_properties(doc):
    properties = {
        'author': (doc.xpath("str[@name='author']/node()") or [''])[0],
        'authorGivenName': (doc.xpath("str[@name='authorGivenName']/node()") or [''])[0],
        'authorSurName': (doc.xpath("str[@name='authorSurName']/node()") or [''])[0],
        'authoritativeMN': (doc.xpath("str[@name='authoritativeMN']/node()") or [''])[0],
        'checksum': (doc.xpath("str[@name='checksum']/node()") or [''])[0],
        'checksumAlgorithm': (doc.xpath("str[@name='checksumAlgorithm']/node()") or [''])[0],
        'dataUrl': (doc.xpath("str[@name='dataUrl']/node()") or [''])[0],
        'datasource': (doc.xpath("str[@name='datasource']/node()") or [''])[0],
        'documents': doc.xpath("arr[@name='documents']/str/node()"),
        'dateModified': (doc.xpath("date[@name='dateModified']/node()") or [''])[0],
        'datePublished': (doc.xpath("date[@name='datePublished']/node()") or [''])[0],
        'dateUploaded': (doc.xpath("date[@name='dateUploaded']/node()") or [''])[0],
        'pubDate': (doc.xpath("date[@name='pubDate']/node()") or [''])[0],
{'fraction_non_alphanumeric': 0.17365771812080538, 'fraction_numerical': 0.012444071588366891, 'mean_word_length': 3.510088272383354, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6873233', 'n_tokens_mistral': 2450, 'n_tokens_neox': 2360, 'n_words': 495}