<commit_before>"""
A harvester for the DoE's SciTech Connect Database. Makes use of SciTech's XML Querying Service. Parses the resulting XML for information and outputs the result as Json in a format that works with the OSF scrapi (scraper API).

Example API query: http://www.osti.gov/scitech/scitechxml?EntryDateFrom=02%2F02%2F2015&page=0
"""


from __future__ import unicode_literals

import datetime

from lxml import etree

from dateutil.parser import *

from scrapi import requests

from scrapi.base import XMLHarvester
from scrapi.linter import RawDocument
from scrapi.base.schemas import DOESCHEMA

NAME = 'scitech'

DEFAULT_ENCODING = 'UTF-8'

record_encoding = None


class SciTechHarvester(XMLHarvester):
    file_format = 'xml'
    short_name = 'scitech'
    long_name = 'DoE\'s SciTech Connect Database'
    url = 'http://www.osti.gov/scitech'

    base_url = 'https://www.osti.gov/scitech/scitechxml'

    namespaces = {
        'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
        'dc': 'http://purl.org/dc/elements/1.1/',
        'dcq': 'http://purl.org/dc/terms/'
    }

    schema = DOESCHEMA

    def harvest(self, days_back=1):
        """A function for querying the SciTech Connect database for raw XML.
        The XML is chunked into smaller pieces, each representing data
        about an article/report. If there are multiple pages of results,
        this function iterates through all the pages."""

        return [
            RawDocument({
                'source': self.short_name,
                'filetype': self.file_format,
                'doc': etree.tostring(record),
                'docID': record.xpath('dc:ostiId/node()', namespaces=self.namespaces)[0].decode('utf-8'),
            })
            for record in self._fetch_records(days_back)
        ]

    def _fetch_records(self, days_back):
        page = 0
        morepages = True
        to_date = datetime.date.today().strftime('%m/%d/%Y')
        from_date = (datetime.date.today() - datetime.timedelta(days_back)).strftime('%m/%d/%Y')

        while morepages:
            resp = requests.get(self.base_url, params={
                'page': page,
                'EntryDateTo': to_date,
                'EntryDateFrom': from_date,
            })

            xml = etree.XML(resp.content)

            for record in xml.xpath('records/record'):
                yield record
<commit_msg>Add date range to scitech<commit_after>"""
A harvester for the DoE's SciTech Connect Database. Makes use of SciTech's XML Querying Service. Parses the resulting XML for information and outputs the result as Json in a format that works with the OSF scrapi (scraper API).

Example API query: http://www.osti.gov/scitech/scitechxml?EntryDateFrom=02%2F02%2F2015&page=0
"""


from __future__ import unicode_literals

from datetime import datetime

from lxml import etree

from dateutil.parser import *

from scrapi import requests

from scrapi.base import XMLHarvester
from scrapi.linter import RawDocument
from scrapi.base.schemas import DOESCHEMA

NAME = 'scitech'

DEFAULT_ENCODING = 'UTF-8'

record_encoding = None


class SciTechHarvester(XMLHarvester):
    file_format = 'xml'
    short_name = 'scitech'
    long_name = 'DoE\'s SciTech Connect Database'
    url = 'http://www.osti.gov/scitech'

    base_url = 'https://www.osti.gov/scitech/scitechxml'

    namespaces = {
        'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
        'dc': 'http://purl.org/dc/elements/1.1/',
        'dcq': 'http://purl.org/dc/terms/'
    }

    schema = DOESCHEMA

    def harvest(self, start_date=None, end_date=None):
        """A function for querying the SciTech Connect database for raw XML.
        The XML is chunked into smaller pieces, each representing data
        about an article/report. If there are multiple pages of results,
        this function iterates through all the pages."""

        return [
            RawDocument({
                'source': self.short_name,
                'filetype': self.file_format,
                'doc': etree.tostring(record),
                'docID': record.xpath('dc:ostiId/node()', namespaces=self.namespaces)[0].decode('utf-8'),
            })
            for record in self._fetch_records(start_date, end_date)
        ]

    def _fetch_records(self, start_date, end_date):
        page = 0
        morepages = True

        start_date = datetime.strptime(start_date, '%Y-%m-%d').date().strftime('%m/%d/%Y')
        end_date = datetime.strptime(end_date, '%Y-%m-%d').date().strftime('%m/%d/%Y')

        while morepages:
            resp = requests.get(self.base_url, params={
                'page': page,
                'EntryDateTo': end_date,
                'EntryDateFrom': start_date,
            })

            xml = etree.XML(resp.content)

            for record in xml.xpath('records/record'):
                yield record

{'fraction_non_alphanumeric': 0.11388775092175338, 'fraction_numerical': 0.010856206472757066, 'mean_word_length': 2.5461147421931734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 8, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363822', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1482, 'n_words': 388}