<commit_before>__author__ = 'faye'
from scrapi_tools.consumer import BaseConsumer, RawFile, NormalizedFile
import requests
import xmltodict
import json
import time
from datetime import date, timedelta
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import settings

TODAY = str(date.today()) + "T00:00:00Z"
YESTERDAY = str(date.today() - timedelta(4)) + "T00:00:00Z"
MAX_ROWS_PER_REQUEST = 999


class PLoSConsumer(BaseConsumer):

    def __init__(self):
        # Nothing to see here
        pass

    def consume(self):
        payload = {"api_key": settings.API_KEY, "rows": "0"}
        base_url = 'http://api.plos.org/search?q=publication_date:[{}%20TO%20{}]'.format(YESTERDAY, TODAY)
        plos_request = requests.get(base_url, params=payload)
        response = xmltodict.parse(plos_request.text)
        num_articles = int(response["response"]["result"]["@numFound"])

        start = 0
        rows = MAX_ROWS_PER_REQUEST
        doc_list = []

        while rows < num_articles + MAX_ROWS_PER_REQUEST:
            payload = {"api_key": settings.API_KEY, "rows": rows, "start": start}
            results = requests.get(base_url, params=payload)
            tick = time.time()

            doc = xmltodict.parse(results.text)

            full_response = doc["response"]["result"]["doc"]

            # TODO Incooporate "Correction" article type
            try:
                for result in full_response:
                    try:
                        if result["arr"][1]["@name"] == "abstract" and result["str"][3]["#text"] == "Research Article":
                            doc_list.append(RawFile({
                                'doc': json.dumps(result, indent=4, sort_keys=True),
                                'source': 'PLoS',
                                'doc_id': result["str"][0]["#text"],
                                'filetype': 'json',
                            }))
                    except KeyError:
                        pass

                start += MAX_ROWS_PER_REQUEST
                rows += MAX_ROWS_PER_REQUEST

                if time.time() - tick < 5:
                    time.sleep(5 - (time.time() - tick))
            except KeyError:
                print "No new files/updates!"
                break

        return doc_list

    def normalize(self, raw_doc, timestamp):
        raw_doc = raw_doc.get('doc')
        record = json.loads(raw_doc)

        return NormalizedFile({
            'title': record["str"][4]["#text"],
            'contributors': {'email': None, 'full_name': record["arr"][0]["str"]},
            'properties': {
                'description': record["arr"][1]["str"],
            },
            'meta': {},
            'id': record["str"][0]["#text"],
            'source': "PLoS",
            'timestamp': timestamp<commit_msg>Fixed error in building contributor list<commit_after>__author__ = 'faye'
from scrapi_tools.consumer import BaseConsumer, RawFile, NormalizedFile
import requests
import xmltodict
import json
import time
from datetime import date, timedelta
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import settings

TODAY = str(date.today()) + "T00:00:00Z"
YESTERDAY = str(date.today() - timedelta(4)) + "T00:00:00Z"
MAX_ROWS_PER_REQUEST = 999


class PLoSConsumer(BaseConsumer):

    def __init__(self):
        # Nothing to see here
        pass

    def consume(self):
        payload = {"api_key": settings.API_KEY, "rows": "0"}
        base_url = 'http://api.plos.org/search?q=publication_date:[{}%20TO%20{}]'.format(YESTERDAY, TODAY)
        plos_request = requests.get(base_url, params=payload)
        response = xmltodict.parse(plos_request.text)
        num_articles = int(response["response"]["result"]["@numFound"])

        start = 0
        rows = MAX_ROWS_PER_REQUEST
        doc_list = []

        while rows < num_articles + MAX_ROWS_PER_REQUEST:
            payload = {"api_key": settings.API_KEY, "rows": rows, "start": start}
            results = requests.get(base_url, params=payload)
            tick = time.time()

            doc = xmltodict.parse(results.text)

            full_response = doc["response"]["result"]["doc"]

            # TODO Incooporate "Correction" article type
            try:
                for result in full_response:
                    try:
                        if result["arr"][1]["@name"] == "abstract" and result["str"][3]["#text"] == "Research Article":
                            doc_list.append(RawFile({
                                'doc': json.dumps(result, indent=4, sort_keys=True),
                                'source': 'PLoS',
                                'doc_id': result["str"][0]["#text"],
                                'filetype': 'json',
                            }))
                    except KeyError:
                        pass

                start += MAX_ROWS_PER_REQUEST
                rows += MAX_ROWS_PER_REQUEST

                if time.time() - tick < 5:
                    time.sleep(5 - (time.time() - tick))
            except KeyError:
                print "No new files/updates!"
                break

        return doc_list

    def normalize(self, raw_doc, timestamp):
        raw_doc = raw_doc.get('doc')
        record = json.loads(raw_doc)

        return NormalizedFile({
            'title': record["str"][4]["#text"],
            'contributors': [{
                'email': None,
                'full_name': name,
            } for name in record['arr'][0]['str']],
            'properties': {
                'description': record["arr"][1]["str"],
            },
            'meta': {},
            'id': record["str"][0]["#text"],
            'source': "PLoS",
            'timestamp': timestamp
{'fraction_non_alphanumeric': 0.13369356806693394, 'fraction_numerical': 0.011504270524664458, 'mean_word_length': 1.849056603773585, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2618186', 'n_tokens_mistral': 1729, 'n_tokens_neox': 1631, 'n_words': 340}