<commit_before>''' Consumer for figshare '''
from __future__ import unicode_literals


import os
import xml
import time
import json
from dateutil.parser import parse
from datetime import date, timedelta, datetime

import requests


from nameparser import HumanName

from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

NAME = 'figshare'
URL = 'http://api.figshare.com/v1/articles/search?search_for=*&from_date='


def consume(days_back=0):
    start_date = date.today() - timedelta(days_back) - timedelta(1)
    end_date = date.today() - timedelta(1)
    search_url = '{0}{1}-{2}-{3}&end_date={4}-{5}-{6}'.format(URL,
                                         start_date.year,
                                         start_date.month,
                                         start_date.day,
                                         end_date.year,
                                         end_date.month,
                                         end_date.day)
    import pdb; pdb.set_trace()
    print search_url

    records = get_records(search_url)

    record_list = []
    for record in records:
        doc_id = record['article_id']

        record_list.append(
            RawDocument(
                {
                    'doc': json.dumps(record),
                    'source': NAME,
                    'docID': unicode(doc_id),
                    'filetype': 'json'
                }
            )
        )

    return record_list


def get_records(search_url):
    records = requests.get(search_url)
    total_records = records.json()['items_found']
    page = 1
    # full_records = requests.get(article_url + str(page))
    all_records = []
    while len(all_records) < total_records:
        record_list = records.json()['items']

        for record in record_list:
            if len(all_records) < total_records:
                all_records.append(record)

        page += 1
        records = requests.get(search_url + '&page='.format(str(page)))
        time.sleep(3)

    return all_records


def get_contributors(record):

    authors = record['authors']

    contributor_list = []
    for person in authors:
        name = HumanName(person['author_name'])
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,<commit_msg>minor changes to search only harvester<commit_after>''' Consumer for figshare '''
from __future__ import unicode_literals


import os
import xml
import time
import json
from dateutil.parser import parse
from datetime import date, timedelta, datetime

import requests


from nameparser import HumanName

from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

NAME = 'figshare'
URL = 'http://api.figshare.com/v1/articles/search?search_for=*&from_date='


def consume(days_back=0):
    start_date = date.today() - timedelta(days_back) - timedelta(1)
    end_date = date.today() - timedelta(1)
    search_url = '{0}{1}-{2}-{3}&end_date={4}-{5}-{6}'.format(URL,
                                         start_date.year,
                                         start_date.month,
                                         start_date.day,
                                         end_date.year,
                                         end_date.month,
                                         end_date.day)

    records = get_records(search_url)

    record_list = []
    for record in records:
        doc_id = record['article_id']

        record_list.append(
            RawDocument(
                {
                    'doc': json.dumps(record),
                    'source': NAME,
                    'docID': unicode(doc_id),
                    'filetype': 'json'
                }
            )
        )

    return record_list


def get_records(search_url):
    records = requests.get(search_url)
    total_records = records.json()['items_found']
    page = 1

    all_records = []
    while len(all_records) < total_records:
        record_list = records.json()['items']

        for record in record_list:
            if len(all_records) < total_records:
                all_records.append(record)

        page += 1
        records = requests.get(search_url + '&page={}'.format(str(page)))
        time.sleep(3)

    return all_records


def get_contributors(record):

    authors = record['authors']

    contributor_list = []
    for person in authors:
        name = HumanName(person['author_name'])
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,
{'fraction_non_alphanumeric': 0.1016803102111159, 'fraction_numerical': 0.006031882809133994, 'mean_word_length': 1.7936221419975933, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4108181', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1243, 'n_words': 274}