<commit_before>"""
Open Science Framework harvester of public projects for the SHARE Notification Service

Example API query: https://osf.io/api/v1/search/
https://staging.osf.io/api/v1/search/?q=category:registration%20AND%20date_created:[2015-01-01%20TO%202015-03-10]&size=1000
https://osf.io/api/v1/search/?q=category:registration%20AND%20NOT%20title=test%20AND%20NOT%20title=%22Test%20Project%22
"""

from __future__ import unicode_literals

import json
import logging
from dateutil.parser import parse
from datetime import date, timedelta

from nameparser import HumanName

from scrapi import requests
from scrapi.base import JSONHarvester
from scrapi.linter.document import RawDocument
from scrapi.base.helpers import build_properties

logger = logging.getLogger(__name__)


def process_contributors(authors):

    contributor_list = []
    for person in authors:
        name = HumanName(person['fullname'])
        contributor = {
            'name': person['fullname'],
            'givenName': name.first,
            'additionalName': name.middle,
            'familyName': name.last,
            'email': '',
            'sameAs': [],
        }
        contributor_list.append(contributor)

    return contributor_list


def process_null(entry):
    if entry is None:
        return ''
    else:
        return entry


def process_tags(entry):
    if isinstance(entry, list):
        return entry
    else:
        return [entry]


def parse_date(entry):
    return parse(entry).date().isoformat().decode('utf-8')


class OSFHarvester(JSONHarvester):
    short_name = 'osf'
    long_name = 'Open Science Framework'
    url = 'http://osf.io/api/v1/search/'
    count = 0

    # Only registrations that aren't just the word "test" or "test project"
    URL = 'https://osf.io/api/v1/search/?q=category:registration ' +\
          ' AND date_created:[{} TO {}]' +\
          ' AND NOT title=test AND NOT title="Test Project"&size=1000'

    @property
    def schema(self):
        return {
            'contributors': ('/contributors', process_contributors),
            'title': ('/title', process_null),
            'providerUpdatedDateTime': ('date_created', parse_date),
            'description': ('/description', process_null),
            'uris': {
                'canonicalUri': ('/url', lambda x: 'http://osf.io' + x),
            },
            'tags': ('tags', process_tags),
            'otherProperties': build_properties(
                ('parent_title', 'parent_title'),
                ('category', 'category'),
                ('wiki_link', 'wiki_link'),
                ('is_component', 'is_component'),
                ('is_registration', 'is_registration'),
                ('parent_url', 'parent_url'),
                ('contributors', 'contributors'),
                ('journal Id', '/journal Id'),
                ('tags', ('tags', process_tags))
            )
        }

    def harvest(self, start_date=None, end_date=None):

        start_date = start_date or date.today() - timedelta(1)
        end_date = end_date or date.today()

        search_url = self.URL.format(start_date.isoformat(), end_date.isoformat())
        records = self.get_records(search_url)
<commit_msg>update osf with settings days back<commit_after>"""
Open Science Framework harvester of public projects for the SHARE Notification Service

Example API query: https://osf.io/api/v1/search/
https://staging.osf.io/api/v1/search/?q=category:registration%20AND%20date_created:[2015-01-01%20TO%202015-03-10]&size=1000
https://osf.io/api/v1/search/?q=category:registration%20AND%20NOT%20title=test%20AND%20NOT%20title=%22Test%20Project%22
"""

from __future__ import unicode_literals

import json
import logging
from dateutil.parser import parse
from datetime import date, timedelta

from nameparser import HumanName

from scrapi import requests
from scrapi import settings
from scrapi.base import JSONHarvester
from scrapi.linter.document import RawDocument
from scrapi.base.helpers import build_properties

logger = logging.getLogger(__name__)


def process_contributors(authors):

    contributor_list = []
    for person in authors:
        name = HumanName(person['fullname'])
        contributor = {
            'name': person['fullname'],
            'givenName': name.first,
            'additionalName': name.middle,
            'familyName': name.last,
            'email': '',
            'sameAs': [],
        }
        contributor_list.append(contributor)

    return contributor_list


def process_null(entry):
    if entry is None:
        return ''
    else:
        return entry


def process_tags(entry):
    if isinstance(entry, list):
        return entry
    else:
        return [entry]


def parse_date(entry):
    return parse(entry).date().isoformat().decode('utf-8')


class OSFHarvester(JSONHarvester):
    short_name = 'osf'
    long_name = 'Open Science Framework'
    url = 'http://osf.io/api/v1/search/'
    count = 0

    # Only registrations that aren't just the word "test" or "test project"
    URL = 'https://osf.io/api/v1/search/?q=category:registration ' +\
          ' AND date_created:[{} TO {}]' +\
          ' AND NOT title=test AND NOT title="Test Project"&size=1000'

    @property
    def schema(self):
        return {
            'contributors': ('/contributors', process_contributors),
            'title': ('/title', process_null),
            'providerUpdatedDateTime': ('date_created', parse_date),
            'description': ('/description', process_null),
            'uris': {
                'canonicalUri': ('/url', lambda x: 'http://osf.io' + x),
            },
            'tags': ('tags', process_tags),
            'otherProperties': build_properties(
                ('parent_title', 'parent_title'),
                ('category', 'category'),
                ('wiki_link', 'wiki_link'),
                ('is_component', 'is_component'),
                ('is_registration', 'is_registration'),
                ('parent_url', 'parent_url'),
                ('contributors', 'contributors'),
                ('journal Id', '/journal Id'),
                ('tags', ('tags', process_tags))
            )
        }

    def harvest(self, start_date=None, end_date=None):

        start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
        end_date = end_date or date.today()

        search_url = self.URL.format(start_date.isoformat(), end_date.isoformat())
        records = self.get_records(search_url)

{'fraction_non_alphanumeric': 0.12935553499845823, 'fraction_numerical': 0.01773049645390071, 'mean_word_length': 2.675354107648725, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2232466', 'n_tokens_mistral': 2004, 'n_tokens_neox': 1841, 'n_words': 423}