<commit_before>## Consumer for the CrossRef metadata service
from __future__ import unicode_literals

import json

import requests
from datetime import date, timedelta

from nameparser import HumanName
from dateutil.parser import parse

from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

NAME = 'crossref'

DEFAULT_ENCODING = 'UTF-8'

record_encoding = None


def copy_to_unicode(element):
    encoding = record_encoding or DEFAULT_ENCODING
    element = ''.join(element)
    if isinstance(element, unicode):
        return element
    else:
        return unicode(element, encoding=encoding)


def consume(days_back=0):
    base_url = 'http://api.crossref.org/v1/works?filter=from-pub-date:{},until-pub-date:{}&rows=1000'
    start_date = date.today() - timedelta(days_back)
    url = base_url.format(str(start_date), str(date.today()))
    print(url)
    data = requests.get(url)
    doc = data.json()

    records = doc['message']['items']

    doc_list = []
    for record in records:
        doc_id = record['DOI'] or record['URL']
        doc_list.append(RawDocument({
            'doc': json.dumps(record),
            'source': NAME,
            'docID': doc_id,
            'filetype': 'xml'
        }))

    return doc_list


def get_contributors(doc):
    contributor_list = []
    contributor_dict_list = doc.get('author') or []
    full_names = []
    orcid = ''
    for entry in contributor_dict_list:
        full_name = '{} {}'.format(entry.get('given'), entry.get('family'))
        full_names.append(full_name)
        orcid = entry.get('ORCID') or ''
    for person in full_names:
        name = HumanName(person)
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,
            'family': name.last,
            'suffix': name.suffix,
            'email': '',<commit_msg>get rid of URL as posibility for doc id<commit_after>## Consumer for the CrossRef metadata service
from __future__ import unicode_literals

import json

import requests
from datetime import date, timedelta

from nameparser import HumanName
from dateutil.parser import parse

from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

NAME = 'crossref'

DEFAULT_ENCODING = 'UTF-8'

record_encoding = None


def copy_to_unicode(element):
    encoding = record_encoding or DEFAULT_ENCODING
    element = ''.join(element)
    if isinstance(element, unicode):
        return element
    else:
        return unicode(element, encoding=encoding)


def consume(days_back=0):
    base_url = 'http://api.crossref.org/v1/works?filter=from-pub-date:{},until-pub-date:{}&rows=1000'
    start_date = date.today() - timedelta(days_back)
    url = base_url.format(str(start_date), str(date.today()))
    print(url)
    data = requests.get(url)
    doc = data.json()

    records = doc['message']['items']

    doc_list = []
    for record in records:
        doc_id = record['DOI']
        doc_list.append(RawDocument({
            'doc': json.dumps(record),
            'source': NAME,
            'docID': doc_id,
            'filetype': 'xml'
        }))

    return doc_list


def get_contributors(doc):
    contributor_list = []
    contributor_dict_list = doc.get('author') or []
    full_names = []
    orcid = ''
    for entry in contributor_dict_list:
        full_name = '{} {}'.format(entry.get('given'), entry.get('family'))
        full_names.append(full_name)
        orcid = entry.get('ORCID') or ''
    for person in full_names:
        name = HumanName(person)
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,
            'family': name.last,
            'suffix': name.suffix,
            'email': '',
{'fraction_non_alphanumeric': 0.12163925867919603, 'fraction_numerical': 0.0036543983294179067, 'mean_word_length': 2.835835835835836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1771112', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1150, 'n_words': 274}