<commit_before>'''
Created on Nov 1, 2014

@author: ehenneken
'''
from flask import current_app
from flask import request
import sys
import os
import urllib
import simplejson as json
from client import client


def get_data(**args):
    """
    Get the references for a set of bibcodes
    """
    references = []
    citations = []
    # This information can be retrieved with one single Solr query
    # (just an 'OR' query of a list of bibcodes)
    # To restrict the size of the query URL, we split the list of
    # bibcodes up in a list of smaller lists
    biblists = list(
        chunks(args['bibcodes'],
               current_app.config.get('CITATION_HELPER_CHUNK_SIZE')))
    for biblist in biblists:
        q = " OR ".join(map(lambda a: "bibcode:%s" % a, biblist))
        # Get the information from Solr
        # We only need the contents of the 'reference' field (i.e. the
        # list of bibcodes referenced by the paper at hand)
        headers = {'Authorization':
                   request.headers.get('X-Forwarded-Authorization', request.headers.get('Authorization', ''))}
        params = {'wt': 'json', 'q': q, 'fl': 'reference,citation',
                  'rows': current_app.config['CITATION_HELPER_MAX_HITS']}
        response = client().get(
            current_app.config.get('CITATION_HELPER_SOLR_PATH'),
            params=params, headers=headers)
        if response.status_code != 200:
            return {"Error": "Unable to get results!",
                    "Error Info": "Solr response: %s" % str(response.text),
                    "Status Code": response.status_code}
        resp = response.json()
        # Collect all bibcodes in a list (do NOT remove multiplicity)
        for doc in resp['response']['docs']:
            if 'reference' in doc:
                references += doc['reference']
            if 'citation' in doc:
                citations += doc['citation']
    return [bib for bib in references + citations]


def get_meta_data(**args):
    """
    Get the meta data for a set of bibcodes
    """
    data_dict = {}
    # This information can be retrieved with one single Solr query
    # (just an 'OR' query of a list of bibcodes)
    bibcodes = [bibcode for (bibcode, score) in args['results']]
    list = " OR ".join(map(lambda a: "bibcode:%s" % a, bibcodes))
    q = '%s' % list
    # Get the information from Solr
    headers = {'X-Forwarded-Authorization':
               request.headers.get('Authorization')}
    params = {'wt': 'json', 'q': q, 'fl': 'bibcode,title,first_author',
              'rows': current_app.config.get('CITATION_HELPER_MAX_HITS')}
    response = client().get(
        current_app.config.get('CITATION_HELPER_SOLR_PATH'), params=params,
        headers=headers)
    if response.status_code != 200:
        return {"Error": "Unable to get results!",
                "Error Info": response.text,
                "Status Code": response.status_code}
    resp = response.json()
    # Collect meta data
    for doc in resp['response']['docs']:
        title = 'NA'<commit_msg>Forgotten to update all functions that make request<commit_after>'''
Created on Nov 1, 2014

@author: ehenneken
'''
from flask import current_app
from flask import request
import sys
import os
import urllib
import simplejson as json
from client import client


def get_data(**args):
    """
    Get the references for a set of bibcodes
    """
    references = []
    citations = []
    # This information can be retrieved with one single Solr query
    # (just an 'OR' query of a list of bibcodes)
    # To restrict the size of the query URL, we split the list of
    # bibcodes up in a list of smaller lists
    biblists = list(
        chunks(args['bibcodes'],
               current_app.config.get('CITATION_HELPER_CHUNK_SIZE')))
    for biblist in biblists:
        q = " OR ".join(map(lambda a: "bibcode:%s" % a, biblist))
        # Get the information from Solr
        # We only need the contents of the 'reference' field (i.e. the
        # list of bibcodes referenced by the paper at hand)
        headers = {'Authorization':
                   request.headers.get('X-Forwarded-Authorization', request.headers.get('Authorization', ''))}
        params = {'wt': 'json', 'q': q, 'fl': 'reference,citation',
                  'rows': current_app.config['CITATION_HELPER_MAX_HITS']}
        response = client().get(
            current_app.config.get('CITATION_HELPER_SOLR_PATH'),
            params=params, headers=headers)
        if response.status_code != 200:
            return {"Error": "Unable to get results!",
                    "Error Info": "Solr response: %s" % str(response.text),
                    "Status Code": response.status_code}
        resp = response.json()
        # Collect all bibcodes in a list (do NOT remove multiplicity)
        for doc in resp['response']['docs']:
            if 'reference' in doc:
                references += doc['reference']
            if 'citation' in doc:
                citations += doc['citation']
    return [bib for bib in references + citations]


def get_meta_data(**args):
    """
    Get the meta data for a set of bibcodes
    """
    data_dict = {}
    # This information can be retrieved with one single Solr query
    # (just an 'OR' query of a list of bibcodes)
    bibcodes = [bibcode for (bibcode, score) in args['results']]
    list = " OR ".join(map(lambda a: "bibcode:%s" % a, bibcodes))
    q = '%s' % list
    # Get the information from Solr
    headers = {'Authorization':
               request.headers.get('X-Forwarded-Authorization', request.headers.get('Authorization', ''))}
    params = {'wt': 'json', 'q': q, 'fl': 'bibcode,title,first_author',
              'rows': current_app.config.get('CITATION_HELPER_MAX_HITS')}
    response = client().get(
        current_app.config.get('CITATION_HELPER_SOLR_PATH'), params=params,
        headers=headers)
    if response.status_code != 200:
        return {"Error": "Unable to get results!",
                "Error Info": response.text,
                "Status Code": response.status_code}
    resp = response.json()
    # Collect meta data
    for doc in resp['response']['docs']:
        title = 'NA'
{'fraction_non_alphanumeric': 0.11870967741935484, 'fraction_numerical': 0.0035483870967741938, 'mean_word_length': 2.5637931034482757, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2953733', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1749, 'n_words': 585}