<commit_before>import base64
from datetime import datetime
import hashlib
import hmac
import json
import urllib

from lxml import etree
import requests

from django.conf import settings
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import render

from ui.models import Database, Journal


# FIXME: make a local_setting
DEFAULT_HIT_COUNT = 3
RFC2616_DATEFORMAT = "%a, %d %b %Y %H:%M:%S GMT"


def home(request):
    q = request.GET.get('q', '')
    if q:
        aquabrowser_response = _aquabrowser_query(request)
        databases_response = _databases_query(request)
        summon_response = _summon_query(request)
        journals_response = _journals_query(request)
    params = {'title': 'home', 'q': q}
    if q:
        params['aquabrowser_response'] = aquabrowser_response
        params['databases_response'] = databases_response
        params['summon_response'] = summon_response
        params['journals_response'] = journals_response
    return render(request, 'home.html', params)


def _aquabrowser_query(request):
    q = request.GET.get('q', '')
    try:
        count = int(request.GET.get('count', DEFAULT_HIT_COUNT))
    except:
        count = DEFAULT_HIT_COUNT
    params = {'output': 'xml', 'q': q}
    # TODO: move url to settings
    r = requests.get('http://surveyor.gelman.gwu.edu/result.ashx',
                     params=params)
    root = etree.fromstring(r.text)
    matches = []
    records = root.findall('./results/record')
    for record in records[:count]:
        match = {}
        d = record.find('d')
        df245 = d.find('df245')
        title_subs = {'a': '', 'b': '', 'c': ''}
        for df245row in df245.findall("df245"):
            key = df245row.attrib['key']
            if key in ['a', 'b', 'c']:
                title_subs[key] = ' '.join(df245row.xpath('.//text()'))
        match['name'] = ' '.join([title_subs['a'], title_subs['b'],
                                  title_subs['c']])
        match['url'] = 'http://surveyor.gelman.gwu.edu/?hreciid=%s' % \
                       record.attrib['extID']
        # TODO: what should go here, if anything?
        match['description'] = ''
        matches.append(match)
    more_url = 'http://surveyor.gelman.gwu.edu/?q=%s' % q
    response = {'matches': matches, 'q': q, 'count_total': len(records),
                'more_url': more_url}
    return response


def aquabrowser_json(request):
    response = _aquabrowser_query(request)
    return HttpResponse(json.dumps(response, encoding='utf-8'),
                        content_type='application/json')


def aquabrowser_html(request):
    response = _aquabrowser_query(request)
    return render(request, 'aquabrowser.html', {'response': response})


def _databases_query(request):
    q = request.GET.get('q', '')
    try:
        count = int(request.GET.get('count', DEFAULT_HIT_COUNT))
    except:
        count = DEFAULT_HIT_COUNT
    response = {'q': q}
    if q:
        matches = []
        qs_databases = Database.objects.filter(Q(name__icontains=q) |
                                               Q(url__icontains=q) |
                                               Q(description__icontains=q))
        response['count_total'] = qs_databases.count()
        response['more_url'] = '%s%s' % (settings.DATABASES_MORE_URL, q)
        for db in qs_databases[:count]:
            match = {'name': db.name, 'url': db.url,
                     'description': db.description}
            matches.append(match)
        response['matches'] = matches
    return response


def databases_html(request):
    response = _databases_query(request)
    return render(request, 'databases.html', {'response': response})


def databases_json(request):
    response = _databases_query(request)
    return HttpResponse(json.dumps(response), content_type='application/json')


def _journals_query(request):
    q = request.GET.get('q', '')
    try:
        count = int(request.GET.get('count', DEFAULT_HIT_COUNT))
    except:
        count = DEFAULT_HIT_COUNT
    response = {'q': q}
    if q:
        matches = []
        qs_journals = Journal.objects.filter(Q(title__icontains=q))
        response['count_total'] = qs_journals.count()
        response['more_url'] = '%s%s' % (settings.JOURNALS_MORE_URL, q)
        for journal in qs_journals[:count]:
            url = "http://findit.library.gwu.edu/?V=1.0&N=100&L=UZ4UG4LZ9G&S=T_M&C=" + urllib.quote_plus(unicode(journal.title).encode('utf-8'))
            # url = "http://findit.library.gwu.edu/?V=1.0&N=100&L=UZ4UG4LZ9G&S=T_M&C=business"
            match = {'title': journal.title, 'ssid': journal.ssid,
                    'issn': journal.issn, 'eissn': journal.eissn, 'url': url}
            matches.append(match)
        response['matches'] = matches
    return response


def journals_html(request):
    response = _journals_query(request)<commit_msg>removed extraneous commented-out line<commit_after>import base64
from datetime import datetime
import hashlib
import hmac
import json
import urllib

from lxml import etree
import requests

from django.conf import settings
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import render

from ui.models import Database, Journal


# FIXME: make a local_setting
DEFAULT_HIT_COUNT = 3
RFC2616_DATEFORMAT = "%a, %d %b %Y %H:%M:%S GMT"


def home(request):
    q = request.GET.get('q', '')
    if q:
        aquabrowser_response = _aquabrowser_query(request)
        databases_response = _databases_query(request)
        summon_response = _summon_query(request)
        journals_response = _journals_query(request)
    params = {'title': 'home', 'q': q}
    if q:
        params['aquabrowser_response'] = aquabrowser_response
        params['databases_response'] = databases_response
        params['summon_response'] = summon_response
        params['journals_response'] = journals_response
    return render(request, 'home.html', params)


def _aquabrowser_query(request):
    q = request.GET.get('q', '')
    try:
        count = int(request.GET.get('count', DEFAULT_HIT_COUNT))
    except:
        count = DEFAULT_HIT_COUNT
    params = {'output': 'xml', 'q': q}
    # TODO: move url to settings
    r = requests.get('http://surveyor.gelman.gwu.edu/result.ashx',
                     params=params)
    root = etree.fromstring(r.text)
    matches = []
    records = root.findall('./results/record')
    for record in records[:count]:
        match = {}
        d = record.find('d')
        df245 = d.find('df245')
        title_subs = {'a': '', 'b': '', 'c': ''}
        for df245row in df245.findall("df245"):
            key = df245row.attrib['key']
            if key in ['a', 'b', 'c']:
                title_subs[key] = ' '.join(df245row.xpath('.//text()'))
        match['name'] = ' '.join([title_subs['a'], title_subs['b'],
                                  title_subs['c']])
        match['url'] = 'http://surveyor.gelman.gwu.edu/?hreciid=%s' % \
                       record.attrib['extID']
        # TODO: what should go here, if anything?
        match['description'] = ''
        matches.append(match)
    more_url = 'http://surveyor.gelman.gwu.edu/?q=%s' % q
    response = {'matches': matches, 'q': q, 'count_total': len(records),
                'more_url': more_url}
    return response


def aquabrowser_json(request):
    response = _aquabrowser_query(request)
    return HttpResponse(json.dumps(response, encoding='utf-8'),
                        content_type='application/json')


def aquabrowser_html(request):
    response = _aquabrowser_query(request)
    return render(request, 'aquabrowser.html', {'response': response})


def _databases_query(request):
    q = request.GET.get('q', '')
    try:
        count = int(request.GET.get('count', DEFAULT_HIT_COUNT))
    except:
        count = DEFAULT_HIT_COUNT
    response = {'q': q}
    if q:
        matches = []
        qs_databases = Database.objects.filter(Q(name__icontains=q) |
                                               Q(url__icontains=q) |
                                               Q(description__icontains=q))
        response['count_total'] = qs_databases.count()
        response['more_url'] = '%s%s' % (settings.DATABASES_MORE_URL, q)
        for db in qs_databases[:count]:
            match = {'name': db.name, 'url': db.url,
                     'description': db.description}
            matches.append(match)
        response['matches'] = matches
    return response


def databases_html(request):
    response = _databases_query(request)
    return render(request, 'databases.html', {'response': response})


def databases_json(request):
    response = _databases_query(request)
    return HttpResponse(json.dumps(response), content_type='application/json')


def _journals_query(request):
    q = request.GET.get('q', '')
    try:
        count = int(request.GET.get('count', DEFAULT_HIT_COUNT))
    except:
        count = DEFAULT_HIT_COUNT
    response = {'q': q}
    if q:
        matches = []
        qs_journals = Journal.objects.filter(Q(title__icontains=q))
        response['count_total'] = qs_journals.count()
        response['more_url'] = '%s%s' % (settings.JOURNALS_MORE_URL, q)
        for journal in qs_journals[:count]:
            url = "http://findit.library.gwu.edu/?V=1.0&N=100&L=UZ4UG4LZ9G&S=T_M&C=" + urllib.quote_plus(unicode(journal.title).encode('utf-8'))
            match = {'title': journal.title, 'ssid': journal.ssid,
                    'issn': journal.issn, 'eissn': journal.eissn, 'url': url}
            matches.append(match)
        response['matches'] = matches
    return response


def journals_html(request):
    response = _journals_query(request)
{'fraction_non_alphanumeric': 0.138657621543838, 'fraction_numerical': 0.008633980881899476, 'mean_word_length': 2.7109077040427154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2706111', 'n_tokens_mistral': 3147, 'n_tokens_neox': 3022, 'n_words': 606}