<commit_before>from collections import defaultdict
import datetime
import itertools
import json
import re
from operator import itemgetter
import copy

from dateutil.parser import parse as dateparse

from cwod.utils import flatten_param_dicts
from django.conf import settings
from django.contrib import messages
from django.contrib.localflavor.us.us_states import US_STATES, STATE_CHOICES, US_TERRITORIES
from django.contrib.auth.decorators import login_required
from django.core.mail import send_mail
from django.core.urlresolvers import reverse, NoReverseMatch
from django.db import connections, DatabaseError
from django.http import Http404, HttpResponse, HttpResponsePermanentRedirect
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.views.generic.simple import direct_to_template

from bioguide.models import *
from capitolwords import capitolwords, ApiError
from ngrams.models import *
from cwod.models import *

from baseconv import base62

TERRITORY_CHOICES = ('PR', 'GU', 'MP', 'AS')

capitolwords = capitolwords(api_key=settings.SUNLIGHT_API_KEY, domain=settings.API_ROOT)

congresses = LegislatorRole.objects.filter(congress__gte=104).values_list('congress', flat=True).distinct().order_by('-congress')

# These are used in a regular expression so must be escaped.
# From NLTK
STOPWORDS = ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you',
             'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his',
             'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself',
             'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which',
             'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are',
             'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having',
             'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if',
             'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for',
             'with', 'about', 'against', 'between', 'into', 'through', 'during',
             'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in',
             'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once',
             'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each',
             'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own',
             'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don',
             'should', 'now', '\.', '\?', '\!' ]

PUNCTUATION = ['?', '!', '.', ',']


def index(request):
    recent_entries = RecentEntry.objects.all()[:10]

    return render_to_response('cwod/index.html',
                              {'recent_entries': recent_entries,
                              }, context_instance=RequestContext(request))

    #return direct_to_template(request, template='index.html')


def faster_term_detail(request, term):
    # For better URLs, replace spaces with underscores and strip trailing punctuation
    if re.search(r'\s', term) or re.search(r'[%s]$' % ''.join(PUNCTUATION), term):
        term = re.sub(r'[%s]$' % ''.join(PUNCTUATION), '', term)
        term = re.sub(r'\s', '_', term.strip())
        url = reverse('cwod_term_detail', kwargs={'term': term})
        return HttpResponsePermanentRedirect(url)

    if request.GET.get('js') == 'false':
        return term_detail(request, term)

    term = re.sub(r'_', ' ', term)

    # Recent entries
    all_entries = capitolwords.text(phrase=term, bioguide_id="['' TO *]", sort='date desc,score desc', per_page=50)

    # Only show one entry for each Congressional Record page.
    entries = []
    urls = []
    for entry in all_entries:
        if entry['origin_url'] in urls:
            continue
        urls.append(entry['origin_url'])
        entries.append(entry)

    entries = _highlight_entries(entries, term)

    uri = request.build_absolute_uri()
    if '?' in uri:
        no_js_uri = uri + '&js=false'
    else:
        no_js_uri = uri + '?js=false'

    return render_to_response('cwod/term_detail.html',
                              {'term': term,
                               'entries': entries,
                               'needs_js': True,
                               'no_js_uri': no_js_uri,
                               'state_choices': US_STATES + US_TERRITORIES,
                              }, context_instance=RequestContext(request))


def term_detail(request, term):

    # For better URLs, replace spaces with underscores & strip trailing punctuation
    if re.search(r'\s', term) or re.search(r'[%s]$' % ''.join(PUNCTUATION), term):<commit_msg>strip leading and trailing quotes from searches<commit_after>from collections import defaultdict
import datetime
import itertools
import json
import re
from operator import itemgetter
import copy

from dateutil.parser import parse as dateparse

from cwod.utils import flatten_param_dicts
from django.conf import settings
from django.contrib import messages
from django.contrib.localflavor.us.us_states import US_STATES, STATE_CHOICES, US_TERRITORIES
from django.contrib.auth.decorators import login_required
from django.core.mail import send_mail
from django.core.urlresolvers import reverse, NoReverseMatch
from django.db import connections, DatabaseError
from django.http import Http404, HttpResponse, HttpResponsePermanentRedirect
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.views.generic.simple import direct_to_template

from bioguide.models import *
from capitolwords import capitolwords, ApiError
from ngrams.models import *
from cwod.models import *

from baseconv import base62

TERRITORY_CHOICES = ('PR', 'GU', 'MP', 'AS')

capitolwords = capitolwords(api_key=settings.SUNLIGHT_API_KEY, domain=settings.API_ROOT)

congresses = LegislatorRole.objects.filter(congress__gte=104).values_list('congress', flat=True).distinct().order_by('-congress')

# These are used in a regular expression so must be escaped.
# From NLTK
STOPWORDS = ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you',
             'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his',
             'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself',
             'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which',
             'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are',
             'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having',
             'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if',
             'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for',
             'with', 'about', 'against', 'between', 'into', 'through', 'during',
             'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in',
             'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once',
             'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each',
             'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own',
             'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don',
             'should', 'now', '\.', '\?', '\!' ]

PUNCTUATION = ['?', '!', '.', ',']


def index(request):
    recent_entries = RecentEntry.objects.all()[:10]

    return render_to_response('cwod/index.html',
                              {'recent_entries': recent_entries,
                              }, context_instance=RequestContext(request))

    #return direct_to_template(request, template='index.html')


def faster_term_detail(request, term):
    # For better URLs, replace spaces with underscores and strip trailing punctuation
    term = term.strip('"\' ')
    if re.search(r'\s', term) or re.search(r'[%s]$' % ''.join(PUNCTUATION), term):
        term = re.sub(r'[%s]$' % ''.join(PUNCTUATION), '', term)
        term = re.sub(r'\s', '_', term.strip())
        url = reverse('cwod_term_detail', kwargs={'term': term})
        return HttpResponsePermanentRedirect(url)

    if request.GET.get('js') == 'false':
        return term_detail(request, term)

    term = re.sub(r'_', ' ', term)

    # Recent entries
    all_entries = capitolwords.text(phrase=term, bioguide_id="['' TO *]", sort='date desc,score desc', per_page=50)

    # Only show one entry for each Congressional Record page.
    entries = []
    urls = []
    for entry in all_entries:
        if entry['origin_url'] in urls:
            continue
        urls.append(entry['origin_url'])
        entries.append(entry)

    entries = _highlight_entries(entries, term)

    uri = request.build_absolute_uri()
    if '?' in uri:
        no_js_uri = uri + '&js=false'
    else:
        no_js_uri = uri + '?js=false'

    return render_to_response('cwod/term_detail.html',
                              {'term': term,
                               'entries': entries,
                               'needs_js': True,
                               'no_js_uri': no_js_uri,
                               'state_choices': US_STATES + US_TERRITORIES,
                              }, context_instance=RequestContext(request))


def term_detail(request, term):

    # For better URLs, replace spaces with underscores & strip trailing punctuation,
    # as well as bordering quotation marks
    term = term.strip('"\' ')
    if re.search(r'\s', term) or re.search(r'[%s]$' % ''.join(PUNCTUATION), term):
{'fraction_non_alphanumeric': 0.16501997056968679, 'fraction_numerical': 0.002522598276224511, 'mean_word_length': 3.1550218340611353, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3913434', 'n_tokens_mistral': 3058, 'n_tokens_neox': 2957, 'n_words': 785}