<commit_before>import re
import urllib2
import datetime
import json
import itertools
from collections import defaultdict

from django.http import HttpResponse

from billy.core import db, feeds_db
from billy.models import Bill
from billy.core import settings
from billy.utils import find_bill, parse_param_dt

import pymongo

from piston.utils import rc
from piston.handler import BaseHandler, HandlerMetaClass

AT_LARGE = ['At-Large', 'Chairman']


_lower_fields = (settings.LEVEL_FIELD, 'chamber')


def _build_mongo_filter(request, keys, icase=True):
    _filter = {}
    keys = set(keys) - set(['fields'])

    for key in keys:
        value = request.GET.get(key)
        if value:
            if key in _lower_fields:
                _filter[key] = value.lower()
            elif key.endswith('__in'):
                values = value.split('|')
                _filter[key[:-4]] = values
            else:
                # We use regex queries to get case insensitive search - this
                # means they won't use any indexes for now. Real case
                # insensitive queries are coming eventually:
                # http://jira.mongodb.org/browse/SERVER-90
                _filter[key] = re.compile('^%s$' % value, re.IGNORECASE)

    return _filter


def _build_field_list(request, default_fields=None):
    # if 'fields' key is specified in request split it on comma<commit_msg>hack for bill_id w/ spaces<commit_after>import re
import urllib2
import datetime
import json
import itertools
from collections import defaultdict

from django.http import HttpResponse

from billy.core import db, feeds_db
from billy.models import Bill
from billy.core import settings
from billy.utils import find_bill, parse_param_dt, fix_bill_id

import pymongo

from piston.utils import rc
from piston.handler import BaseHandler, HandlerMetaClass

AT_LARGE = ['At-Large', 'Chairman']


_lower_fields = (settings.LEVEL_FIELD, 'chamber')


def _build_mongo_filter(request, keys, icase=True):
    _filter = {}
    keys = set(keys) - set(['fields'])

    for key in keys:
        value = request.GET.get(key)
        if value:
            if key in _lower_fields:
                _filter[key] = value.lower()
            elif key.endswith('__in'):
                values = value.split('|')
                _filter[key[:-4]] = values
            elif key == 'bill_id':
                _filter[key] = fix_bill_id(value.upper())
            else:
                # We use regex queries to get case insensitive search - this
                # means they won't use any indexes for now. Real case
                # insensitive queries are coming eventually:
                # http://jira.mongodb.org/browse/SERVER-90
                _filter[key] = re.compile('^%s$' % value, re.IGNORECASE)

    return _filter


def _build_field_list(request, default_fields=None):
    # if 'fields' key is specified in request split it on comma
{'fraction_non_alphanumeric': 0.08804385063377869, 'fraction_numerical': 0.0027406646111682084, 'mean_word_length': 2.591635916359164, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4790774', 'n_tokens_mistral': 887, 'n_tokens_neox': 830, 'n_words': 265}