<commit_before>import sqlalchemy as sa
from sunshine.database import db_session
from sunshine.models import Candidate, Candidacy, Committee, Officer, \
    D2Report, Receipt, Expenditure, Investment, candidate_committees
from sunshine.cache import cache, make_cache_key, CACHE_TIMEOUT
from flask import Blueprint, render_template, request, make_response
import json
from datetime import datetime, date
from collections import OrderedDict
from operator import attrgetter, itemgetter
from itertools import groupby
from string import punctuation
import re
import sqlalchemy as sa
from io import StringIO, BytesIO
import csv
import zipfile

api = Blueprint('api', __name__)

dthandler = lambda obj: obj.isoformat() if isinstance(obj, date) else None

operator_lookup = {
    'ge': '>=',
    'gt': '>',
    'le': '<=',
    'lt': '<'
}

def getSearchResults(term, 
                     table_name, 
                     q_params={}):
    
    engine = db_session.bind

    if table_name in ['receipts', 'expenditures', 'investments', 'officers']:
        
        if table_name in ['receipts', 'expenditures']:
            table_name = 'condensed_%s' % table_name

        result = ''' 
            SELECT *
            FROM (
              SELECT 
                c.name AS committee_name,
                r.*
              FROM {0} AS r
              JOIN committees AS c
                ON r.committee_id = c.id,
                   to_tsquery('english', :term) AS query
              WHERE query @@ r.search_name
            ) AS {0}
            WHERE 1=1
        '''.format(table_name)

    else:
        result = ''' 
            SELECT *
            FROM {0},
                 to_tsquery('english', :term) AS query
            WHERE query @@ search_name
        '''.format(table_name)
    
    if q_params:
        sa_table = sa.Table(table_name, 
                            sa.MetaData(), 
                            autoload=True, 
                            autoload_with=engine)

        valid_query, _, _, _  = make_query(sa_table, q_params)
        
        if valid_query:
            clauses = []
            for key in q_params.keys():
                try:
                    fieldname, operator = key.split('__')
                    operator = operator_lookup[operator]
                except ValueError:
                    fieldname = key
                    operator = '='
                clauses.append('%s %s :%s' % (fieldname, operator, key))
            
            result = '{0} AND {1}'.format(result, ' AND '.join(clauses))

            q_params['term'] = term

        else:
            raise ValueError
    else:
        q_params = {'term': term}

    punc = re.compile('[%s]' % re.escape(punctuation))
    term = punc.sub('', term)

    q_params['term'] = ' & '.join([t for t in term.split()])

    results = engine.execute(sa.text(result), **q_params)<commit_msg>Adding wildcard back into search<commit_after>import sqlalchemy as sa
from sunshine.database import db_session
from sunshine.models import Candidate, Candidacy, Committee, Officer, \
    D2Report, Receipt, Expenditure, Investment, candidate_committees
from sunshine.cache import cache, make_cache_key, CACHE_TIMEOUT
from flask import Blueprint, render_template, request, make_response
import json
from datetime import datetime, date
from collections import OrderedDict
from operator import attrgetter, itemgetter
from itertools import groupby
from string import punctuation
import re
import sqlalchemy as sa
from io import StringIO, BytesIO
import csv
import zipfile

api = Blueprint('api', __name__)

dthandler = lambda obj: obj.isoformat() if isinstance(obj, date) else None

operator_lookup = {
    'ge': '>=',
    'gt': '>',
    'le': '<=',
    'lt': '<'
}

def getSearchResults(term, 
                     table_name, 
                     q_params={}):
    
    engine = db_session.bind

    if table_name in ['receipts', 'expenditures', 'investments', 'officers']:
        
        if table_name in ['receipts', 'expenditures']:
            table_name = 'condensed_%s' % table_name

        result = ''' 
            SELECT *
            FROM (
              SELECT 
                c.name AS committee_name,
                r.*
              FROM {0} AS r
              JOIN committees AS c
                ON r.committee_id = c.id,
                   to_tsquery('english', :term) AS query
              WHERE query @@ r.search_name
            ) AS {0}
            WHERE 1=1
        '''.format(table_name)

    else:
        result = ''' 
            SELECT *
            FROM {0},
                 to_tsquery('english', :term) AS query
            WHERE query @@ search_name
        '''.format(table_name)
    
    if q_params:
        sa_table = sa.Table(table_name, 
                            sa.MetaData(), 
                            autoload=True, 
                            autoload_with=engine)

        valid_query, _, _, _  = make_query(sa_table, q_params)
        
        if valid_query:
            clauses = []
            for key in q_params.keys():
                try:
                    fieldname, operator = key.split('__')
                    operator = operator_lookup[operator]
                except ValueError:
                    fieldname = key
                    operator = '='
                clauses.append('%s %s :%s' % (fieldname, operator, key))
            
            result = '{0} AND {1}'.format(result, ' AND '.join(clauses))

            q_params['term'] = term

        else:
            raise ValueError
    else:
        q_params = {'term': term}

    punc = re.compile('[%s]' % re.escape(punctuation))
    term = punc.sub('', term)

    q_params['term'] = ' & '.join(['{0}:*'.format(t) for t in term.split()])

    results = engine.execute(sa.text(result), **q_params)
{'fraction_non_alphanumeric': 0.10396975425330812, 'fraction_numerical': 0.00292146416910122, 'mean_word_length': 1.7873563218390804, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6047474', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1610, 'n_words': 439}