<commit_before>## this is a wsgi-compatible script, receiving request from browser and uWSGI

import re

## User beaker to save search (cache). See documentation on http://beaker.readthedocs.org/en/latest/caching.html
from beaker.cache import CacheManager
from beaker.util import parse_cache_config_options

cache_opts = {
    'cache.type': 'file',
    'cache.data_dir': '/tmp/cache/data',
    'cache.lock_dir': '/tmp/cache/lock'
}

cache = CacheManager(**parse_cache_config_options(cache_opts))

from webob import Request, Response

from clic.dickens.keywords import Keywords

def paramHandler(params):
    form = params
    print vars(form)
    ## IdxGroup 1-gram, 3-6-gram. IdxGroup is the same for test and ref (name testIdxGroup for now)
    testIdxGroup = form.get('testIdxGroup', 'idx') ## from url request: http://127.0.0.1:8080/?testIdxGroup=3gram-idx&testIdxMod=quote&testCollection[]=dickens&testCollection[]=dickens
 
    testIdxMod = form.get('testIdxMod', '')
    refIdxMod = form.get('refIdxMod', '')
    ## IdxNames assuming both Group (3-gram etc.) and Mod is selected
    testIdxName = "{0}-{1}".format(testIdxMod, testIdxGroup)
    refIdxName = "{0}-{1}".format(refIdxMod, testIdxGroup)
    
    args = []
    ## if no ngram is specified the index is specific to Mod. If Mod is not specified default to sentence idx
    if not re.match('\dgram-idx', testIdxGroup):
        if not testIdxMod == '':
            testIdxName = testIdxMod + '-idx'
            refIdxName = refIdxMod + '-idx'
        else:
            testIdxName = 'sentence-idx'
            refIdxName = 'sentence-idx'
    args.insert(0, testIdxName)
    args.insert(2, refIdxName)
    
    book_collection = []
    refbook_collection = []         
    
    ## run loop along the search parameters. each key represents search category
    for i, w in enumerate(form.keys()):
        if w == 'testCollection':
            book_collection.append(form.values()[i])
        if w == 'refCollection':
            refbook_collection.append(form.values()[i])
    
    args.insert(1, book_collection)
    args.insert(3, refbook_collection)    
    
    return args    

def application(env, start_response):
    req = Request(env)
    resp = Response()

    args = paramHandler(req.params)      

    resp.json =  fetchKeywords(args)
    
    return resp(env, start_response)

@cache.cache('keyword', expire=3600) ## expires after 3600 secs

def fetchKeywords(args):
    
    keyworder = Keywords()

    return keyworder.list_keywords(args[0], args[1], args[2], args[3])
    
    
    

    <commit_msg>modified the keywords file to wrap json output in object<commit_after>## this is a wsgi-compatible script, receiving request from browser and uWSGI

import re

## User beaker to save search (cache). See documentation on http://beaker.readthedocs.org/en/latest/caching.html
from beaker.cache import CacheManager
from beaker.util import parse_cache_config_options

cache_opts = {
    'cache.type': 'file',
    'cache.data_dir': '/tmp/cache/data',
    'cache.lock_dir': '/tmp/cache/lock'
}

cache = CacheManager(**parse_cache_config_options(cache_opts))

from webob import Request, Response

from clic.dickens.keywords import Keywords

def paramHandler(params):
    form = params
    print vars(form)
    ## IdxGroup 1-gram, 3-6-gram. IdxGroup is the same for test and ref (name testIdxGroup for now)
    testIdxGroup = form.get('testIdxGroup', 'idx') ## from url request: http://127.0.0.1:8080/?testIdxGroup=3gram-idx&testIdxMod=quote&testCollection[]=dickens&testCollection[]=dickens

    testIdxMod = form.get('testIdxMod', '')
    refIdxMod = form.get('refIdxMod', '')
    ## IdxNames assuming both Group (3-gram etc.) and Mod is selected
    testIdxName = "{0}-{1}".format(testIdxMod, testIdxGroup)
    refIdxName = "{0}-{1}".format(refIdxMod, testIdxGroup)

    args = []
    ## if no ngram is specified the index is specific to Mod. If Mod is not specified default to sentence idx
    if not re.match('\dgram-idx', testIdxGroup):
        if not testIdxMod == '':
            testIdxName = testIdxMod + '-idx'
            refIdxName = refIdxMod + '-idx'
        else:
            testIdxName = 'sentence-idx'
            refIdxName = 'sentence-idx'
    args.insert(0, testIdxName)
    args.insert(2, refIdxName)

    book_collection = []
    refbook_collection = []

    ## run loop along the search parameters. each key represents search category
    for i, w in enumerate(form.keys()):
        if w == 'testCollection':
            book_collection.append(form.values()[i])
        if w == 'refCollection':
            refbook_collection.append(form.values()[i])
        if re.match('^vol',w)
            book_collection.append(form.values()[i])
            break;

    args.insert(1, book_collection)
    args.insert(3, refbook_collection)

    return args

def application(env, start_response):
    req = Request(env)
    resp = Response()

    args = paramHandler(req.params)

    resp.json =  fetchKeywords(args)

    return resp(env, start_response)



@cache.cache('keyword', expire=3600) ## expires after 3600 secs
def fetchKeywords(args):

    keyworder = Keywords()

    keywords = keyworder.list_keywords(args[0], args[1], args[2], args[3])

    return {"keywords":keywords}
{'fraction_non_alphanumeric': 0.12041785375118709, 'fraction_numerical': 0.013295346628679962, 'mean_word_length': 3.236524537409493, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4304613', 'n_tokens_mistral': 1684, 'n_tokens_neox': 1621, 'n_words': 419}