<commit_before>from config import *

from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import mapper, sessionmaker

import json
from collections import OrderedDict
from stringhelp import listify
 
class CourseDB(object):
    pass
class ReviewDB(object):
    pass

#----------------------------------------------------------------------

def loadSession():
    """This function generates a session, with which you can make queries to the database."""    
    
    engine = create_engine(dbPath) #, echo=True)  #<<< let's you see the database commands in stdout
 
    metadata = MetaData(engine)
    
    courses = Table('courses', metadata, autoload=True)
    mapper(CourseDB, courses)
 
    reviews = Table('reviews',metadata, autoload=True)
    mapper(ReviewDB, reviews)
    
    Session = sessionmaker(bind=engine)
    session = Session()
    return session

def search_preprocess(session,args):
    if 'dept' in args:
        dept = args['dept'].upper()
    else:
        dept = None

    if 'keywords' in args:
        keywords = args['keywords'].lower().split()
    else:
        keywords = []

    if 'gen_eds' in args:
        gen_eds = listify(args['gen_eds'])
    else:
        gen_eds = []

    if len(keywords):
        sort = 'keyword'
    else:
        sort = 'alpha'

    if 'overqualifying_gen_eds' in args:
        overqualifying_gen_eds = eval(args['overqualifying_gen_eds'])
    else:
        overqualifying_gen_eds = True
    if overqualifying_gen_eds:
        if 'HBSSM' not in gen_eds and 'HB' in gen_eds:
            gen_eds.append('HBSSM')
        if 'HEPT' not in gen_eds and 'HE' in gen_eds:
            gen_eds.append('HEPT')
        if 'NWL' not in gen_eds and 'NWNL' in gen_eds:
            gen_eds.append('NWL')

    return search(session = session,
                  dept = dept,
                  keywords = keywords,
                  gen_eds = gen_eds,
                  sort = sort)

def search(session,dept=None, keywords=[], gen_eds=[], sort=None):
    res = session.query(CourseDB)

    if dept:
        res = res.filter(CourseDB.dept == dept)

    if gen_eds:
        new_res = None
        for gen_ed in gen_eds:
            only_str = gen_ed
            first_str = gen_ed + ',%' #this gen_ed starts list
            mid_str = '% ' + gen_ed + ',%' #in middle of lsit
            last_str = '% ' + gen_ed #at end of list
            gen_ed_only = res.filter(CourseDB.gen_eds.like(only_str))
            gen_ed_first = res.filter(CourseDB.gen_eds.like(first_str))
            gen_ed_mid = res.filter(CourseDB.gen_eds.like(mid_str))
            gen_ed_last = res.filter(CourseDB.gen_eds.like(last_str))
            temp_res = gen_ed_only.\
                       union(gen_ed_first).\
                       union(gen_ed_mid).\<commit_msg>split auto_search_overqualifying into three (auto_hbssm, auto_hbssm, and auto_nwl) and moved the code for handling them into a separate function. auto_nwl is by default false<commit_after>from config import *

from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import mapper, sessionmaker

import json
from collections import OrderedDict
from stringhelp import listify
 
class CourseDB(object):
    pass
class ReviewDB(object):
    pass

#----------------------------------------------------------------------

def loadSession():
    """This function generates a session, with which you can make queries to the database."""    
    
    engine = create_engine(dbPath) #, echo=True)  #<<< let's you see the database commands in stdout
 
    metadata = MetaData(engine)
    
    courses = Table('courses', metadata, autoload=True)
    mapper(CourseDB, courses)
 
    reviews = Table('reviews',metadata, autoload=True)
    mapper(ReviewDB, reviews)
    
    Session = sessionmaker(bind=engine)
    session = Session()
    return session

def search_preprocess(session,args):
    if 'dept' in args:
        dept = args['dept'].upper()
    else:
        dept = None

    if 'keywords' in args:
        keywords = args['keywords'].lower().split()
    else:
        keywords = []

    if 'gen_eds' in args:
        gen_eds = listify(args['gen_eds'])
    else:
        gen_eds = []
    auto_add_overqualifying(gen_eds,args)

    if len(keywords):
        sort = 'keyword'
    else:
        sort = 'alpha'

    return search(session = session,
                  dept = dept,
                  keywords = keywords,
                  gen_eds = gen_eds,
                  sort = sort)

def auto_add_overqualifying(gen_eds,args={}):
    if 'auto_hbssm' in args:
        auto_hbssm = eval(args['auto_hbssm'])
    else:
        auto_hbssm = True

    if 'auto_hept' in args:
        auto_hept = eval(args['auto_hept'])
    else:
        auto_hept = True

    if 'auto_nwl' in args:
        auto_nwl = eval(args['auto_nwl'])
    else:
        auto_nwl = False

    if auto_hbssm:
        if 'HBSSM' not in gen_eds and 'HB' in gen_eds:
            gen_eds.append('HBSSM')
    if auto_hept:
        if 'HEPT' not in gen_eds and 'HE' in gen_eds:
            gen_eds.append('HEPT')
    if auto_nwl:
        if 'NWL' not in gen_eds and 'NWNL' in gen_eds:
            gen_eds.append('NWL')

def search(session,dept=None, keywords=[], gen_eds=[], sort=None):
    res = session.query(CourseDB)

    if dept:
        res = res.filter(CourseDB.dept == dept)

    if gen_eds:
        new_res = None
        for gen_ed in gen_eds:
            only_str = gen_ed
            first_str = gen_ed + ',%' #this gen_ed starts list
            mid_str = '% ' + gen_ed + ',%' #in middle of lsit
            last_str = '% ' + gen_ed #at end of list
            gen_ed_only = res.filter(CourseDB.gen_eds.like(only_str))
            gen_ed_first = res.filter(CourseDB.gen_eds.like(first_str))
            gen_ed_mid = res.filter(CourseDB.gen_eds.like(mid_str))
            gen_ed_last = res.filter(CourseDB.gen_eds.like(last_str))
            temp_res = gen_ed_only.\
                       union(gen_ed_first).\
                       union(gen_ed_mid).\
{'fraction_non_alphanumeric': 0.12305399138787677, 'fraction_numerical': 0.0, 'mean_word_length': 2.153524804177546, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '285011', 'n_tokens_mistral': 1948, 'n_tokens_neox': 1873, 'n_words': 475}