<commit_before>import B64
import DQXDbTools
import DQXUtils



def response(returndata):
    mytablename=returndata['tbname']
    encodedquery=returndata['qry']

    databaseName=None
    if 'database' in returndata:
        databaseName = returndata['database']
    db = DQXDbTools.OpenDatabase(DQXDbTools.ParseCredentialInfo(returndata), databaseName)
    cur = db.cursor()

    whc=DQXDbTools.WhereClause()
    whc.ParameterPlaceHolder='%s'#NOTE!: MySQL PyODDBC seems to require this nonstardard coding
    whc.Decode(encodedquery)
    whc.CreateSelectStatement()


    #Determine total number of records
    sqlquery="SELECT COUNT(*) FROM {0}".format(mytablename)
    if len(whc.querystring_params) > 0:
        sqlquery += " WHERE {0}".format(whc.querystring_params)
    # DQXUtils.LogServer('   executing count query...')
    tm = DQXUtils.Timer()
    cur.execute(sqlquery, whc.queryparams)
    # DQXUtils.LogServer('   finished in {0}s'.format(tm.Elapsed()))
    returndata['TotalRecordCount'] = cur.fetchone()[0]

    cur.close()
    db.close()

    return returndata
<commit_msg>Limit table result set size determination<commit_after>import B64
import DQXDbTools
import DQXUtils



def response(returndata):
    mytablename=returndata['tbname']
    encodedquery=returndata['qry']

    maxrecordcount = 10000
    if 'maxrecordcount' in returndata:
        maxrecordcount = int(returndata['maxrecordcount'])

    databaseName=None
    if 'database' in returndata:
        databaseName = returndata['database']
    db = DQXDbTools.OpenDatabase(DQXDbTools.ParseCredentialInfo(returndata), databaseName)
    cur = db.cursor()

    whc=DQXDbTools.WhereClause()
    whc.ParameterPlaceHolder='%s'#NOTE!: MySQL PyODDBC seems to require this nonstardard coding
    whc.Decode(encodedquery)
    whc.CreateSelectStatement()


    #Determine total number of records
    sqlquery="SELECT COUNT(*) FROM (SELECT * FROM {0}".format(mytablename)
    if len(whc.querystring_params) > 0:
        sqlquery += " WHERE {0}".format(whc.querystring_params)
    sqlquery += ' LIMIT '+str(maxrecordcount)
    sqlquery += ') as tmp_table'
    # DQXUtils.LogServer('   executing count query...')
    tm = DQXUtils.Timer()
    cur.execute(sqlquery, whc.queryparams)
    # DQXUtils.LogServer('   finished in {0}s'.format(tm.Elapsed()))
    recordcount = cur.fetchone()[0]
    returndata['TotalRecordCount'] = recordcount
    if recordcount >= maxrecordcount:
        returndata['Truncated'] = True

    cur.close()
    db.close()

    return returndata

{'fraction_non_alphanumeric': 0.11615962070327934, 'fraction_numerical': 0.0075069142631371, 'mean_word_length': 3.984251968503937, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2334348', 'n_tokens_mistral': 882, 'n_tokens_neox': 791, 'n_words': 162}