# Path: appAux.py
# def update_progress(progress, text="Progress", logmessage=False):
#     if not stdout_redirect():
#         oh = sys.stdout
#     else:
#         oh = sys.stderr
# 
#     barLength = 25  # Modify this to change the length of the progress bar
#     status = ""
#     if isinstance(progress, int):
#         progress = float(progress)
#     if not isinstance(progress, float):
#         progress = 0
#         status = "error: progress var must be float\r\n"
#     if progress < 0:
#         progress = 0
#         status = "Halt...\r\n"
#     if progress > 1:
#         status = "Progress out of bounds...\r\n"
#     if progress == 1:
#         progress = 1
#         oh.write('\x1b[2K\r')
#         oh.flush()
#         return ""
#     block = int(round(barLength * progress))
#     if logmessage:
#         text = "{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
#         return text
#     else:
#         text = "\r{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
#         oh.write(text)
#         oh.flush()
# 
# def outputcolum(data):
#     # Calculate terminal size
#     maxStrLength = getTerminalWidth()
#     if settings.rawOutput or stdout_redirect(): maxStrLength = 1000
# 
#     # Calculate number of fields and maxStrLengthField
#     # todo: grab max size per field to do a more intelligent distribution of available term realestate
#     num_fields = len(data[len(data)-1][1])
#     maxStrLengthField = maxStrLength / num_fields
# 
#     # Truncate fields
#     # todo: There has to be a better approach to this
#     for ii in xrange(0, len(data)):
#         tmpList = []
#         for i in xrange(0, len(data[ii][1])):
#             tmpList.append(str(data[ii][1][i])[:maxStrLengthField] + (str(data[ii][1][i])[maxStrLengthField:] and '...'))
#         t1 = []
#         t1.append(data[ii][0])
#         t1.append(tuple(tmpList))
#         data[ii] = t1
#         data[ii][1] = tuple(tmpList)
#     widths = [max(map(len,map(str, col))) for col in zip(*[x[1] for x in data])]
#     for row in data:
#         if settings.rawOutput or stdout_redirect():
#             print ("  ".join((str(val).ljust(width) for val, width in zip(row[1], widths))))
#         else: print colored("  ".join((str(val).ljust(width) for val, width in zip(row[1], widths))), row[0])
#     return data
# 
# def file_size(file_name_fullpath):
#     if ".zip/" in file_name_fullpath:
#         file_object = loadFile(file_name_fullpath)
#         file_object.seek(0, os.SEEK_END)
#         return file_object.tell()
#     else: return os.path.getsize(file_name_fullpath)


import settings
import logging
import multiprocessing
import collections
import os
import re
import time
import ntpath
import appDB
import hashlib
import operator
import glob
from contextlib import closing
from appAux import update_progress, outputcolum, file_size
from datetime import timedelta
from collections import namedtuple
from namedlist import namedlist
        from itertools import islice
                for rowID in rowID_list:
                    # Grab entry data we want to save to the output file:
                    record = retrieveSearchData(rowID, self.DB, self.search_space)

                    # De-dup results:
                    entryMD5 = hashlib.md5(''.join([str(e) for e in [record[0],record[1],record[2],record[3],record[4],record[5],record[9]]])).hexdigest()
                    if entryMD5 in dumped_set:
                        # print("Suppressing row %d" % entry[6])
                        with self.num_hits_suppressed.get_lock():
                            self.num_hits_suppressed.value += 1
                    else:
                        dumped_set.add(entryMD5)
                        # Re-filter against known bad individually to build histogram and highlight
                        regex_hit_name = None
                        search_space = None
                        if self.searchType == 'KNOWNBAD':
                            # Search for known_bad one by one and filter if required
                            for x in list(known_bad_search_terms):
                                if re.compile(x.regex, re.IGNORECASE).search(str(record.Search_Space)) is not None:
                                    if x.filter is not None:
                                        if re.compile(x.filter, re.IGNORECASE).search(str(record.Search_Space)) is not None:
                                            regex_hit_name = x.name
                                            continue
                                    # 'u200b' is a zero width unicode character I have to use to avoid messy markdown highlighting:
                                    search_space = re.compile('(.*)('+x.regex+')(.*)', re.I).sub(r'\1'+u'\u200b'+r'**'+u'\u200b'+r'\2'+u'\u200b'+'**'+u'\u200b'+r'\3', record.Search_Space, re.IGNORECASE)
                                    # Add hit to know_bad hit counter:
                                    regex_hit_name = x.name
                                    hit_dict[x.regex][0] += 1

                                    # We only report the match with the first regex from our set

# Path: appAux.py
# def update_progress(progress, text="Progress", logmessage=False):
#     if not stdout_redirect():
#         oh = sys.stdout
#     else:
#         oh = sys.stderr
# 
#     barLength = 25  # Modify this to change the length of the progress bar
#     status = ""
#     if isinstance(progress, int):
#         progress = float(progress)
#     if not isinstance(progress, float):
#         progress = 0
#         status = "error: progress var must be float\r\n"
#     if progress < 0:
#         progress = 0
#         status = "Halt...\r\n"
#     if progress > 1:
#         status = "Progress out of bounds...\r\n"
#     if progress == 1:
#         progress = 1
#         oh.write('\x1b[2K\r')
#         oh.flush()
#         return ""
#     block = int(round(barLength * progress))
#     if logmessage:
#         text = "{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
#         return text
#     else:
#         text = "\r{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
#         oh.write(text)
#         oh.flush()
# 
# def outputcolum(data):
#     # Calculate terminal size
#     maxStrLength = getTerminalWidth()
#     if settings.rawOutput or stdout_redirect(): maxStrLength = 1000
# 
#     # Calculate number of fields and maxStrLengthField
#     # todo: grab max size per field to do a more intelligent distribution of available term realestate
#     num_fields = len(data[len(data)-1][1])
#     maxStrLengthField = maxStrLength / num_fields
# 
#     # Truncate fields
#     # todo: There has to be a better approach to this
#     for ii in xrange(0, len(data)):
#         tmpList = []
#         for i in xrange(0, len(data[ii][1])):
#             tmpList.append(str(data[ii][1][i])[:maxStrLengthField] + (str(data[ii][1][i])[maxStrLengthField:] and '...'))
#         t1 = []
#         t1.append(data[ii][0])
#         t1.append(tuple(tmpList))
#         data[ii] = t1
#         data[ii][1] = tuple(tmpList)
#     widths = [max(map(len,map(str, col))) for col in zip(*[x[1] for x in data])]
#     for row in data:
#         if settings.rawOutput or stdout_redirect():
#             print ("  ".join((str(val).ljust(width) for val, width in zip(row[1], widths))))
#         else: print colored("  ".join((str(val).ljust(width) for val, width in zip(row[1], widths))), row[0])
#     return data
# 
# def file_size(file_name_fullpath):
#     if ".zip/" in file_name_fullpath:
#         file_object = loadFile(file_name_fullpath)
#         file_object.seek(0, os.SEEK_END)
#         return file_object.tell()
#     else: return os.path.getsize(file_name_fullpath)


# Path: appSearch.py
import settings
import logging
import multiprocessing
import collections
import os
import re
import time
import ntpath
import appDB
import hashlib
import operator
import glob
from contextlib import closing
from appAux import update_progress, outputcolum, file_size
from datetime import timedelta
from collections import namedtuple
from namedlist import namedlist
        from itertools import islice

                for rowID in rowID_list:
                    # Grab entry data we want to save to the output file:
                    record = retrieveSearchData(rowID, self.DB, self.search_space)

                    # De-dup results:
                    entryMD5 = hashlib.md5(''.join([str(e) for e in [record[0],record[1],record[2],record[3],record[4],record[5],record[9]]])).hexdigest()
                    if entryMD5 in dumped_set:
                        # print("Suppressing row %d" % entry[6])
                        with self.num_hits_suppressed.get_lock():
                            self.num_hits_suppressed.value += 1
                    else:
                        dumped_set.add(entryMD5)
                        # Re-filter against known bad individually to build histogram and highlight
                        regex_hit_name = None
                        search_space = None
                        if self.searchType == 'KNOWNBAD':
                            # Search for known_bad one by one and filter if required
                            for x in list(known_bad_search_terms):
                                if re.compile(x.regex, re.IGNORECASE).search(str(record.Search_Space)) is not None:
                                    if x.filter is not None:
                                        if re.compile(x.filter, re.IGNORECASE).search(str(record.Search_Space)) is not None:
                                            regex_hit_name = x.name
                                            continue
                                    # 'u200b' is a zero width unicode character I have to use to avoid messy markdown highlighting:
                                    search_space = re.compile('(.*)('+x.regex+')(.*)', re.I).sub(r'\1'+u'\u200b'+r'**'+u'\u200b'+r'\2'+u'\u200b'+'**'+u'\u200b'+r'\3', record.Search_Space, re.IGNORECASE)
                                    # Add hit to know_bad hit counter:
                                    regex_hit_name = x.name
                                    hit_dict[x.regex][0] += 1

                                    # We only report the match with the first regex from our set

                                    break