# Path: appAux.py
# def update_progress(progress, text="Progress", logmessage=False):
#     if not stdout_redirect():
#         oh = sys.stdout
#     else:
#         oh = sys.stderr
# 
#     barLength = 25  # Modify this to change the length of the progress bar
#     status = ""
#     if isinstance(progress, int):
#         progress = float(progress)
#     if not isinstance(progress, float):
#         progress = 0
#         status = "error: progress var must be float\r\n"
#     if progress < 0:
#         progress = 0
#         status = "Halt...\r\n"
#     if progress > 1:
#         status = "Progress out of bounds...\r\n"
#     if progress == 1:
#         progress = 1
#         oh.write('\x1b[2K\r')
#         oh.flush()
#         return ""
#     block = int(round(barLength * progress))
#     if logmessage:
#         text = "{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
#         return text
#     else:
#         text = "\r{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
#         oh.write(text)
#         oh.flush()
# 
# def outputcolum(data):
#     # Calculate terminal size
#     maxStrLength = getTerminalWidth()
#     if settings.rawOutput or stdout_redirect(): maxStrLength = 1000
# 
#     # Calculate number of fields and maxStrLengthField
#     # todo: grab max size per field to do a more intelligent distribution of available term realestate
#     num_fields = len(data[len(data)-1][1])
#     maxStrLengthField = maxStrLength / num_fields
# 
#     # Truncate fields
#     # todo: There has to be a better approach to this
#     for ii in xrange(0, len(data)):
#         tmpList = []
#         for i in xrange(0, len(data[ii][1])):
#             tmpList.append(str(data[ii][1][i])[:maxStrLengthField] + (str(data[ii][1][i])[maxStrLengthField:] and '...'))
#         t1 = []
#         t1.append(data[ii][0])
#         t1.append(tuple(tmpList))
#         data[ii] = t1
#         data[ii][1] = tuple(tmpList)
#     widths = [max(map(len,map(str, col))) for col in zip(*[x[1] for x in data])]
#     for row in data:
#         if settings.rawOutput or stdout_redirect():
#             print ("  ".join((str(val).ljust(width) for val, width in zip(row[1], widths))))
#         else: print colored("  ".join((str(val).ljust(width) for val, width in zip(row[1], widths))), row[0])
#     return data
# 
# def file_size(file_name_fullpath):
#     if ".zip/" in file_name_fullpath:
#         file_object = loadFile(file_name_fullpath)
#         file_object.seek(0, os.SEEK_END)
#         return file_object.tell()
#     else: return os.path.getsize(file_name_fullpath)


import settings
import logging
import multiprocessing
import collections
import os
import re
import time
import ntpath
import appDB
import hashlib
import operator
import glob
from contextlib import closing
from appAux import update_progress, outputcolum, file_size
from datetime import timedelta
from collections import namedtuple
from namedlist import namedlist
        from itertools import islice
            num_tasks += 1
        logger.debug("Creating search job %d: [%d - %d]" % (num_tasks, num_tasks*(rowsPerJob), ((num_tasks*rowsPerJob) + (entriesCount - (num_tasks*(rowsPerJob)- 1)))))
        # Special consideration for the last one:
        tasks.put(Task(num_tasks*(rowsPerJob), (entriesCount - ((num_tasks*rowsPerJob)- 1))))
        logger.debug("Number of tasks: %d" % num_tasks)

        # Add a poison pill for each producer
        for i in xrange(num_producers):
            tasks.put(None)

        # Start producer threads
        producers = [Producer(tasks, resultsProducers, dbfilenameFullPath, progProducers, num_consumers, \
                              searchType, search_space, options, num_hits, known_bad_data) for i in xrange(num_producers)]
        for producer in producers:
            producer.daemon = True # Remove for debugging
            producer.start()

        # Start consumer threads
        consumers = [Consumer(resultsProducers, resultsConsumers, progConsumers, num_producers, outputFile, \
                              dbfilenameFullPath, searchType, search_space, options, num_hits, \
                              num_hits_suppressed, hitHistogram_queue, known_bad_data) for i in xrange(num_consumers)]
        for consumer in consumers:
            consumer.daemon = True  # Remove for debugging
            consumer.start()

        # Producer progress loop
        while (num_tasks > progProducers.value and progProducers.value >= 0):
            logger.debug("Producer num_tasks: %d - v.value: %d" % (num_tasks, progProducers.value))
            update_progress(min(1, float(progProducers.value) / float(num_tasks)), "Searching [%d]" % (num_hits.value - num_hits_suppressed.value))
            time.sleep(0.5)

# Path: appAux.py
# def update_progress(progress, text="Progress", logmessage=False):
#     if not stdout_redirect():
#         oh = sys.stdout
#     else:
#         oh = sys.stderr
# 
#     barLength = 25  # Modify this to change the length of the progress bar
#     status = ""
#     if isinstance(progress, int):
#         progress = float(progress)
#     if not isinstance(progress, float):
#         progress = 0
#         status = "error: progress var must be float\r\n"
#     if progress < 0:
#         progress = 0
#         status = "Halt...\r\n"
#     if progress > 1:
#         status = "Progress out of bounds...\r\n"
#     if progress == 1:
#         progress = 1
#         oh.write('\x1b[2K\r')
#         oh.flush()
#         return ""
#     block = int(round(barLength * progress))
#     if logmessage:
#         text = "{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
#         return text
#     else:
#         text = "\r{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
#         oh.write(text)
#         oh.flush()
# 
# def outputcolum(data):
#     # Calculate terminal size
#     maxStrLength = getTerminalWidth()
#     if settings.rawOutput or stdout_redirect(): maxStrLength = 1000
# 
#     # Calculate number of fields and maxStrLengthField
#     # todo: grab max size per field to do a more intelligent distribution of available term realestate
#     num_fields = len(data[len(data)-1][1])
#     maxStrLengthField = maxStrLength / num_fields
# 
#     # Truncate fields
#     # todo: There has to be a better approach to this
#     for ii in xrange(0, len(data)):
#         tmpList = []
#         for i in xrange(0, len(data[ii][1])):
#             tmpList.append(str(data[ii][1][i])[:maxStrLengthField] + (str(data[ii][1][i])[maxStrLengthField:] and '...'))
#         t1 = []
#         t1.append(data[ii][0])
#         t1.append(tuple(tmpList))
#         data[ii] = t1
#         data[ii][1] = tuple(tmpList)
#     widths = [max(map(len,map(str, col))) for col in zip(*[x[1] for x in data])]
#     for row in data:
#         if settings.rawOutput or stdout_redirect():
#             print ("  ".join((str(val).ljust(width) for val, width in zip(row[1], widths))))
#         else: print colored("  ".join((str(val).ljust(width) for val, width in zip(row[1], widths))), row[0])
#     return data
# 
# def file_size(file_name_fullpath):
#     if ".zip/" in file_name_fullpath:
#         file_object = loadFile(file_name_fullpath)
#         file_object.seek(0, os.SEEK_END)
#         return file_object.tell()
#     else: return os.path.getsize(file_name_fullpath)


# Path: appSearch.py
import settings
import logging
import multiprocessing
import collections
import os
import re
import time
import ntpath
import appDB
import hashlib
import operator
import glob
from contextlib import closing
from appAux import update_progress, outputcolum, file_size
from datetime import timedelta
from collections import namedtuple
from namedlist import namedlist
        from itertools import islice

            num_tasks += 1
        logger.debug("Creating search job %d: [%d - %d]" % (num_tasks, num_tasks*(rowsPerJob), ((num_tasks*rowsPerJob) + (entriesCount - (num_tasks*(rowsPerJob)- 1)))))
        # Special consideration for the last one:
        tasks.put(Task(num_tasks*(rowsPerJob), (entriesCount - ((num_tasks*rowsPerJob)- 1))))
        logger.debug("Number of tasks: %d" % num_tasks)

        # Add a poison pill for each producer
        for i in xrange(num_producers):
            tasks.put(None)

        # Start producer threads
        producers = [Producer(tasks, resultsProducers, dbfilenameFullPath, progProducers, num_consumers, \
                              searchType, search_space, options, num_hits, known_bad_data) for i in xrange(num_producers)]
        for producer in producers:
            producer.daemon = True # Remove for debugging
            producer.start()

        # Start consumer threads
        consumers = [Consumer(resultsProducers, resultsConsumers, progConsumers, num_producers, outputFile, \
                              dbfilenameFullPath, searchType, search_space, options, num_hits, \
                              num_hits_suppressed, hitHistogram_queue, known_bad_data) for i in xrange(num_consumers)]
        for consumer in consumers:
            consumer.daemon = True  # Remove for debugging
            consumer.start()

        # Producer progress loop
        while (num_tasks > progProducers.value and progProducers.value >= 0):
            logger.debug("Producer num_tasks: %d - v.value: %d" % (num_tasks, progProducers.value))
            update_progress(min(1, float(progProducers.value) / float(num_tasks)), "Searching [%d]" % (num_hits.value - num_hits_suppressed.value))
            time.sleep(0.5)

        update_progress(1, "Searching [%d]" % (num_hits.value - num_hits_suppressed.value))