# Path: appAux.py
# def psutil_phymem_usage():
#     """
#     Return physical memory usage (float)
#     Requires the cross-platform psutil (>=v0.3) library
#     (http://code.google.com/p/psutil/)
#     """
#     # This is needed to avoid a deprecation warning error with
#     # newer psutil versions
# 
#     if settings.__PSUTIL__ == False:
#         return 0.0
# 
#     try:
#         percent = psutil.virtual_memory().percent
#     except:
#         percent = psutil.phymem_usage().percent
#     return percent


import settings
import logging
import multiprocessing
import Queue
import time
import gc
from datetime import timedelta, datetime
from appAux import psutil_phymem_usage
    @rate_limited(1.0 / 10.0)
    def rebalance(self):
        if self.governorOffFlag:
            return
        progProducers = self.getProgressProducers()
        progConsumers = self.getProgressConsumers()
        num_tasks = self.get_num_tasks()
        elapsed_backoff_time = (datetime.now() - self._rebalance_last_kick).seconds

        logger.debug("Starting balancing (timer: %d/%d)" % (elapsed_backoff_time, self.rebalance_backoff_timer))

        # Kill producers if all tasks have been served
        if num_tasks == progProducers and progProducers > 0:
            self.endProducers()
            return

        # Restart paused production on backoff timer or if we have at least 20% memory available
        if self.num_producers == 0 and ((elapsed_backoff_time > self.rebalance_backoff_timer) or psutil_phymem_usage() < 80):
            logger.debug("Rebalancing, restarting production")
            self.addProducer()
            return

        # Memory governor
        # Pause production if we're over 90%
        if psutil_phymem_usage() > 90:
            logger.debug("Rebalancing, mem > 90%, pausing production")
            self.endProducers()
            self._rebalance_last_kick = datetime.now()
            return
        # Reduce production if we're over 75%

# Path: appAux.py
# def psutil_phymem_usage():
#     """
#     Return physical memory usage (float)
#     Requires the cross-platform psutil (>=v0.3) library
#     (http://code.google.com/p/psutil/)
#     """
#     # This is needed to avoid a deprecation warning error with
#     # newer psutil versions
# 
#     if settings.__PSUTIL__ == False:
#         return 0.0
# 
#     try:
#         percent = psutil.virtual_memory().percent
#     except:
#         percent = psutil.phymem_usage().percent
#     return percent


# Path: mpEngineProdCons.py
import settings
import logging
import multiprocessing
import Queue
import time
import gc
from datetime import timedelta, datetime
from appAux import psutil_phymem_usage

    @rate_limited(1.0 / 10.0)
    def rebalance(self):
        if self.governorOffFlag:
            return
        progProducers = self.getProgressProducers()
        progConsumers = self.getProgressConsumers()
        num_tasks = self.get_num_tasks()
        elapsed_backoff_time = (datetime.now() - self._rebalance_last_kick).seconds

        logger.debug("Starting balancing (timer: %d/%d)" % (elapsed_backoff_time, self.rebalance_backoff_timer))

        # Kill producers if all tasks have been served
        if num_tasks == progProducers and progProducers > 0:
            self.endProducers()
            return

        # Restart paused production on backoff timer or if we have at least 20% memory available
        if self.num_producers == 0 and ((elapsed_backoff_time > self.rebalance_backoff_timer) or psutil_phymem_usage() < 80):
            logger.debug("Rebalancing, restarting production")
            self.addProducer()
            return

        # Memory governor
        # Pause production if we're over 90%
        if psutil_phymem_usage() > 90:
            logger.debug("Rebalancing, mem > 90%, pausing production")
            self.endProducers()
            self._rebalance_last_kick = datetime.now()
            return
        # Reduce production if we're over 75%

        if psutil_phymem_usage() > 75 and self.num_producers > 1: