<commit_before>from gevent import monkey
monkey.patch_all()
from restkit.session import set_session
set_session("gevent")

from django.db.models import get_apps
from django.core.management.base import BaseCommand
from django.core.mail import send_mail
from datetime import datetime
from gevent.pool import Pool
import logging
import time
from django.conf import settings
setattr(settings, 'COUCHDB_TIMEOUT', 999999)
from couchdbkit.ext.django.loading import couchdbkit_handler
from dimagi.utils.couch.database import get_db


POOL_SIZE = getattr(settings, 'PREINDEX_POOL_SIZE', 8)
POOL_WAIT = getattr(settings, 'PREINDEX_POOL_WAIT', 10)
MAX_TRIES = getattr(settings, 'PREINDEX_MAX_TRIES', 3)

def do_sync(app_index):
    """
    Get the app for the given index.
    For multiprocessing can't pass a complex object hence the call here...again
    """
    #sanity check:
    try:
        app = get_apps()[app_index]
        couchdbkit_handler.sync(app, verbosity=2, temp='tmp')
    except Exception, ex:
        logging.error("Exception running sync, but ignoring.\n\tapp=%s\n\t%s" % (app, ex))
        return None
    print "Done with %s!!!" % app_index
    return app_index

class Command(BaseCommand):
    help = 'Sync design docs to temporary ids...but multithreaded'

    def handle(self, *args, **options):


        start = datetime.utcnow()
        if len(args) == 0:
            num_pool = POOL_SIZE
        else:
            num_pool = int(args[0])

        if len(args) > 1:
            username = args[1]
        else:
            username = 'unknown'

        pool = Pool(num_pool)

        apps = get_apps()
        tries_dict = {}
        
        completed = set()
        running = []
        app_ids = set(range(len(apps)))
        for app_id in sorted(app_ids.difference(completed)):
            #keep trying all the preindexes until they all complete satisfactorily.
            print "Trying to preindex view (%d/%d) %s" % (app_id, len(apps), apps[app_id])
            curr_g = pool.spawn(do_sync, app_id)
            running.append(curr_g)

            for g in running:
                if g.ready(): #finished execution?
                    if g.value is not None:
                        completed.add(g.value)
                        running.remove(g)
                    else:
                        print "\tSync failed for %s, trying again" % (apps[app_id])
                        curr_tries = tries_dict.get(app_id, 0)
                        if curr_tries > MAX_TRIES:
                            completed.add(app_id)
                            print "\tmax tries exceeded, marking as done"
                        else:
                            tries_dict[app_id] = curr_tries + 1
            print "end of main app for loop"
<commit_msg>cleaning up preindex - turns out the logging call may gummed up the event workers it seems. removing it...but still seems to be fading in concurrency towards the end. still seems like something is getting unreleased so to speak on the workers as time goes on<commit_after>from gevent import monkey
monkey.patch_all()
from restkit.session import set_session
set_session("gevent")

from django.db.models import get_apps
from django.core.management.base import BaseCommand
from django.core.mail import send_mail
from datetime import datetime
from gevent.pool import Pool
import logging
import time
from django.conf import settings
setattr(settings, 'COUCHDB_TIMEOUT', 999999)
from couchdbkit.ext.django.loading import couchdbkit_handler
from dimagi.utils.couch.database import get_db


POOL_SIZE = getattr(settings, 'PREINDEX_POOL_SIZE', 8)
POOL_WAIT = getattr(settings, 'PREINDEX_POOL_WAIT', 10)
MAX_TRIES = getattr(settings, 'PREINDEX_MAX_TRIES', 3)

def do_sync(app_index):
    """
    Get the app for the given index.
    For multiprocessing can't pass a complex object hence the call here...again
    """
    #sanity check:
    try:
        app = get_apps()[app_index]
        couchdbkit_handler.sync(app, verbosity=2, temp='tmp')
    except Exception, ex:
        print "Exception running sync, but ignoring.\n\tapp=%s\n\t%s" % (app, ex)
        return "%s-error" % app_index
    print "preindex %s complete" % app_index
    return app_index

class Command(BaseCommand):
    help = 'Sync design docs to temporary ids...but multithreaded'

    def handle(self, *args, **options):


        start = datetime.utcnow()
        if len(args) == 0:
            num_pool = POOL_SIZE
        else:
            num_pool = int(args[0])

        if len(args) > 1:
            username = args[1]
        else:
            username = 'unknown'

        pool = Pool(num_pool)

        apps = get_apps()

        completed = set()
        app_ids = set(range(len(apps)))
        for app_id in sorted(app_ids.difference(completed)):
            #keep trying all the preindexes until they all complete satisfactorily.
            print "Trying to preindex view (%d/%d) %s" % (app_id, len(apps), apps[app_id])
            pool.spawn(do_sync, app_id)

{'fraction_non_alphanumeric': 0.0924520766773163, 'fraction_numerical': 0.006389776357827476, 'mean_word_length': 2.4808895066018066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3018597', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1472, 'n_words': 447}