<commit_before>from optparse import make_option
from dimagi.utils.couch import sync_docs
from gevent import monkey
from corehq.preindex import get_preindex_plugins

monkey.patch_all()
from restkit.session import set_session
set_session("gevent")

from django.db.models import get_apps
from django.core.management.base import BaseCommand
from django.core.mail import send_mail
from datetime import datetime
from gevent.pool import Pool
from django.conf import settings
setattr(settings, 'COUCHDB_TIMEOUT', 999999)
from couchdbkit.ext.django.loading import couchdbkit_handler


POOL_SIZE = getattr(settings, 'PREINDEX_POOL_SIZE', 8)
POOL_WAIT = getattr(settings, 'PREINDEX_POOL_WAIT', 10)
MAX_TRIES = getattr(settings, 'PREINDEX_MAX_TRIES', 3)


def do_sync(app_index):
    """
    Get the app for the given index.
    For multiprocessing can't pass a complex object hence the call here...again
    """
    #sanity check:
    app = get_apps()[app_index]
    sync_docs.sync(app, verbosity=2, temp='tmp')
    print "preindex %s complete" % app_index
    return app_index<commit_msg>unused import<commit_after>from optparse import make_option
from dimagi.utils.couch import sync_docs
from gevent import monkey
from corehq.preindex import get_preindex_plugins

monkey.patch_all()
from restkit.session import set_session
set_session("gevent")

from django.db.models import get_apps
from django.core.management.base import BaseCommand
from django.core.mail import send_mail
from datetime import datetime
from gevent.pool import Pool
from django.conf import settings
setattr(settings, 'COUCHDB_TIMEOUT', 999999)


POOL_SIZE = getattr(settings, 'PREINDEX_POOL_SIZE', 8)
POOL_WAIT = getattr(settings, 'PREINDEX_POOL_WAIT', 10)
MAX_TRIES = getattr(settings, 'PREINDEX_MAX_TRIES', 3)


def do_sync(app_index):
    """
    Get the app for the given index.
    For multiprocessing can't pass a complex object hence the call here...again
    """
    #sanity check:
    app = get_apps()[app_index]
    sync_docs.sync(app, verbosity=2, temp='tmp')
    print "preindex %s complete" % app_index
    return app_index
{'fraction_non_alphanumeric': 0.08162291169451073, 'fraction_numerical': 0.010501193317422435, 'mean_word_length': 5.2942942942942945, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '937467', 'n_tokens_mistral': 683, 'n_tokens_neox': 661, 'n_words': 194}