# Path: sesql/config.py
# CONFIG_PATH_FALLBACK = 'sesql_config'
#     CONFIG_PATH = getattr(settings, 'SESQL_CONFIG_PATH', CONFIG_PATH_FALLBACK)
#     CONFIG_PATH = CONFIG_PATH_FALLBACK
# 
# Path: sesql/lemmatize.py
# def lemmatize(words, index = None):
#     """
#     Give a lemmatized version of those words
# 
#     Use the configuration for the given index, or the default one if
#     index is None
#     """
#     if index is None:
#         index = fieldmap.primary
# 
#     if index is None:
#         raise ValueError, "Not index given and no primary one"
# 
#     words = [ index.marshall(word) for word in words ]
# 
#     index = fieldmap.get_field(index)
#     return lemmatize_for(words, index.dictionnary)
# 
# Path: sesql/models.py
# class SearchHit(models.Model):
#     """Used to store queries made to the search engine"""
#     query = models.CharField(max_length=500)
#     nb_results = models.PositiveIntegerField()
#     date = models.DateField(auto_now=True)
# 
# Path: sesql/models.py
# class SearchQuery(models.Model):
#     """A table containing statistics and scores about search queries"""
#     query = models.CharField(max_length=500)
#     phonex = models.FloatField()
#     clean_query = models.CharField(max_length=500)
#     clean_phonex = models.FloatField()
#     
#     nb_results = models.PositiveIntegerField()
#     
#     nb_recent_search = models.PositiveIntegerField()
#     nb_total_search = models.PositiveIntegerField()
#     pondered_search_nb = models.FloatField()
#     
#     weight = models.FloatField()
# 
# Path: sesql/models.py
# class SearchHitHistoric(models.Model):
#     """Same as SearchHit used as an archive"""
#     query = models.CharField(max_length=500)
#     nb_results = models.PositiveIntegerField()
#     date = models.DateField(auto_now=True, db_index=True)
# 
# Path: sesql/suggest.py
# def levenshtein(a, b):


from optparse import make_option
from django.core.management.base import BaseCommand
from sesql import config
from sesql.lemmatize import lemmatize
from sesql.models import SearchHit
from sesql.models import SearchQuery
from sesql.models import SearchHitHistoric
from sesql.suggest import phonex
            if query in config.HISTORY_BLACKLIST:
                continue

            if hit.nb_results < filter_nb:
                SearchHitHistoric(query=hit.query,
                                  nb_results=hit.nb_results,
                                  date=hit.date).save()
                hit.delete()
                continue

            # manual get_or_create
            try:
                search_query = SearchQuery.objects.get(query=query)
                created = False
            except SearchQuery.DoesNotExist:
                search_query = SearchQuery(query=query)
                created = True

            # if it's a new one, initialize it
            if created:
                search_query.phonex = phonex(query)

                # clean the query, the '_' char cause bugy clean_query
                query = query.replace('_', '')

                lems = lemmatize(query.split())

                clean_query = [lem for lem in lems if lem]
                clean_query = ' '.join(clean_query)


# Path: sesql/config.py
# CONFIG_PATH_FALLBACK = 'sesql_config'
#     CONFIG_PATH = getattr(settings, 'SESQL_CONFIG_PATH', CONFIG_PATH_FALLBACK)
#     CONFIG_PATH = CONFIG_PATH_FALLBACK
# 
# Path: sesql/lemmatize.py
# def lemmatize(words, index = None):
#     """
#     Give a lemmatized version of those words
# 
#     Use the configuration for the given index, or the default one if
#     index is None
#     """
#     if index is None:
#         index = fieldmap.primary
# 
#     if index is None:
#         raise ValueError, "Not index given and no primary one"
# 
#     words = [ index.marshall(word) for word in words ]
# 
#     index = fieldmap.get_field(index)
#     return lemmatize_for(words, index.dictionnary)
# 
# Path: sesql/models.py
# class SearchHit(models.Model):
#     """Used to store queries made to the search engine"""
#     query = models.CharField(max_length=500)
#     nb_results = models.PositiveIntegerField()
#     date = models.DateField(auto_now=True)
# 
# Path: sesql/models.py
# class SearchQuery(models.Model):
#     """A table containing statistics and scores about search queries"""
#     query = models.CharField(max_length=500)
#     phonex = models.FloatField()
#     clean_query = models.CharField(max_length=500)
#     clean_phonex = models.FloatField()
#     
#     nb_results = models.PositiveIntegerField()
#     
#     nb_recent_search = models.PositiveIntegerField()
#     nb_total_search = models.PositiveIntegerField()
#     pondered_search_nb = models.FloatField()
#     
#     weight = models.FloatField()
# 
# Path: sesql/models.py
# class SearchHitHistoric(models.Model):
#     """Same as SearchHit used as an archive"""
#     query = models.CharField(max_length=500)
#     nb_results = models.PositiveIntegerField()
#     date = models.DateField(auto_now=True, db_index=True)
# 
# Path: sesql/suggest.py
# def levenshtein(a, b):


# Path: sesql/management/commands/build_search_query_index.py
from optparse import make_option
from django.core.management.base import BaseCommand
from sesql import config
from sesql.lemmatize import lemmatize
from sesql.models import SearchHit
from sesql.models import SearchQuery
from sesql.models import SearchHitHistoric
from sesql.suggest import phonex

            if query in config.HISTORY_BLACKLIST:
                continue

            if hit.nb_results < filter_nb:
                SearchHitHistoric(query=hit.query,
                                  nb_results=hit.nb_results,
                                  date=hit.date).save()
                hit.delete()
                continue

            # manual get_or_create
            try:
                search_query = SearchQuery.objects.get(query=query)
                created = False
            except SearchQuery.DoesNotExist:
                search_query = SearchQuery(query=query)
                created = True

            # if it's a new one, initialize it
            if created:
                search_query.phonex = phonex(query)

                # clean the query, the '_' char cause bugy clean_query
                query = query.replace('_', '')

                lems = lemmatize(query.split())

                clean_query = [lem for lem in lems if lem]
                clean_query = ' '.join(clean_query)


                clean_phonex = phonex(clean_query)