<commit_before>#-*- coding:utf-8 -*-
VERSION = '0.1'

import redis

try:
  import simplejson
except:
  from django.utils import simplejson

try:
  from django.core import serializers
  from django.db.models.loading import get_model
except:
  pass

import mmseg
from autocomplete.utils import queryset_iterator

class Autocomplete (object):
  """
  An example usage:

  items=['{"score": "9", "id": "1", "title": "轻轻地你走了"}', \
  '{"score": "8", "id": "2", "title": "正如你轻轻地来"}', \
  '{"score": "8.5", "id": "3", "title": "你挥一挥衣袖，不带走一片云彩"}']

  a=Autocomplete(items=items,mapping=index_mapping)
  a.rebuild_index ()
  print a.search_query (u'你 轻轻')
  print a.search_query (u'轻轻')
  print a.search_query (u'你 带走')
  
  ac=Autocomplete (items=items)
  ac.rebuild_index ()
  print a.search_query (u'你 轻轻')
  """

  def __init__ (self, redisaddr="localhost", modelname="book",
                limits=5, cached=True, mapping=None, filename=None,
                jsonitems=None, app_label=None, model_label=None, fields=None):

    self.r = redis.Redis (redisaddr)
    if model_label: modelname = model_label
    self.database='database:%s'%modelname
    self.indexbase='indexbase:%s'%modelname
    self.limits=limits
    self.fields=fields
    self.cached=True
    self.mapping=mapping
    self.filename=filename
    self.app_label=app_label
    self.model_label=model_label
    self.jsonitems=jsonitems
    mmseg.Dictionary.load_dictionaries()
    self._init_objs_auto ()

  def _init_objs_from_json_file (self):
    """
    Initialize indexed objects from file self.filename.
    Each line of the file should be one json object.
    """
    fi = open (self.filename, 'r')
    for line in fi.xreadlines ():
      yield (simplejson.loads (line))

  def _init_objs_from_model (self):
    """
    Initialize indexed objects from model with
    app.model.
    """
    mod = get_model (self.app_label, self.model_label)
    for item in queryset_iterator (mod.objects.all ()):
      modelitem = simplejson.loads(serializers.serialize ('json', [item,], fields=self.fields))[0]
      cleanitem = modelitem['fields']
      cleanitem['pk'] = modelitem['pk']
      yield (cleanitem)

  def _init_objs_from_json_lists (self):
    """
    Initialize indexed objects from a list of
    json objects.
    """
    for jsonitem in self.jsonitems:
      yield simplejson.loads (jsonitem)

  def _init_objs_auto (self):
    """
    Automatically decide which one to use as the index
    item generator.
    """
    if self.filename:
      self.item_generator = self._init_objs_from_json_file
    elif self.app_label and self.model_label:
      self.item_generator = self._init_objs_from_model
    elif self.jsonitems:
      self.item_generator = self._init_objs_from_json_lists
    else:
      raise Exception ('You have to specify the objects source you want to index.')

  def rebuild_index (self):
    self.del_index ()
    self.build_index ()

  def del_index (self):
    prefixs = self.r.smembers (self.indexbase)
    for prefix in prefixs:
      self.r.delete('%s:%s'%(self.indexbase,prefix))
    self.r.delete(self.indexbase)
    self.r.delete(self.database)

  def build_index (self):
    """
    Build index for items.
    """
    for item in self.item_generator ():
      self.add_item (item)

  def add_item (self,item):
    """
    Create index for ITEM.
    """
    self.r.hset (self.database, self._getval ('id',item), simplejson.dumps(item))
    for prefix in self.prefixs_for_term (self._getval('term',item)):
      self.r.sadd (self.indexbase, prefix)
      self.r.zadd ('%s:%s'%(self.indexbase,prefix),self._getval('id',item), self._getval('score',item))

  def _getval (self, attr, item):
    if self.mapping and self.mapping.has_key (attr):
      if callable (self.mapping[attr]):
        return self.mapping[attr] (item)
      return item[self.mapping[attr]]
    return item[attr]

  def del_item (self,item):
    """
    Delete ITEM from the index<commit_msg>    Add progress indicator when building index for model     in case there are too many.<commit_after>#-*- coding:utf-8 -*-
VERSION = '0.1'

import redis

try:
  import simplejson
except:
  from django.utils import simplejson

try:
  from django.core import serializers
  from django.db.models.loading import get_model
except:
  pass

import mmseg
from autocomplete.utils import queryset_iterator

class Autocomplete (object):
  """
  An example usage:

  items=['{"score": "9", "id": "1", "title": "轻轻地你走了"}', \
  '{"score": "8", "id": "2", "title": "正如你轻轻地来"}', \
  '{"score": "8.5", "id": "3", "title": "你挥一挥衣袖，不带走一片云彩"}']

  a=Autocomplete(items=items,mapping=index_mapping)
  a.rebuild_index ()
  print a.search_query (u'你 轻轻')
  print a.search_query (u'轻轻')
  print a.search_query (u'你 带走')
  
  ac=Autocomplete (items=items)
  ac.rebuild_index ()
  print a.search_query (u'你 轻轻')
  """

  def __init__ (self, redisaddr="localhost", modelname="book",
                limits=5, cached=True, mapping=None, filename=None,
                jsonitems=None, app_label=None, model_label=None, fields=None):

    self.r = redis.Redis (redisaddr)
    if model_label: modelname = model_label
    self.database='database:%s'%modelname
    self.indexbase='indexbase:%s'%modelname
    self.limits=limits
    self.fields=fields
    self.cached=True
    self.mapping=mapping
    self.filename=filename
    self.app_label=app_label
    self.model_label=model_label
    self.jsonitems=jsonitems
    mmseg.Dictionary.load_dictionaries()
    self._init_objs_auto ()

  def _init_objs_from_json_file (self):
    """
    Initialize indexed objects from file self.filename.
    Each line of the file should be one json object.
    """
    fi = open (self.filename, 'r')
    for line in fi.xreadlines ():
      yield (simplejson.loads (line))

  def _init_objs_from_model (self):
    """
    Initialize indexed objects from model with
    app.model.
    """
    mod = get_model (self.app_label, self.model_label)
    for item in queryset_iterator (mod.objects.all ()):
      modelitem = simplejson.loads(serializers.serialize ('json', [item,], fields=self.fields))[0]
      cleanitem = modelitem['fields']
      cleanitem['pk'] = modelitem['pk']
      yield (cleanitem)

  def _init_objs_from_json_lists (self):
    """
    Initialize indexed objects from a list of
    json objects.
    """
    for jsonitem in self.jsonitems:
      yield simplejson.loads (jsonitem)

  def _init_objs_auto (self):
    """
    Automatically decide which one to use as the index
    item generator.
    """
    if self.filename:
      self.item_generator = self._init_objs_from_json_file
    elif self.app_label and self.model_label:
      self.item_generator = self._init_objs_from_model
    elif self.jsonitems:
      self.item_generator = self._init_objs_from_json_lists
    else:
      raise Exception ('You have to specify the objects source you want to index.')

  def rebuild_index (self):
    self.del_index ()
    self.build_index ()

  def del_index (self):
    prefixs = self.r.smembers (self.indexbase)
    for prefix in prefixs:
      self.r.delete('%s:%s'%(self.indexbase,prefix))
    self.r.delete(self.indexbase)
    self.r.delete(self.database)

  def build_index (self):
    """
    Build index for items.
    """
    cnt=0
    for item in self.item_generator ():
      if cnt%1000 == 0: print 'building index for %d items' % cnt
      self.add_item (item)
      cnt=cnt+1

  def add_item (self,item):
    """
    Create index for ITEM.
    """
    self.r.hset (self.database, self._getval ('id',item), simplejson.dumps(item))
    for prefix in self.prefixs_for_term (self._getval('term',item)):
      self.r.sadd (self.indexbase, prefix)
      self.r.zadd ('%s:%s'%(self.indexbase,prefix),self._getval('id',item), self._getval('score',item))

  def _getval (self, attr, item):
    if self.mapping and self.mapping.has_key (attr):
      if callable (self.mapping[attr]):
        return self.mapping[attr] (item)
      return item[self.mapping[attr]]
    return item[attr]

  def del_item (self,item):
    """
    Delete ITEM from the index
{'fraction_non_alphanumeric': 0.12900829515909373, 'fraction_numerical': 0.0038380586851553796, 'mean_word_length': 3.3500269251480885, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5290377', 'n_tokens_mistral': 2784, 'n_tokens_neox': 2767, 'n_words': 645}