<commit_before>#!/usr/bin/python

"""Indexing users."""

__author__ = '<EMAIL> (<NAME>)'

import logging

from google.appengine.api import search


def tokenize(phrase):
  """ Create a list of partial tokens from s string. eg
  dog -> [d, do, dog]

  Args:
    phrase (str): The phrase to tokenize

  Return:
    list The list of tokens.
  """
  a = set()
  for word in phrase.lower().split():
    j = 1
    while True:
      for i in range(len(word) - j + 1):
        a.add(word[i:i + j])
      if j == len(word):
        break
      j += 1
  return list(a)


def index_users(users):
  """Creates or replaces a user document from a User model.

  Args:
    users (models.User[]): The User or list of users.
  """
  if type(users) is not list: users = [users]
  docs = []
  index = search.Index(name='users')

  for user in users:
    # Don't index admin users
    if user.is_admin:
      continue
    tokens = ','.join(tokenize('%s %s' % (user.name, user.username)))
    docs.append(search.Document(doc_id=str(user.key.id()),
                                language='en',
                                fields=[
                                    search.TextField(
                                      name='username', value=user.username),
                                    search.TextField(
                                      name='name', value=user.name),
                                    search.TextField(<commit_msg>Don't index anonymous users<commit_after>#!/usr/bin/python

"""Indexing users."""

__author__ = '<EMAIL> (<NAME>)'

import logging

from google.appengine.api import search


def tokenize(phrase):
  """ Create a list of partial tokens from s string. eg
  dog -> [d, do, dog]

  Args:
    phrase (str): The phrase to tokenize

  Return:
    list The list of tokens.
  """
  a = set()
  for word in phrase.lower().split():
    j = 1
    while True:
      for i in range(len(word) - j + 1):
        a.add(word[i:i + j])
      if j == len(word):
        break
      j += 1
  return list(a)


def index_users(users):
  """Creates or replaces a user document from a User model.

  Args:
    users (models.User[]): The User or list of users.
  """
  if type(users) is not list: users = [users]
  docs = []
  index = search.Index(name='users')

  for user in users:
    # Don't index admin/anon users
    if user.is_admin or user.is_anonymous:
      continue
    tokens = ','.join(tokenize('%s %s' % (user.name, user.username)))
    docs.append(search.Document(doc_id=str(user.key.id()),
                                language='en',
                                fields=[
                                    search.TextField(
                                      name='username', value=user.username),
                                    search.TextField(
                                      name='name', value=user.name),
                                    search.TextField(
{'fraction_non_alphanumeric': 0.12551581843191198, 'fraction_numerical': 0.0020632737276478678, 'mean_word_length': 1.683579335793358, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4507939', 'n_tokens_mistral': 879, 'n_tokens_neox': 832, 'n_words': 252}