<commit_before>import logging
import re

from google.appengine.api import search

import model

# The index name for full text search
PERSON_FULLTEXT_INDEX_NAME = 'person_information'

def search_with_index(repo, query_obj, max_results):
    """
    Searches person with index.
    Args:
        repo: The name of repository
        query_obj: Search word
        max_results: The max results you want.(Maximum: 1000)
    Returns:
        results[<model.Person>, ...]
    """
    def create_query(query):
        query_words = query.split(' ')
        query_string = ''
        for word in query_words:
            if (word != ''):
                query_string += word + ' AND '
        return query_string[:-5]

    results = []
    if (query_obj == ''):
        return results
    query = create_query(query_obj)
    try:
        index = search.Index(name=PERSON_FULLTEXT_INDEX_NAME)
        options = search.QueryOptions(
            limit=max_results,
            returned_fields=['record_id'])
        results_index = index.search(search.Query(
            query_string=query, options=options))
        record_ids = []
        for document in results_index:
            record_ids.append(document.fields[0].value)
        for id in record_ids:
            results.append(model.Person.get_by_key_name(repo + ':' + id))
    except search.Error:
        logging.exception('Search exception')
    return results


def create_document(**kwargs):
    """
    Creates document for full text search.
    It should be called in creat_index method.
    """
    TEXT_FIELD_TABLE = ['record_id', 'repo', 'given_name', 'family_name',
                       'full_name', 'alternate_names']
    fields = []
    for field in TEXT_FIELD_TABLE:
        if field in kwargs:
            fields.append(search.TextField(name=field, value=kwargs[field]))
    return search.Document(fields=fields)


def create_index(**kwargs):
    """
    Creates index.
    (Field: record_id, repo, given_name, family_name, full_name, alternate?name)
    """
    try:
        index_name = search.Index(name=PERSON_FULLTEXT_INDEX_NAME)
        index_name.put(create_document(**kwargs))
    except search.Error:
        logging.exception('Put failed')


def delete_index(person):
    """Deletes index.
    Args:
        person: Person who should be removed
    """
    index = search.Index(name=PERSON_FULLTEXT_INDEX_NAME)
    splited_record = re.compile(r'[:./]').split(person.key().name())
    repo = splited_record[0]
    person_record_id = splited_record[-1]
    try:
        result = index.search(
            'repo:' + repo + ' AND record_id:' + person_record_id)
        if result.results:
            document_id = result.results[0].doc_id
            index.delete(document_id)
    except search.Error:
        logging.exception('Search failed')
<commit_msg>apply comment<commit_after>import logging
import re

from google.appengine.api import search

import model

# The index name for full text search
PERSON_FULLTEXT_INDEX_NAME = 'person_information'

def search_with_index(repo, query_obj, max_results):
    """
    Searches person with index.
    Args:
        repo: The name of repository
        query_obj: Search word
        max_results: The max results you want.(Maximum: 1000)
    Returns:
        results[<model.Person>, ...]
    """
    #TODO: Sanitaize query_obj
    results = []
    if not query_obj:
        return results
    try:
        index = search.Index(name=PERSON_FULLTEXT_INDEX_NAME)
        options = search.QueryOptions(
            limit=max_results,
            returned_fields=['record_id'])
        results_index = index.search(search.Query(
            query_string=query_obj, options=options))
        record_ids = []
        for document in results_index:
            record_ids.append(document.fields[0].value)
        for id in record_ids:
            results.append(model.Person.get_by_key_name(repo + ':' + id))
    except search.Error:
        logging.exception('Search exception')
    return results


def create_document(**kwargs):
    """
    Creates document for full text search.
    It should be called in creat_index method.
    """
    TEXT_FIELD_TABLE = ['record_id', 'repo', 'given_name', 'family_name',
                       'full_name', 'alternate_names']
    fields = []
    for field in TEXT_FIELD_TABLE:
        if field in kwargs:
            fields.append(search.TextField(name=field, value=kwargs[field]))
    return search.Document(fields=fields)


def create_index(**kwargs):
    """
    Creates index.
    (Field: record_id, repo, given_name, family_name, full_name, alternate?name)
    """
    try:
        index_name = search.Index(name=PERSON_FULLTEXT_INDEX_NAME)
        index_name.put(create_document(**kwargs))
    except search.Error:
        logging.exception('Put failed')


def delete_index(person):
    """Deletes index.
    Args:
        person: Person who should be removed
    """
    index = search.Index(name=PERSON_FULLTEXT_INDEX_NAME)
    splited_record = re.compile(r'[:./]').split(person.key().name())
    repo = splited_record[0]
    person_record_id = splited_record[-1]
    try:
        result = index.search(
            'repo:' + repo + ' AND record_id:' + person_record_id)
        if result.results:
            document_id = result.results[0].doc_id
            index.delete(document_id)
    except search.Error:
        logging.exception('Search failed')

{'fraction_non_alphanumeric': 0.10705057216685124, 'fraction_numerical': 0.003137689184200812, 'mean_word_length': 2.671409214092141, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6054697', 'n_tokens_mistral': 1597, 'n_tokens_neox': 1576, 'n_words': 361}