<commit_before>#!/usr/bin/python2.5
# Copyright 2011 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Unittest for external_search.py module."""

__author__ = '<EMAIL> (<NAME>)'

import datetime
import logging
import mox
import random
import simplejson
import sys
import unittest

import external_search
import model
import text_query
import utils
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors


class MockPerson:
    """Mock Person with minimal attributes."""
    def __init__(self, subdomain, record_id, first_name, last_name,
                 is_expired=False):
        self.subdomain = subdomain
        self.record_id = record_id
        self.key_name = '%s:%s' % (subdomain, record_id)
        self.first_name = first_name
        self.last_name = last_name
        self.alternate_first_names = self.alternate_last_names = ''
        self.names_prefixes = text_query.TextQuery(
            '%s %s' % (first_name, last_name)).query_words
        self.is_expired = is_expired

    @staticmethod
    def get_by_key_name(key_names):
        person_dict = dict([(p.key_name, p) for p in MOCK_PERSONS])
        return [person_dict.get(key_name, None) for key_name in key_names]

MOCK_PERSONS = [
    MockPerson('japan', 'test/1', '', 'Mori'),
    # missing test/2
    MockPerson('japan', 'test/3', 'Ogai', 'Mori'),
    MockPerson('japan', 'test/4', 'Ogai', 'Mori'),
    MockPerson('japan', 'test/5', 'Natsume', 'Souseki'),
    MockPerson('japan', 'test/6', 'Natsume', 'Souseki', True),
]


class MockUrlFetchResponse:
    """Mock response object returned from urlfetch.fetch."""
    def __init__(self, status_code, content):
        self.status_code = status_code
        self.content = simplejson.dumps(content)


class MockLoggingHandler(logging.Handler):
    """Mock logging handler to check for expected logs."""
    def __init__(self, *args, **kwargs):
        self.reset()
        logging.Handler.__init__(self, *args, **kwargs)

    def emit(self, record):
        self.messages[record.levelname.lower()].append(record.getMessage())

    def reset(self):
        self.messages = {
            'debug': [],
            'info': [],
            'warning': [],
            'error': [],
            'critical': [],
        }


def IsSeconds(float_value):
    """Return a mox comparator that checks 6 decimal places, intended to be
    used for comparing seconds."""
    return mox.IsAlmost(float_value, 6)


class ExternalSearchTests(unittest.TestCase):
    def setUp(self):
        self.mox = mox.Mox()
        self.mox.StubOutWithMock(urlfetch, 'fetch')
        self.mox.StubOutWithMock(random, 'shuffle')
        random.shuffle(mox.IsA(list))

        self.orig_person = model.Person
        model.Person = MockPerson

        self.mock_logging_handler = MockLoggingHandler()
        logging.getLogger().addHandler(self.mock_logging_handler)

        # The first two calls of utils.get_utcnow_seconds() at line 45 and 49 in
        # external_search.py consult the following date setting for debug.
        utils.set_utcnow_for_test(datetime.datetime(2011, 1, 1))

    def tearDown(self):
        self.mox.UnsetStubs()
        model.Person = self.orig_person
        logging.getLogger().removeHandler(self.mock_logging_handler)

    def advance_seconds(self, seconds):
        utils.set_utcnow_for_test(
            utils.get_utcnow() + datetime.timedelta(seconds=seconds))

    def test_search_missing_entries(self):
        response = MockUrlFetchResponse(200, {
            'name_entries': [
                {'person_record_id': 'test/1'},
                {'person_record_id': 'test/2'},
                {'person_record_id': 'test/3'},
                {'person_record_id': 'test/4'},
                {'person_record_id': 'test/5'},
                {'person_record_id': 'test/6'},
            ],
            'all_entries': []
        })
        urlfetch.fetch('http://backend/?q=mori',
                       deadline=IsSeconds(0.9)).AndReturn(response)
        self.mox.ReplayAll()
        results = external_search.search(
            'japan', text_query.TextQuery('mori'), 100,
            ['http://backend/?q=%s'])
        self.assertEquals(4, len(results))
        self.assertEquals('test/1', results[0].record_id)
        self.assertEquals('test/3', results[1].record_id)
        self.assertEquals('test/4', results[2].record_id)
        self.assertEquals('test/5', results[3].record_id)<commit_msg>update the CL again, more merging..<commit_after>#!/usr/bin/python2.5
# Copyright 2011 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Unittest for external_search.py module."""

__author__ = '<EMAIL> (<NAME>)'

import datetime
import logging
import mox
import random
import simplejson
import sys
import unittest

import external_search
import model
import text_query
import utils
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors


class MockPerson:
    """Mock Person with minimal attributes."""
    def __init__(self, subdomain, record_id, first_name, last_name,
                 is_expired=False):
        self.subdomain = subdomain
        self.record_id = record_id
        self.key_name = '%s:%s' % (subdomain, record_id)
        self.first_name = first_name
        self.last_name = last_name
        self.alternate_first_names = self.alternate_last_names = ''
        self.names_prefixes = text_query.TextQuery(
            '%s %s' % (first_name, last_name)).query_words
        self.is_expired = is_expired

    @staticmethod
    def get_by_key_name(key_names):
        person_dict = dict([(p.key_name, p) for p in MOCK_PERSONS])
        return [person_dict.get(key_name, None) for key_name in key_names]

MOCK_PERSONS = [
    MockPerson('japan', 'test/1', '', 'Mori'),
    # missing test/2
    MockPerson('japan', 'test/3', 'Ogai', 'Mori'),
    MockPerson('japan', 'test/4', 'Ogai', 'Mori'),
    MockPerson('japan', 'test/5', 'Natsume', 'Souseki'),
    MockPerson('japan', 'test/6', 'Natsume', 'Souseki', True),
]


class MockUrlFetchResponse:
    """Mock response object returned from urlfetch.fetch."""
    def __init__(self, status_code, content):
        self.status_code = status_code
        self.content = simplejson.dumps(content)


class MockLoggingHandler(logging.Handler):
    """Mock logging handler to check for expected logs."""
    def __init__(self, *args, **kwargs):
        self.reset()
        logging.Handler.__init__(self, *args, **kwargs)

    def emit(self, record):
        self.messages[record.levelname.lower()].append(record.getMessage())

    def reset(self):
        self.messages = {
            'debug': [],
            'info': [],
            'warning': [],
            'error': [],
            'critical': [],
        }


def IsSeconds(float_value):
    """Return a mox comparator that checks 6 decimal places, intended to be
    used for comparing seconds."""
    return mox.IsAlmost(float_value, 6)


class ExternalSearchTests(unittest.TestCase):
    def setUp(self):
        self.mox = mox.Mox()
        self.mox.StubOutWithMock(urlfetch, 'fetch')
        self.mox.StubOutWithMock(random, 'shuffle')
        random.shuffle(mox.IsA(list))

        self.orig_person = model.Person
        model.Person = MockPerson

        self.mock_logging_handler = MockLoggingHandler()
        logging.getLogger().setLevel(logging.INFO)

        # The first two calls of utils.get_utcnow_seconds() at line 45 and 49 in
        # external_search.py consult the following date setting for debug.
        utils.set_utcnow_for_test(datetime.datetime(2011, 1, 1))

    def tearDown(self):
        self.mox.UnsetStubs()
        model.Person = self.orig_person
        logging.getLogger().removeHandler(self.mock_logging_handler)
        logging.getLogger().setLevel(logging.WARNING)

    def advance_seconds(self, seconds):
        utils.set_utcnow_for_test(
            utils.get_utcnow() + datetime.timedelta(seconds=seconds))

    def test_search_missing_entries(self):
        response = MockUrlFetchResponse(200, {
            'name_entries': [
                {'person_record_id': 'test/1'},
                {'person_record_id': 'test/2'},
                {'person_record_id': 'test/3'},
                {'person_record_id': 'test/4'},
                {'person_record_id': 'test/5'},
                {'person_record_id': 'test/6'},
            ],
            'all_entries': []
        })
        urlfetch.fetch('http://backend/?q=mori',
                       deadline=IsSeconds(0.9)).AndReturn(response)
        self.mox.ReplayAll()
        results = external_search.search(
            'japan', text_query.TextQuery('mori'), 100,
            ['http://backend/?q=%s'])
        self.assertEquals(4, len(results))
        self.assertEquals('test/1', results[0].record_id)
        self.assertEquals('test/3', results[1].record_id)
        self.assertEquals('test/4', results[2].record_id)
        self.assertEquals('test/5', results[3].record_id)
{'fraction_non_alphanumeric': 0.12296311106668, 'fraction_numerical': 0.010196940917724683, 'mean_word_length': 2.9078125, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '494757', 'n_tokens_mistral': 3055, 'n_tokens_neox': 2968, 'n_words': 711}