<commit_before>from __future__ import unicode_literals

import random

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import connection, reset_queries
from django.utils import timezone
from temba.contacts.models import Contact, ContactField, ContactGroup, ContactURN, TEL_SCHEME, TWITTER_SCHEME
from temba.orgs.models import Org
from temba.channels.models import Channel
from temba.flows.models import FlowRun, FlowStep
from temba.msgs.models import Broadcast, Call, Label, Msg, INCOMING, OUTGOING, PENDING
from temba.utils import truncate
from temba.values.models import Value, TEXT, DECIMAL
from tests import TembaTest
from timeit import default_timer


MAX_QUERIES_PRINT = 15


class SegmentProfiler(object):
    """
    Used in a with block to profile a segment of code
    """
    def __init__(self, test, name, db_profile):
        self.test = test
        self.test.segments.append(self)
        self.name = name
        self.db_profile = db_profile
        self.old_debug = settings.DEBUG

        self.time_total = 0.0
        self.time_queries = 0.0
        self.queries = []

    def __enter__(self):
        if self.db_profile:
            settings.DEBUG = True
            reset_queries()

        self.start_time = default_timer()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.time_total = default_timer() - self.start_time

        if self.db_profile:
            settings.DEBUG = self.old_debug
            self.queries = connection.queries
            reset_queries()

    def __unicode__(self):
        def format_query(q):
            return "Query [%s] %.3f secs" % (truncate(q['sql'], 60), float(q['time']))

        message = "Segment [%s] time: %.3f secs" % (self.name, self.time_total)
        if self.db_profile:
            num_queries = len(self.queries)
            time_db = sum([float(q['time']) for q in self.queries])

            message += ", %.3f secs db time, %d db queries" % (time_db, num_queries)

            # if we have only have a few queries, include them all in order of execution
            if len(self.queries) <= MAX_QUERIES_PRINT:
                message += ":"
                for query in self.queries:
                    message += "\n\t%s" % format_query(query)
            # if there are too many, only include slowest in order of duration
            else:
                message += ". %d slowest:" % MAX_QUERIES_PRINT
                slowest = sorted(list(self.queries), key=lambda q: float(q['time']), reverse=True)[:MAX_QUERIES_PRINT]
                for query in slowest:
                    message += "\n\t%s" % format_query(query)

        return message


class PerformanceTest(TembaTest):
    segments = []

    def setUp(self):
        self.clear_cache()

        self.user = self.create_user("tito")
        self.admin = self.create_user("Administrator")
        self.org = Org.objects.create(name="Nyaruka Ltd.", timezone="Africa/Kigali",
                                      created_by=self.user, modified_by=self.user)
        self.org.initialize()

        self.org.administrators.add(self.admin)
        self.admin.set_org(self.org)
        self.org.administrators.add(self.user)
        self.user.set_org(self.org)

        self.tel_mtn = Channel.objects.create(org=self.org, name="MTN", channel_type="A", role="SR",
                                              address="+250780000000", secret="12345", gcm_id="123",
                                              created_by=self.user, modified_by=self.user)
        self.tel_tigo = Channel.objects.create(org=self.org, name="Tigo", channel_type="A", role="SR",
                                               address="+250720000000", secret="23456", gcm_id="234",
                                               created_by=self.user, modified_by=self.user)
        self.tel_bulk = Channel.objects.create(org=self.org, name="Nexmo", channel_type="NX", role="S",
                                               parent=self.tel_tigo, secret="34567",
                                               created_by=self.user, modified_by=self.user)
        self.twitter = Channel.objects.create(org=self.org, name="Twitter", channel_type="TT", role="SR",
                                              created_by=self.user, modified_by=self.user)

        # for generating tuples of scheme, path and channel
        generate_tel_mtn = lambda num: (TEL_SCHEME, "+25078%07d" % (num + 1), self.tel_mtn)
        generate_tel_tigo = lambda num: (TEL_SCHEME, "+25072%07d" % (num + 1), self.tel_tigo)
        generate_twitter = lambda num: (TWITTER_SCHEME, "tweep_%d" % (num + 1), self.twitter)
        self.urn_generators = (generate_tel_mtn, generate_tel_tigo, generate_twitter)

        self.field_nick = ContactField.get_or_create(self.org, 'nick', 'Nickname', show_in_table=True, value_type=TEXT)
        self.field_age = ContactField.get_or_create(self.org, 'age', 'Age', show_in_table=True, value_type=DECIMAL)

    @classmethod
    def tearDownClass(cls):
        print "\n------------------ Segment Profiles ------------------"
        for segment in cls.segments:
            print unicode(segment)

    def _create_contacts(self, count, base_names):
        """
        Creates the given number of contacts with URNs of each type, and fields value for dob and nickname
        """
        contacts = []

        for c in range(0, count):
            name = '%s %d' % (base_names[c % len(base_names)], c + 1)
            scheme, path, channel = self.urn_generators[c % len(self.urn_generators)](c)
            contacts.append(Contact.get_or_create(self.org, self.user, name, urns=[(scheme, path)]))
        return contacts

    def _create_groups(self, count, base_names, contacts):
        """
        Creates the given number of groups and fills them with contacts
        """
        groups = []
        num_bases = len(base_names)
        for g in range(0, count):
            name = '%s %d' % (base_names[g % num_bases], g + 1)
            group = ContactGroup.create(self.org, self.user, name)
            group.contacts.add(*contacts[(g % num_bases)::num_bases])
            groups.append(group)
        return groups

    def _create_broadcast(self, text, recipients):
        """
        Creates the a single broadcast to the given recipients (which can groups, contacts, URNs)
        """
        return Broadcast.create(self.org, self.user, text, recipients)
<commit_msg>fix perf test to use new contact triggers<commit_after>from __future__ import unicode_literals

import random

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import connection, reset_queries
from django.utils import timezone
from temba.contacts.models import Contact, ContactField, ContactGroup, ContactURN, TEL_SCHEME, TWITTER_SCHEME
from temba.orgs.models import Org
from temba.channels.models import Channel
from temba.flows.models import FlowRun, FlowStep
from temba.msgs.models import Broadcast, Call, Label, Msg, INCOMING, OUTGOING, PENDING
from temba.utils import truncate
from temba.values.models import Value, TEXT, DECIMAL
from tests import TembaTest
from timeit import default_timer


MAX_QUERIES_PRINT = 15


class SegmentProfiler(object):
    """
    Used in a with block to profile a segment of code
    """
    def __init__(self, test, name, db_profile):
        self.test = test
        self.test.segments.append(self)
        self.name = name
        self.db_profile = db_profile
        self.old_debug = settings.DEBUG

        self.time_total = 0.0
        self.time_queries = 0.0
        self.queries = []

    def __enter__(self):
        if self.db_profile:
            settings.DEBUG = True
            reset_queries()

        self.start_time = default_timer()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.time_total = default_timer() - self.start_time

        if self.db_profile:
            settings.DEBUG = self.old_debug
            self.queries = connection.queries
            reset_queries()

    def __unicode__(self):
        def format_query(q):
            return "Query [%s] %.3f secs" % (truncate(q['sql'], 60), float(q['time']))

        message = "Segment [%s] time: %.3f secs" % (self.name, self.time_total)
        if self.db_profile:
            num_queries = len(self.queries)
            time_db = sum([float(q['time']) for q in self.queries])

            message += ", %.3f secs db time, %d db queries" % (time_db, num_queries)

            # if we have only have a few queries, include them all in order of execution
            if len(self.queries) <= MAX_QUERIES_PRINT:
                message += ":"
                for query in self.queries:
                    message += "\n\t%s" % format_query(query)
            # if there are too many, only include slowest in order of duration
            else:
                message += ". %d slowest:" % MAX_QUERIES_PRINT
                slowest = sorted(list(self.queries), key=lambda q: float(q['time']), reverse=True)[:MAX_QUERIES_PRINT]
                for query in slowest:
                    message += "\n\t%s" % format_query(query)

        return message


class PerformanceTest(TembaTest):
    segments = []

    def setUp(self):
        self.clear_cache()

        self.user = self.create_user("tito")
        self.admin = self.create_user("Administrator")
        self.org = Org.objects.create(name="Nyaruka Ltd.", timezone="Africa/Kigali",
                                      created_by=self.user, modified_by=self.user)
        self.org.initialize()

        self.org.administrators.add(self.admin)
        self.admin.set_org(self.org)
        self.org.administrators.add(self.user)
        self.user.set_org(self.org)

        self.tel_mtn = Channel.objects.create(org=self.org, name="MTN", channel_type="A", role="SR",
                                              address="+250780000000", secret="12345", gcm_id="123",
                                              created_by=self.user, modified_by=self.user)
        self.tel_tigo = Channel.objects.create(org=self.org, name="Tigo", channel_type="A", role="SR",
                                               address="+250720000000", secret="23456", gcm_id="234",
                                               created_by=self.user, modified_by=self.user)
        self.tel_bulk = Channel.objects.create(org=self.org, name="Nexmo", channel_type="NX", role="S",
                                               parent=self.tel_tigo, secret="34567",
                                               created_by=self.user, modified_by=self.user)
        self.twitter = Channel.objects.create(org=self.org, name="Twitter", channel_type="TT", role="SR",
                                              created_by=self.user, modified_by=self.user)

        # for generating tuples of scheme, path and channel
        generate_tel_mtn = lambda num: (TEL_SCHEME, "+25078%07d" % (num + 1), self.tel_mtn)
        generate_tel_tigo = lambda num: (TEL_SCHEME, "+25072%07d" % (num + 1), self.tel_tigo)
        generate_twitter = lambda num: (TWITTER_SCHEME, "tweep_%d" % (num + 1), self.twitter)
        self.urn_generators = (generate_tel_mtn, generate_tel_tigo, generate_twitter)

        self.field_nick = ContactField.get_or_create(self.org, 'nick', 'Nickname', show_in_table=True, value_type=TEXT)
        self.field_age = ContactField.get_or_create(self.org, 'age', 'Age', show_in_table=True, value_type=DECIMAL)

    @classmethod
    def tearDownClass(cls):
        print "\n------------------ Segment Profiles ------------------"
        for segment in cls.segments:
            print unicode(segment)

    def _create_contacts(self, count, base_names):
        """
        Creates the given number of contacts with URNs of each type, and fields value for dob and nickname
        """
        contacts = []

        for c in range(0, count):
            name = '%s %d' % (base_names[c % len(base_names)], c + 1)
            scheme, path, channel = self.urn_generators[c % len(self.urn_generators)](c)
            contacts.append(Contact.get_or_create(self.org, self.user, name, urns=[(scheme, path)]))
        return contacts

    def _create_groups(self, count, base_names, contacts):
        """
        Creates the given number of groups and fills them with contacts
        """
        groups = []
        num_bases = len(base_names)
        for g in range(0, count):
            name = '%s %d' % (base_names[g % num_bases], g + 1)
            group = ContactGroup.create(self.org, self.user, name)
            group.contacts.add(*contacts[(g % num_bases)::num_bases])
            groups.append(ContactGroup.user_groups.get(pk=group.pk))
        return groups

    def _create_broadcast(self, text, recipients):
        """
        Creates the a single broadcast to the given recipients (which can groups, contacts, URNs)
        """
        return Broadcast.create(self.org, self.user, text, recipients)

{'fraction_non_alphanumeric': 0.11424172440338722, 'fraction_numerical': 0.011855273287143957, 'mean_word_length': 2.278072167549836, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6158802', 'n_tokens_mistral': 4014, 'n_tokens_neox': 3802, 'n_words': 921}