<commit_before>import json
import datetime
import calendar
from sqlalchemy import desc
from inbox.models import Message, Thread, Namespace, Block, Category
from inbox.util.misc import dt_to_timestamp
from tests.util.base import (test_client, add_fake_message,
                             add_fake_thread)
from tests.api.base import api_client

__all__ = ['api_client', 'test_client']


def test_filtering(db, api_client, default_namespace):
    thread = add_fake_thread(db.session, default_namespace.id)
    message = add_fake_message(db.session, default_namespace.id, thread,
                               to_addr=[('Bob', '<EMAIL>')],
                               from_addr=[('Alice', '<EMAIL>')],
                               subject='some subject')
    message.categories.add(
        Category(namespace_id=message.namespace_id,
                 name='inbox', display_name='Inbox', type_='label'))
    thread.subject = message.subject
    db.session.commit()

    t_start = dt_to_timestamp(thread.subjectdate)
    t_lastmsg = dt_to_timestamp(thread.recentdate)
    subject = message.subject
    to_addr = message.to_addr[0][1]
    from_addr = message.from_addr[0][1]
    received_date = message.received_date
    unread = not message.is_read
    starred = message.is_starred

    results = api_client.get_data('/threads?thread_id={}'
                                  .format(thread.public_id))
    assert len(results) == 1

    results = api_client.get_data('/messages?thread_id={}'
                                  .format(thread.public_id))
    assert len(results) == 1

    results = api_client.get_data('/threads?cc={}'
                                  .format(message.cc_addr))
    assert len(results) == 0

    results = api_client.get_data('/messages?cc={}'
                                  .format(message.cc_addr))
    assert len(results) == 0

    results = api_client.get_data('/threads?bcc={}'
                                  .format(message.bcc_addr))
    assert len(results) == 0

    results = api_client.get_data('/messages?bcc={}'
                                  .format(message.bcc_addr))
    assert len(results) == 0

    results = api_client.get_data('/threads?filename=test')
    assert len(results) == 0

    results = api_client.get_data('/messages?filename=test')
    assert len(results) == 0

    results = api_client.get_data('/threads?started_after={}'
                                  .format(t_start - 1))
    assert len(results) == 1

    results = api_client.get_data('/messages?started_after={}'
                                  .format(t_start - 1))
    assert len(results) == 1

    results = api_client.get_data('/messages?last_message_before={}&limit=1'
                                  .format(t_lastmsg + 1))
    assert len(results) == 1

    results = api_client.get_data('/threads?last_message_before={}&limit=1'
                                  .format(t_lastmsg + 1))
    assert len(results) == 1

    results = api_client.get_data('/threads?in=inbox&limit=1')
    assert len(results) == 1

    results = api_client.get_data('/messages?in=inbox&limit=1')
    assert len(results) == 1

    results = api_client.get_data('/messages?in=banana%20rama')
    assert len(results) == 0

    results = api_client.get_data('/threads?subject={}'.format(subject))
    assert len(results) == 1

    results = api_client.get_data('/messages?subject={}'.format(subject))
    assert len(results) == 1

    results = api_client.get_data('/threads?unread={}'.format(unread))
    assert len(results) == 1

    results = api_client.get_data('/messages?unread={}'.format((not unread)))
    assert len(results) == 0

    results = api_client.get_data('/threads?starred={}'.format((not starred)))
    assert len(results) == 0

    results = api_client.get_data('/messages?starred={}'.format(starred))
    assert len(results) == 1

    for _ in range(3):
        add_fake_message(db.session, default_namespace.id,
                         to_addr=[('', '<EMAIL>')],
                         thread=add_fake_thread(db.session,
                                                default_namespace.id))

    results = api_client.get_data('/messages?any_email={}'.
                                  format('<EMAIL>'))
    assert len(results) > 1

    # Test multiple any_email params
    multiple_results = api_client.get_data('/messages?any_email={},{},{}'.
                                           format('<EMAIL>',
                                                  '<EMAIL>',
                                                  '<EMAIL>'))
    assert len(multiple_results) > len(results)

    # Check that we canonicalize when searching.
    alternate_results = api_client.get_data('/threads?any_email={}'.
                                            format('<EMAIL>'))
    assert len(alternate_results) == len(results)

    results = api_client.get_data('/messages?from={}'.format(from_addr))
    assert len(results) == 1
    results = api_client.get_data('/threads?from={}'.format(from_addr))
    assert len(results) == 1

    early_time = received_date - datetime.timedelta(seconds=1)
    late_time = received_date + datetime.timedelta(seconds=1)
    early_ts = calendar.timegm(early_time.utctimetuple())
    late_ts = calendar.timegm(late_time.utctimetuple())

    results = api_client.get_data('/messages?subject={}&started_before={}'.
                                  format(subject, early_ts))
    assert len(results) == 0
    results = api_client.get_data('/threads?subject={}&started_before={}'.
                                  format(subject, early_ts))
    assert len(results) == 0

    results = api_client.get_data('/messages?subject={}&started_before={}'.
                                  format(subject, late_ts))
    assert len(results) == 1
    results = api_client.get_data('/threads?subject={}&started_before={}'.
                                  format(subject, late_ts))
    assert len(results) == 1

    results = api_client.get_data('/messages?subject={}&last_message_after={}'.
                                  format(subject, early_ts))
    assert len(results) == 1
    results = api_client.get_data('/threads?subject={}&last_message_after={}'.
                                  format(subject, early_ts))
    assert len(results) == 1

    results = api_client.get_data('/messages?subject={}&last_message_after={}'.
                                  format(subject, late_ts))
    assert len(results) == 0
    results = api_client.get_data('/threads?subject={}&last_message_after={}'.
                                  format(subject, late_ts))
    assert len(results) == 0

    results = api_client.get_data('/messages?subject={}&started_before={}'.
                                  format(subject, early_ts))
    assert len(results) == 0
    results = api_client.get_data('/threads?subject={}&started_before={}'.
                                  format(subject, early_ts))
    assert len(results) == 0

    results = api_client.get_data('/messages?subject={}&started_before={}'.
                                  format(subject, late_ts))
    assert len(results) == 1
    results = api_client.get_data('/threads?subject={}&started_before={}'.
                                  format(subject, late_ts))
    assert len(results) == 1

    results = api_client.get_data('/messages?from={}&to={}'.
                                  format(from_addr, to_addr))
    assert len(results) == 1

    results = api_client.get_data('/threads?from={}&to={}'.
                                  format(from_addr, to_addr))
    assert len(results) == 1

    results = api_client.get_data('/messages?to={}&limit={}&offset={}'.
                                  format('<EMAIL>', 2, 1))
    assert len(results) == 2

    results = api_client.get_data('/threads?to={}&limit={}'.
                                  format('<EMAIL>', 3))
    assert len(results) == 3

    results = api_client.get_data('/threads?view=count')

    assert results['count'] == 4

    results = api_client.get_data('/threads?view=ids&to={}&limit=3'.
                                  format('<EMAIL>', 3))

    assert len(results) == 3
    assert all(isinstance(r, basestring)
               for r in results), "Returns a list of string"


def test_query_target(db, api_client, thread, default_namespace):
    cat = Category(namespace_id=default_namespace.id,
                   name='inbox', display_name='Inbox', type_='label')
    for _ in range(3):
        message = add_fake_message(db.session, default_namespace.id, thread,
                                   to_addr=[('Bob', '<EMAIL>')],
                                   from_addr=[('Alice', '<EMAIL>')],
                                   subject='some subject')
        message.categories.add(cat)
    db.session.commit()

    results = api_client.get_data('/messages?in=inbox')
    assert len(results) == 3

    count = api_client.get_data('/messages?in=inbox&view=count')
    assert count['count'] == 3


def test_ordering(api_client, db, default_namespace):
    for i in range(3):
        thr = add_fake_thread(db.session, default_namespace.id)
        received_date = (datetime.datetime.utcnow() +
                         datetime.timedelta(seconds=22 * (i + 1)))
        add_fake_message(db.session, default_namespace.id,
                         thr, received_date=received_date)
    ordered_results = api_client.get_data('/messages')
    ordered_dates = [result['date'] for result in ordered_results]
    assert ordered_dates == sorted(ordered_dates, reverse=True)

    ordered_results = api_client.get_data('/messages?limit=3')
    expected_public_ids = [
        public_id for public_id, in
        db.session.query(Message.public_id).
        filter(Message.namespace_id == default_namespace.id).
        order_by(desc(Message.received_date)).limit(3)]
    assert expected_public_ids == [r['id'] for r in ordered_results]


def test_strict_argument_parsing(api_client):
    r = api_client.get_raw('/threads?foo=bar')
    assert r.status_code == 400


def test_distinct_results(api_client, db, default_namespace):
    """Test that limit and offset parameters work correctly when joining on
    multiple matching messages per thread."""
    # Create a thread with multiple messages on it.
    first_thread = add_fake_thread(db.session, default_namespace.id)
    add_fake_message(db.session, default_namespace.id, first_thread,
                     from_addr=[('', '<EMAIL>')],
                     received_date=datetime.datetime.utcnow())
    add_fake_message(db.session, default_namespace.id, first_thread,
                     from_addr=[('', '<EMAIL>')],
                     received_date=datetime.datetime.utcnow())

    # Now create another thread with the same participants
    older_date = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
    second_thread = add_fake_thread(db.session, default_namespace.id)
    add_fake_message(db.session, default_namespace.id, second_thread,
                     from_addr=[('', '<EMAIL>')],
                     received_date=older_date)
    add_fake_message(db.session, default_namespace.id, second_thread,
                     from_addr=[('', '<EMAIL>')],
                     received_date=older_date)
    second_thread.recentdate = older_date
    db.session.commit()

    filtered_results = api_client.get_data('/threads?from=<EMAIL>'
                                           '&limit=1&offset=0')
    assert len(filtered_results) == 1
    assert filtered_results[0]['id'] == first_thread.public_id

    filtered_results = api_client.get_data('/threads?from=<EMAIL>'
                                           '&limit=1&offset=1')
    assert len(filtered_results) == 1
    assert filtered_results[0]['id'] == second_thread.public_id

    filtered_results = api_client.get_data('/threads?from=<EMAIL>'
                                           '&limit=2&offset=0')
    assert len(filtered_results) == 2

    filtered_results = api_client.get_data('/threads?from=<EMAIL>'
                                           '&limit=2&offset=1')
    assert len(filtered_results) == 1


def test_filtering_accounts(db, test_client, default_namespace):
    all_accounts = json.loads(test_client.get('/accounts/?limit=100').data)
    email = all_accounts[0]['email_address']

    some_accounts = json.loads(test_client.get('/accounts/?offset=1&limit=99').data)
    assert len(some_accounts) == len(all_accounts) - 1

    no_all_accounts = json.loads(test_client.get('/accounts/?limit=0').data)
    assert no_all_accounts == []

    all_accounts = json.loads(test_client.get('/accounts/?limit=1').data)
    assert len(all_accounts) == 1

    filter_ = '?email_address={}'.format(email)
    all_accounts = json.loads(test_client.get('/accounts/' + filter_).data)
    assert all_accounts[0]['email_address'] == email

    filter_ = '?email_address=<EMAIL>'
    accounts = json.loads(test_client.get('/accounts/' + filter_).data)
    assert len(accounts) == 0<commit_msg>Thread count test<commit_after>import json
import datetime
import calendar
from sqlalchemy import desc
from inbox.models import Message, Thread, Namespace, Block, Category
from inbox.util.misc import dt_to_timestamp
from tests.util.base import (test_client, add_fake_message,
                             add_fake_thread)
from tests.api.base import api_client

__all__ = ['api_client', 'test_client']


def test_filtering(db, api_client, default_namespace):
    thread = add_fake_thread(db.session, default_namespace.id)
    message = add_fake_message(db.session, default_namespace.id, thread,
                               to_addr=[('Bob', '<EMAIL>')],
                               from_addr=[('Alice', '<EMAIL>')],
                               subject='some subject')
    message.categories.add(
        Category(namespace_id=message.namespace_id,
                 name='inbox', display_name='Inbox', type_='label'))
    thread.subject = message.subject
    db.session.commit()

    t_start = dt_to_timestamp(thread.subjectdate)
    t_lastmsg = dt_to_timestamp(thread.recentdate)
    subject = message.subject
    to_addr = message.to_addr[0][1]
    from_addr = message.from_addr[0][1]
    received_date = message.received_date
    unread = not message.is_read
    starred = message.is_starred

    results = api_client.get_data('/threads?thread_id={}'
                                  .format(thread.public_id))
    assert len(results) == 1

    results = api_client.get_data('/messages?thread_id={}'
                                  .format(thread.public_id))
    assert len(results) == 1

    results = api_client.get_data('/threads?cc={}'
                                  .format(message.cc_addr))
    assert len(results) == 0

    results = api_client.get_data('/messages?cc={}'
                                  .format(message.cc_addr))
    assert len(results) == 0

    results = api_client.get_data('/threads?bcc={}'
                                  .format(message.bcc_addr))
    assert len(results) == 0

    results = api_client.get_data('/messages?bcc={}'
                                  .format(message.bcc_addr))
    assert len(results) == 0

    results = api_client.get_data('/threads?filename=test')
    assert len(results) == 0

    results = api_client.get_data('/messages?filename=test')
    assert len(results) == 0

    results = api_client.get_data('/threads?started_after={}'
                                  .format(t_start - 1))
    assert len(results) == 1

    results = api_client.get_data('/messages?started_after={}'
                                  .format(t_start - 1))
    assert len(results) == 1

    results = api_client.get_data('/messages?last_message_before={}&limit=1'
                                  .format(t_lastmsg + 1))
    assert len(results) == 1

    results = api_client.get_data('/threads?last_message_before={}&limit=1'
                                  .format(t_lastmsg + 1))
    assert len(results) == 1

    results = api_client.get_data('/threads?in=inbox&limit=1')
    assert len(results) == 1

    results = api_client.get_data('/messages?in=inbox&limit=1')
    assert len(results) == 1

    results = api_client.get_data('/messages?in=banana%20rama')
    assert len(results) == 0

    results = api_client.get_data('/threads?subject={}'.format(subject))
    assert len(results) == 1

    results = api_client.get_data('/messages?subject={}'.format(subject))
    assert len(results) == 1

    results = api_client.get_data('/threads?unread={}'.format(unread))
    assert len(results) == 1

    results = api_client.get_data('/messages?unread={}'.format((not unread)))
    assert len(results) == 0

    results = api_client.get_data('/threads?starred={}'.format((not starred)))
    assert len(results) == 0

    results = api_client.get_data('/messages?starred={}'.format(starred))
    assert len(results) == 1

    for _ in range(3):
        add_fake_message(db.session, default_namespace.id,
                         to_addr=[('', '<EMAIL>')],
                         thread=add_fake_thread(db.session,
                                                default_namespace.id))

    results = api_client.get_data('/messages?any_email={}'.
                                  format('<EMAIL>'))
    assert len(results) > 1

    # Test multiple any_email params
    multiple_results = api_client.get_data('/messages?any_email={},{},{}'.
                                           format('<EMAIL>',
                                                  '<EMAIL>',
                                                  '<EMAIL>'))
    assert len(multiple_results) > len(results)

    # Check that we canonicalize when searching.
    alternate_results = api_client.get_data('/threads?any_email={}'.
                                            format('<EMAIL>'))
    assert len(alternate_results) == len(results)

    results = api_client.get_data('/messages?from={}'.format(from_addr))
    assert len(results) == 1
    results = api_client.get_data('/threads?from={}'.format(from_addr))
    assert len(results) == 1

    early_time = received_date - datetime.timedelta(seconds=1)
    late_time = received_date + datetime.timedelta(seconds=1)
    early_ts = calendar.timegm(early_time.utctimetuple())
    late_ts = calendar.timegm(late_time.utctimetuple())

    results = api_client.get_data('/messages?subject={}&started_before={}'.
                                  format(subject, early_ts))
    assert len(results) == 0
    results = api_client.get_data('/threads?subject={}&started_before={}'.
                                  format(subject, early_ts))
    assert len(results) == 0

    results = api_client.get_data('/messages?subject={}&started_before={}'.
                                  format(subject, late_ts))
    assert len(results) == 1
    results = api_client.get_data('/threads?subject={}&started_before={}'.
                                  format(subject, late_ts))
    assert len(results) == 1

    results = api_client.get_data('/messages?subject={}&last_message_after={}'.
                                  format(subject, early_ts))
    assert len(results) == 1
    results = api_client.get_data('/threads?subject={}&last_message_after={}'.
                                  format(subject, early_ts))
    assert len(results) == 1

    results = api_client.get_data('/messages?subject={}&last_message_after={}'.
                                  format(subject, late_ts))
    assert len(results) == 0
    results = api_client.get_data('/threads?subject={}&last_message_after={}'.
                                  format(subject, late_ts))
    assert len(results) == 0

    results = api_client.get_data('/messages?subject={}&started_before={}'.
                                  format(subject, early_ts))
    assert len(results) == 0
    results = api_client.get_data('/threads?subject={}&started_before={}'.
                                  format(subject, early_ts))
    assert len(results) == 0

    results = api_client.get_data('/messages?subject={}&started_before={}'.
                                  format(subject, late_ts))
    assert len(results) == 1
    results = api_client.get_data('/threads?subject={}&started_before={}'.
                                  format(subject, late_ts))
    assert len(results) == 1

    results = api_client.get_data('/messages?from={}&to={}'.
                                  format(from_addr, to_addr))
    assert len(results) == 1

    results = api_client.get_data('/threads?from={}&to={}'.
                                  format(from_addr, to_addr))
    assert len(results) == 1

    results = api_client.get_data('/messages?to={}&limit={}&offset={}'.
                                  format('<EMAIL>', 2, 1))
    assert len(results) == 2

    results = api_client.get_data('/threads?to={}&limit={}'.
                                  format('<EMAIL>', 3))
    assert len(results) == 3

    results = api_client.get_data('/threads?view=count')

    assert results['count'] == 4

    results = api_client.get_data('/threads?view=ids&to={}&limit=3'.
                                  format('<EMAIL>', 3))

    assert len(results) == 3
    assert all(isinstance(r, basestring)
               for r in results), "Returns a list of string"


def test_query_target(db, api_client, thread, default_namespace):
    cat = Category(namespace_id=default_namespace.id,
                   name='inbox', display_name='Inbox', type_='label')
    for _ in range(3):
        message = add_fake_message(db.session, default_namespace.id, thread,
                                   to_addr=[('Bob', '<EMAIL>')],
                                   from_addr=[('Alice', '<EMAIL>')],
                                   subject='some subject')
        message.categories.add(cat)
    db.session.commit()

    results = api_client.get_data('/messages?in=inbox')
    assert len(results) == 3

    count = api_client.get_data('/messages?in=inbox&view=count')
    assert count['count'] == 3


def test_ordering(api_client, db, default_namespace):
    for i in range(3):
        thr = add_fake_thread(db.session, default_namespace.id)
        received_date = (datetime.datetime.utcnow() +
                         datetime.timedelta(seconds=22 * (i + 1)))
        add_fake_message(db.session, default_namespace.id,
                         thr, received_date=received_date)
    ordered_results = api_client.get_data('/messages')
    ordered_dates = [result['date'] for result in ordered_results]
    assert ordered_dates == sorted(ordered_dates, reverse=True)

    ordered_results = api_client.get_data('/messages?limit=3')
    expected_public_ids = [
        public_id for public_id, in
        db.session.query(Message.public_id).
        filter(Message.namespace_id == default_namespace.id).
        order_by(desc(Message.received_date)).limit(3)]
    assert expected_public_ids == [r['id'] for r in ordered_results]


def test_strict_argument_parsing(api_client):
    r = api_client.get_raw('/threads?foo=bar')
    assert r.status_code == 400


def test_distinct_results(api_client, db, default_namespace):
    """Test that limit and offset parameters work correctly when joining on
    multiple matching messages per thread."""
    # Create a thread with multiple messages on it.
    first_thread = add_fake_thread(db.session, default_namespace.id)
    add_fake_message(db.session, default_namespace.id, first_thread,
                     from_addr=[('', '<EMAIL>')],
                     received_date=datetime.datetime.utcnow(),
                     add_sent_category=True)
    add_fake_message(db.session, default_namespace.id, first_thread,
                     from_addr=[('', '<EMAIL>')],
                     received_date=datetime.datetime.utcnow(),
                     add_sent_category=True)

    # Now create another thread with the same participants
    older_date = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
    second_thread = add_fake_thread(db.session, default_namespace.id)
    add_fake_message(db.session, default_namespace.id, second_thread,
                     from_addr=[('', '<EMAIL>')],
                     received_date=older_date,
                     add_sent_category=True)
    add_fake_message(db.session, default_namespace.id, second_thread,
                     from_addr=[('', '<EMAIL>')],
                     received_date=older_date,
                     add_sent_category=True)

    second_thread.recentdate = older_date
    db.session.commit()

    filtered_results = api_client.get_data('/threads?from=<EMAIL>'
                                           '&limit=1&offset=0')
    assert len(filtered_results) == 1
    assert filtered_results[0]['id'] == first_thread.public_id

    filtered_results = api_client.get_data('/threads?from=<EMAIL>'
                                           '&limit=1&offset=1')
    assert len(filtered_results) == 1
    assert filtered_results[0]['id'] == second_thread.public_id

    filtered_results = api_client.get_data('/threads?from=<EMAIL>'
                                           '&limit=2&offset=0')
    assert len(filtered_results) == 2

    filtered_results = api_client.get_data('/threads?from=<EMAIL>'
                                           '&limit=2&offset=1')
    assert len(filtered_results) == 1

    # Ensure that it works when using the _in filter
    filtered_results = api_client.get_data('/threads?in=sent'
                                           '&limit=2&offset=0')
    assert len(filtered_results) == 2

    filtered_results = api_client.get_data('/threads?in=sent'
                                           '&limit=1&offset=0')
    assert len(filtered_results) == 1


def test_filtering_accounts(db, test_client, default_namespace):
    all_accounts = json.loads(test_client.get('/accounts/?limit=100').data)
    email = all_accounts[0]['email_address']

    some_accounts = json.loads(test_client.get('/accounts/?offset=1&limit=99').data)
    assert len(some_accounts) == len(all_accounts) - 1

    no_all_accounts = json.loads(test_client.get('/accounts/?limit=0').data)
    assert no_all_accounts == []

    all_accounts = json.loads(test_client.get('/accounts/?limit=1').data)
    assert len(all_accounts) == 1

    filter_ = '?email_address={}'.format(email)
    all_accounts = json.loads(test_client.get('/accounts/' + filter_).data)
    assert all_accounts[0]['email_address'] == email

    filter_ = '?email_address=<EMAIL>'
    accounts = json.loads(test_client.get('/accounts/' + filter_).data)
    assert len(accounts) == 0
{'fraction_non_alphanumeric': 0.13009628403791185, 'fraction_numerical': 0.008048743794192868, 'mean_word_length': 2.41497559722579, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 102, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6483345', 'n_tokens_mistral': 7891, 'n_tokens_neox': 7482, 'n_words': 1280}