<commit_before>import time
import gevent
import collections
from datetime import datetime

from sqlalchemy import asc, desc
from sqlalchemy.orm import subqueryload
from inbox.api.kellogs import APIEncoder, encode
from inbox.models import Transaction, Message, Thread
from inbox.models.session import session_scope
from inbox.models.util import transaction_objects


QUERY_OPTIONS = {
    Message: (
        subqueryload('parts').joinedload('block'),
        subqueryload('thread').load_only('public_id', 'discriminator'),
        subqueryload('events').load_only('public_id', 'discriminator'),
        subqueryload('messagecategories').joinedload('category')
    ),
    Thread: (
        subqueryload('messages').load_only(
            'public_id', 'is_draft', 'from_addr', 'to_addr', 'cc_addr',
            'bcc_addr', 'is_read', 'is_starred').
        joinedload('messagecategories').joinedload('category'),
        subqueryload('messages').joinedload('parts').joinedload('block')
    )
}

EVENT_NAME_FOR_COMMAND = {
    'insert': 'create',
    'update': 'modify',
    'delete': 'delete'
}


def get_transaction_cursor_near_timestamp(namespace_id, timestamp, db_session):
    """
    Exchange a timestamp for a 'cursor' into the transaction log entry near
    to that timestamp in age. The cursor is the public_id of that transaction
    (or '0' if there are no such transactions).

    Arguments
    ---------
    namespace_id: int
        Id of the namespace for which to get a cursor.
    timestamp: int
        Unix timestamp
    db_session: InboxSession
        database session

    Returns
    -------
    string
        A transaction public_id that can be passed as a 'cursor' parameter by
        API clients.

    """
    dt = datetime.utcfromtimestamp(timestamp)

    # We want this guarantee: if you pass a timestamp for, say,
    # '2015-03-20 12:22:20', and you have multiple transactions immediately
    # prior, e.g.:
    # id | created_at
    # ---+-----------
    # 23 | 2015-03-20 12:22:19
    # 24 | 2015-03-20 12:22:19
    # 25 | 2015-03-20 12:22:19
    # then you get the last one by id (25). Otherwise you might pass a
    # timestamp far in the future, but not actually get the last cursor.
    # The obvious way to accomplish this is to filter by `created_at` but order
    # by `id`. However, that causes MySQL to perform a potentially expensive
    # filesort. Instead, get transactions with timestamp *matching* the last
    # one before what you have, and sort those by id:
    latest_timestamp = db_session.query(Transaction.created_at). \
        order_by(desc(Transaction.created_at)). \
        filter(Transaction.created_at < dt,
               Transaction.namespace_id == namespace_id).limit(1).subquery()
    latest_transaction = db_session.query(Transaction). \
        filter(Transaction.created_at == latest_timestamp,
               Transaction.namespace_id == namespace_id). \
        order_by(desc(Transaction.id)).first()

    if latest_transaction is None:
        # If there are no earlier deltas, use '0' as a special stamp parameter
        # to signal 'process from the start of the log'.
        return '0'

    return latest_transaction.public_id


def format_transactions_after_pointer(namespace, pointer, db_session,
                                      result_limit, exclude_types=None,
                                      include_types=None,
                                      exclude_folders=True):
    """
    Return a pair (deltas, new_pointer), where deltas is a list of change
    events, represented as dictionaries:
    {
      "object": <API object type, e.g. "thread">,
      "event": <"create", "modify", or "delete>,
      "attributes": <API representation of the object for insert/update events>
      "cursor": <public_id of the transaction>
    }

    and new_pointer is the integer id of the last included transaction

    Arguments
    ---------
    namespace_id: int
        Id of the namespace for which to get changes.
    pointer: int
        Process transactions starting after this id.
    db_session: InboxSession
        database session
    result_limit: int
        Maximum number of results to return. (Because we may roll up multiple
        changes to the same object, fewer results can be returned.)
    format_transaction_fn: function pointer
        Function that defines how to format the transactions.
    exclude_types: list, optional
        If given, don't include transactions for these types of objects.

    """
    # Begin backwards-compatibility shim -- suppress new object types for now,
    # because clients may not be able to deal with them.
    exclude_types = exclude_types or []
    if exclude_folders is True:
        exclude_types.extend(('folder', 'label'))
    # End backwards-compatibility shim.

    while True:
        # deleted_at condition included to allow this query to be satisfied via
        # the legacy index on (namespace_id, deleted_at) for performance.
        # Also need to explicitly specify the index hint because the query
        # planner is dumb as nails and otherwise would make this super slow for
        # some values of namespace_id and pointer.
        # TODO(emfree): Remove this hack and ensure that the right index (on
        # namespace_id only) exists.
        transactions = db_session.query(Transaction). \
            filter(
                Transaction.id > pointer,
                Transaction.namespace_id == namespace.id,<commit_msg>Don't repeatedly add folders/labels to delta sync exclude_types.<commit_after>import time
import gevent
import collections
from datetime import datetime

from sqlalchemy import asc, desc
from sqlalchemy.orm import subqueryload
from inbox.api.kellogs import APIEncoder, encode
from inbox.models import Transaction, Message, Thread
from inbox.models.session import session_scope
from inbox.models.util import transaction_objects


QUERY_OPTIONS = {
    Message: (
        subqueryload('parts').joinedload('block'),
        subqueryload('thread').load_only('public_id', 'discriminator'),
        subqueryload('events').load_only('public_id', 'discriminator'),
        subqueryload('messagecategories').joinedload('category')
    ),
    Thread: (
        subqueryload('messages').load_only(
            'public_id', 'is_draft', 'from_addr', 'to_addr', 'cc_addr',
            'bcc_addr', 'is_read', 'is_starred').
        joinedload('messagecategories').joinedload('category'),
        subqueryload('messages').joinedload('parts').joinedload('block')
    )
}

EVENT_NAME_FOR_COMMAND = {
    'insert': 'create',
    'update': 'modify',
    'delete': 'delete'
}


def get_transaction_cursor_near_timestamp(namespace_id, timestamp, db_session):
    """
    Exchange a timestamp for a 'cursor' into the transaction log entry near
    to that timestamp in age. The cursor is the public_id of that transaction
    (or '0' if there are no such transactions).

    Arguments
    ---------
    namespace_id: int
        Id of the namespace for which to get a cursor.
    timestamp: int
        Unix timestamp
    db_session: InboxSession
        database session

    Returns
    -------
    string
        A transaction public_id that can be passed as a 'cursor' parameter by
        API clients.

    """
    dt = datetime.utcfromtimestamp(timestamp)

    # We want this guarantee: if you pass a timestamp for, say,
    # '2015-03-20 12:22:20', and you have multiple transactions immediately
    # prior, e.g.:
    # id | created_at
    # ---+-----------
    # 23 | 2015-03-20 12:22:19
    # 24 | 2015-03-20 12:22:19
    # 25 | 2015-03-20 12:22:19
    # then you get the last one by id (25). Otherwise you might pass a
    # timestamp far in the future, but not actually get the last cursor.
    # The obvious way to accomplish this is to filter by `created_at` but order
    # by `id`. However, that causes MySQL to perform a potentially expensive
    # filesort. Instead, get transactions with timestamp *matching* the last
    # one before what you have, and sort those by id:
    latest_timestamp = db_session.query(Transaction.created_at). \
        order_by(desc(Transaction.created_at)). \
        filter(Transaction.created_at < dt,
               Transaction.namespace_id == namespace_id).limit(1).subquery()
    latest_transaction = db_session.query(Transaction). \
        filter(Transaction.created_at == latest_timestamp,
               Transaction.namespace_id == namespace_id). \
        order_by(desc(Transaction.id)).first()

    if latest_transaction is None:
        # If there are no earlier deltas, use '0' as a special stamp parameter
        # to signal 'process from the start of the log'.
        return '0'

    return latest_transaction.public_id


def format_transactions_after_pointer(namespace, pointer, db_session,
                                      result_limit, exclude_types=None,
                                      include_types=None,
                                      exclude_folders=True):
    """
    Return a pair (deltas, new_pointer), where deltas is a list of change
    events, represented as dictionaries:
    {
      "object": <API object type, e.g. "thread">,
      "event": <"create", "modify", or "delete>,
      "attributes": <API representation of the object for insert/update events>
      "cursor": <public_id of the transaction>
    }

    and new_pointer is the integer id of the last included transaction

    Arguments
    ---------
    namespace_id: int
        Id of the namespace for which to get changes.
    pointer: int
        Process transactions starting after this id.
    db_session: InboxSession
        database session
    result_limit: int
        Maximum number of results to return. (Because we may roll up multiple
        changes to the same object, fewer results can be returned.)
    format_transaction_fn: function pointer
        Function that defines how to format the transactions.
    exclude_types: list, optional
        If given, don't include transactions for these types of objects.

    """
    # Begin backwards-compatibility shim -- suppress new object types for now,
    # because clients may not be able to deal with them.
    exclude_types = set(exclude_types) if exclude_types else set()
    if exclude_folders is True:
        exclude_types.update(('folder', 'label'))
    # End backwards-compatibility shim.

    while True:
        # deleted_at condition included to allow this query to be satisfied via
        # the legacy index on (namespace_id, deleted_at) for performance.
        # Also need to explicitly specify the index hint because the query
        # planner is dumb as nails and otherwise would make this super slow for
        # some values of namespace_id and pointer.
        # TODO(emfree): Remove this hack and ensure that the right index (on
        # namespace_id only) exists.
        transactions = db_session.query(Transaction). \
            filter(
                Transaction.id > pointer,
                Transaction.namespace_id == namespace.id,
{'fraction_non_alphanumeric': 0.08690540784952071, 'fraction_numerical': 0.012298788207632484, 'mean_word_length': 2.7110738255033557, 'pattern_counts': {'":': 8, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '104254', 'n_tokens_mistral': 3014, 'n_tokens_neox': 2876, 'n_words': 1051}