<commit_before>import pytz
import logging
from dateutil.parser import parse
from datetime import datetime, timedelta

from modularodm import Q

from website.app import init_app
from website.models import User, NodeLog
from scripts.analytics.base import SummaryAnalytics

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

LOG_THRESHOLD = 11


# Modified from scripts/analytics/depth_users.py
def count_user_logs(user):
    logs = NodeLog.find(Q('user', 'eq', user._id))
    length = logs.count()
    if length == LOG_THRESHOLD:
        item = logs[0]
        if item.action == 'project_created' and item.node.is_bookmark_collection:
            length -= 1
    return length


# Modified from scripts/analytics/depth_users.py
def get_number_of_depth_users(active_users):
    depth_users = 0
    for user in active_users:
        log_count = count_user_logs(user)
        if log_count >= LOG_THRESHOLD:
            depth_users += 1

    return depth_users


class UserSummary(SummaryAnalytics):

    @property
    def collection_name(self):
        return 'user_summary'

    def get_events(self, date):
        super(UserSummary, self).get_events(date)

        # Convert to a datetime at midnight for queries and the timestamp
        timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
        query_datetime = timestamp_datetime + timedelta(1)

        active_users = User.find(
                    Q('is_registered', 'eq', True) &
                    Q('password', 'ne', None) &
                    Q('merged_by', 'eq', None) &
                    Q('date_disabled', 'eq', None) &
                    Q('date_confirmed', 'ne', None) &
                    Q('date_confirmed', 'lt', query_datetime)
                )

        depth_users = get_number_of_depth_users(active_users)

        counts = {
            'keen': {
                'timestamp': timestamp_datetime.isoformat()
            },
            'status': {
                'active': active_users.count(),
                'depth': depth_users,
                'unconfirmed': User.find(
                    Q('date_registered', 'lt', query_datetime) &
                    Q('date_confirmed', 'eq', None)
                ).count(),
                'deactivated': User.find(
                    Q('date_disabled', 'ne', None) &
                    Q('date_disabled', 'lt', query_datetime)
                ).count(),
                'merged': User.find(
                    Q('date_registered', 'lt', query_datetime) &
                    Q('merged_by', 'ne', None)
                ).count()
            }
        }<commit_msg>Paginate for user_summary for active and depth users<commit_after>import pytz
import logging
from dateutil.parser import parse
from datetime import datetime, timedelta

from modularodm import Q

from website.app import init_app
from website.models import User, NodeLog
from framework.mongo.utils import paginated
from scripts.analytics.base import SummaryAnalytics

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

LOG_THRESHOLD = 11


# Modified from scripts/analytics/depth_users.py
def count_user_logs(user):
    logs = NodeLog.find(Q('user', 'eq', user._id))
    length = logs.count()
    if length == LOG_THRESHOLD:
        item = logs[0]
        if item.action == 'project_created' and item.node.is_bookmark_collection:
            length -= 1
    return length


class UserSummary(SummaryAnalytics):

    @property
    def collection_name(self):
        return 'user_summary'

    def get_events(self, date):
        super(UserSummary, self).get_events(date)

        # Convert to a datetime at midnight for queries and the timestamp
        timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
        query_datetime = timestamp_datetime + timedelta(1)

        active_user_query = (
            Q('is_registered', 'eq', True) &
            Q('password', 'ne', None) &
            Q('merged_by', 'eq', None) &
            Q('date_disabled', 'eq', None) &
            Q('date_confirmed', 'ne', None) &
            Q('date_confirmed', 'lt', query_datetime)
        )

        active_users = 0
        depth_users = 0
        user_pages = paginated(User, query=active_user_query, increment=200, each=True)
        for user in user_pages:
            active_users += 1
            log_count = count_user_logs(user)
            if log_count >= LOG_THRESHOLD:
                depth_users += 1

        counts = {
            'keen': {
                'timestamp': timestamp_datetime.isoformat()
            },
            'status': {
                'active': active_users,
                'depth': depth_users,
                'unconfirmed': User.find(
                    Q('date_registered', 'lt', query_datetime) &
                    Q('date_confirmed', 'eq', None)
                ).count(),
                'deactivated': User.find(
                    Q('date_disabled', 'ne', None) &
                    Q('date_disabled', 'lt', query_datetime)
                ).count(),
                'merged': User.find(
                    Q('date_registered', 'lt', query_datetime) &
                    Q('merged_by', 'ne', None)
                ).count()
            }
        }
{'fraction_non_alphanumeric': 0.10762839879154079, 'fraction_numerical': 0.0035876132930513596, 'mean_word_length': 2.0600808781051416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2116411', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1454, 'n_words': 346}