<commit_before>import time
import logging
import argparse
from modularodm import Q
from datetime import datetime
from dateutil.parser import parse

from website.app import init_app
from website.project.model import NodeLog
from website.settings import KEEN as keen_settings
from keen.client import KeenClient

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def get_node_log_events(end_date, start_date=None):

    node_log_query = Q('date', 'lt', end_date)

    if start_date:
        node_log_query = node_log_query & Q('date', 'gt', start_date)

    node_logs = NodeLog.find(node_log_query)
    node_log_events = []
    for node_log in node_logs:
        event = {
            'keen': {'timestamp': end_date.isoformat()},
            'date': node_log.date.isoformat(),
            'action': node_log.action
        }

        if node_log.user:
            event.update({'user_id': node_log.user._id})

        node_log_events.append(event)

    return node_log_events


def parse_args():
    parser = argparse.ArgumentParser(description='Get node log counts!')
    parser.add_argument('-e', '--end', dest='end_date', required=False)

    return parser.parse_args()


def main():
    args = parse_args()
    end_date = parse(args.end_date) if args.end_date else datetime.today()
    start_date = parse(args.start_date) if args.start_date else None

    node_log_events = get_node_log_events(end_date, start_date)
    keen_project = keen_settings['private']['project_id']
    write_key = keen_settings['private']['write_key']
    if keen_project and write_key:
        client = KeenClient(
            project_id=keen_project,
            write_key=write_key,
        )

        start, stop = 0, 5000  # Keen API Docs suggest sending in batches of no more than 5K
        while start <= len(node_log_events):
            keen_payload = {'node_log_analytics': node_log_events[start:stop]}
            client.add_events(keen_payload)
            start += 5000
            stop += 5000
            time.sleep(1)
    else:
        print(node_log_events)

    logger.info('NodeLogs counted. {} NodeLogs.'.format(len(node_log_events)))

if __name__ == '__main__':
    init_app()
    main()<commit_msg>Update start and end date for node log count script<commit_after>import time
import logging
import argparse
from modularodm import Q
from datetime import datetime
from dateutil.parser import parse

from website.app import init_app
from website.project.model import NodeLog
from website.settings import KEEN as keen_settings
from keen.client import KeenClient

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)


def get_node_log_events(start_date, end_date):

    node_log_query = Q('date', 'lt', end_date) & Q('date', 'gt', start_date)

    node_logs = NodeLog.find(node_log_query)
    node_log_events = []
    for node_log in node_logs:
        event = {
            'keen': {'timestamp': end_date.isoformat()},
            'date': node_log.date.isoformat(),
            'action': node_log.action
        }

        if node_log.user:
            event.update({'user_id': node_log.user._id})

        node_log_events.append(event)

    return node_log_events


def parse_args():
    parser = argparse.ArgumentParser(description='Get node log counts!')
    parser.add_argument('-e', '--end', dest='end_date', required=False)

    return parser.parse_args()


def main():
    now = datetime.datetime.utcnow()
    args = parse_args()
    end_date = parse(args.end_date) if args.end_date else now
    start_date = parse(args.start_date) if args.start_date else now - datetime.timedelta(1)

    node_log_events = get_node_log_events(start_date, end_date)
    keen_project = keen_settings['private']['project_id']
    write_key = keen_settings['private']['write_key']
    if keen_project and write_key:
        client = KeenClient(
            project_id=keen_project,
            write_key=write_key,
        )

        start, stop = 0, 5000  # Keen API Docs suggest sending in batches of no more than 5K
        while start <= len(node_log_events):
            keen_payload = {'node_log_analytics': node_log_events[start:stop]}
            client.add_events(keen_payload)
            start += 5000
            stop += 5000
            time.sleep(1)
    else:
        print(node_log_events)

    logger.info('NodeLogs counted. {} NodeLogs.'.format(len(node_log_events)))

if __name__ == '__main__':
    init_app()
    main()
{'fraction_non_alphanumeric': 0.10216856695729935, 'fraction_numerical': 0.0069304717192041135, 'mean_word_length': 3.20093896713615, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6420668', 'n_tokens_mistral': 1429, 'n_tokens_neox': 1373, 'n_words': 321}