<commit_before>"""
Selects number of distinct revisions.



Usage:
    session_stats (-h|--help)
    session_stats <input>
                     [--debug]
                     [--verbose]

Options:
    -h, --help  This help message is printed
    <input>     Path to input file to process.                           
    --debug     Print debug logging to stderr
    --verbose   Print dots and stuff to stderr  
"""


import docopt
import logging
import operator
import sys
import mysqltsv
from collections import defaultdict


logger = logging.getLogger(__name__)


def main(argv=None):
    args = docopt.docopt(__doc__)
    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )

    input_file = mysqltsv.Reader(
        open(args['<input>'],'rt'), headers=True, 
        types=[str, str, str, str, str, int, str, str, str, str, str, str, 
            str, str])


    verbose = args['--verbose']

    run(input_file, verbose)


def run(input_file, verbose):
    
    sessions = defaultdict(lambda: defaultdict(int))
    bot_sessions = defaultdict(lambda: defaultdict(int))
    human_sessions = defaultdict(lambda: defaultdict(int))


    for i, line in enumerate(input_file):
        sessions[line["user"]][line["session_start"]] = 1
        if line["edit_type"] == 'bot':
            bot_sessions[line["user"]][line["session_start"]] = 1
        else:
            human_sessions[line["user"]][line["session_start"]] = 1


        if verbose and i % 10000 == 0 and i != 0:
            sys.stderr.write("Revisions analyzed: {0}\n".format(i))  
            sys.stderr.flush()


    session_sum = 0
    for user in sessions:
        for session_start in sessions[user]:
            session_sum += 1

    bot_session_sum = 0
    for user in bot_sessions:
        for session_start in bot_sessions[user]:
            bot_session_sum += 1

    human_session_sum = 0
    for user in human_sessions:
        for session_start in human_sessions[user]:
            human_session_sum += 1


    print("Sessions: {0}".format(session_sum))
    print("Bot sessions: {0}".format(bot_session_sum))
    print("Human sessions: {0}".format(human_session_sum))


main()
<commit_msg>Calculated reivsion namespace totals<commit_after>"""
Selects number of distinct revisions.



Usage:
    session_stats (-h|--help)
    session_stats <input>
                     [--debug]
                     [--verbose]

Options:
    -h, --help  This help message is printed
    <input>     Path to input file to process.                           
    --debug     Print debug logging to stderr
    --verbose   Print dots and stuff to stderr  
"""


import docopt
import logging
import operator
import sys
import mysqltsv
from collections import defaultdict


logger = logging.getLogger(__name__)


def main(argv=None):
    args = docopt.docopt(__doc__)
    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )

    input_file = mysqltsv.Reader(
        open(args['<input>'],'rt'), headers=True, 
        types=[str, str, str, str, str, int, str, str, str, str, str, str, 
            str, str])


    verbose = args['--verbose']

    run(input_file, verbose)


def run(input_file, verbose):
    
    sessions = defaultdict(lambda: defaultdict(int))
    bot_sessions = defaultdict(lambda: defaultdict(int))
    human_sessions = defaultdict(lambda: defaultdict(int))
    revision_namespaces = defaultdict(int)


    for i, line in enumerate(input_file):
        sessions[line["user"]][line["session_start"]] = 1
        revision_namespaces[line["namespace"]] += 1

        if line["edit_type"] == 'bot':
            bot_sessions[line["user"]][line["session_start"]] = 1
        else:
            human_sessions[line["user"]][line["session_start"]] = 1


        if verbose and i % 10000 == 0 and i != 0:
            sys.stderr.write("Revisions analyzed: {0}\n".format(i))  
            sys.stderr.flush()


    session_sum = 0
    for user in sessions:
        for session_start in sessions[user]:
            session_sum += 1

    bot_session_sum = 0
    for user in bot_sessions:
        for session_start in bot_sessions[user]:
            bot_session_sum += 1

    human_session_sum = 0
    for user in human_sessions:
        for session_start in human_sessions[user]:
            human_session_sum += 1


    print("Sessions: {0}".format(session_sum))
    print("Bot sessions: {0}".format(bot_session_sum))
    print("Human sessions: {0}".format(human_session_sum))
    print("Revision namespaces: {0}".format(revision_namespaces))


main()

{'fraction_non_alphanumeric': 0.12148742869216142, 'fraction_numerical': 0.008873864356644834, 'mean_word_length': 2.6054836252856055, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3517760', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1416, 'n_words': 366}