<commit_before>"""
Prints page views for entities.

Usage:
    entity_page_views (-h|--help)
    entity_page_views <aggregated-entity-usage-file> --page-view-file=<location>
                      [<dbname-file>]
                      [--output=<location>]
                      [--debug] 
                      [--verbose]


Options:
    -h, --help                      This help message is printed
    <aggregated-entity-usage-file>  Path to aggregated entity usage file.
    --page-view-file=<location>     Path to tsv file to process.
    <dbname-file>                   Path to json file to process. If no file is 
                                    provided, uses stdin
    --output=<location>             Where results will be writen.
                                    [default: <stdout>]
    --debug                         Print debug logging to stderr
    --verbose                       Print dots and stuff to stderr  
"""
import logging
import requests
import sys
import json
import bz2
import re
from collections import defaultdict

import docopt

logger = logging.getLogger(__name__)

def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)
    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )
    aggregated_entity_usage_file = open(args['<aggregated-entity-usage-file>'])
    page_view_file = args['--page-view-file']

    if args['<dbname-file>']:
        dbname_file = args['<dbname-file>']
    else:
        logger.info("Reading from <stdin>")
        dbname_file = sys.stdin

    if args['--output'] == '<stdout>':
        output = sys.stdout
    else:
        output = open(args['--output'], "w")

    verbose = args['--verbose']

    run(aggregated_entity_usage_file, dbname_file, page_view_file, output,
        verbose)


def run(aggregated_entity_usage_file, dbname_file, page_view_file, output,
    verbose):

    view_dict = defaultdict(lambda: defaultdict(dict))
    wikidb_dict = {}
    entity_values = {}

    f = bz2.open(page_view_file)


    if verbose:
        sys.stderr.write("Inserting page views into Python dictionary")
        sys.stderr.flush()


    for i, entry in enumerate(f):
        sys.stderr.write(entry.decode().strip().split("\t"))
        sys.stderr.flush()
        project, page, views = entry.decode().strip().split("\t")
        if i == 0:
            continue
        view_dict[project][page] = int(views)
<commit_msg>small change again<commit_after>"""
Prints page views for entities.

Usage:
    entity_page_views (-h|--help)
    entity_page_views <aggregated-entity-usage-file> --page-view-file=<location>
                      [<dbname-file>]
                      [--output=<location>]
                      [--debug] 
                      [--verbose]


Options:
    -h, --help                      This help message is printed
    <aggregated-entity-usage-file>  Path to aggregated entity usage file.
    --page-view-file=<location>     Path to tsv file to process.
    <dbname-file>                   Path to json file to process. If no file is 
                                    provided, uses stdin
    --output=<location>             Where results will be writen.
                                    [default: <stdout>]
    --debug                         Print debug logging to stderr
    --verbose                       Print dots and stuff to stderr  
"""
import logging
import requests
import sys
import json
import bz2
import re
from collections import defaultdict

import docopt

logger = logging.getLogger(__name__)

def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)
    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )
    aggregated_entity_usage_file = open(args['<aggregated-entity-usage-file>'])
    page_view_file = args['--page-view-file']

    if args['<dbname-file>']:
        dbname_file = args['<dbname-file>']
    else:
        logger.info("Reading from <stdin>")
        dbname_file = sys.stdin

    if args['--output'] == '<stdout>':
        output = sys.stdout
    else:
        output = open(args['--output'], "w")

    verbose = args['--verbose']

    run(aggregated_entity_usage_file, dbname_file, page_view_file, output,
        verbose)


def run(aggregated_entity_usage_file, dbname_file, page_view_file, output,
    verbose):

    view_dict = defaultdict(lambda: defaultdict(dict))
    wikidb_dict = {}
    entity_values = {}

    f = bz2.open(page_view_file)


    if verbose:
        sys.stderr.write("Inserting page views into Python dictionary")
        sys.stderr.flush()


    for i, entry in enumerate(f):
        sys.stderr.write(entry.decode().strip())
        sys.stderr.flush()
        project, page, views = entry.decode().strip().split("\t")
        if i == 0:
            continue
        view_dict[project][page] = int(views)

{'fraction_non_alphanumeric': 0.12047228337002201, 'fraction_numerical': 0.0012007204322593557, 'mean_word_length': 2.098574085554867, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '238807', 'n_tokens_mistral': 1519, 'n_tokens_neox': 1432, 'n_words': 356}