<commit_before>
from csxj.db import Provider
from csxj.datasources import rtlinfo, sudpresse, lesoir, lalibre, dhnet


def show_queue_info(json_db):
    sources = [rtlinfo, sudpresse, lesoir, lalibre, dhnet]
    for source in sources:
        p = Provider(json_db, source.SOURCE_NAME)

        batches_by_day = p.get_queued_batches_by_day()

        for day, batches in batches_by_day:
            queued_item_count = 0
            for batch, items in batches:

                queued_item_count += len(items['articles'])
            print day, queued_item_count


def try_download_queue(json_db):
    sources = [rtlinfo, sudpresse, lesoir, lalibre, dhnet]
    for source in sources:
        p = Provider(json_db, source.SOURCE_NAME)
        batches_by_day = p.get_queued_batches_by_day()
        print source.SOURCE_NAME
        for day, batches in batches_by_day:
            print "\tDay:", day
            for batch, items in batches:
                print "\t\tBatch:", batch
                articles = items['articles']
                for title, url in articles:
                    print "\t\t\tDownloading {0}".format(url)
                    art, html = source.extract_article_data(url)
                    print "\t\t\t\t got {0} links".format(len(art.links))



if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Utility functions to troubleshoot queue management')
    parser.add_argument('--jsondb', type=str, dest='jsondb', required=True, help='json db root directory')
    #('--show-only', type=bool, dest='show_queue', )

    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--download-queue',  action='store_true')
    group.add_argument('--show-queue', action='store_true')

    args = parser.parse_args()
    print args
    if args.download_queue:
        try_download_queue(args.jsondb)
    elif args.show_queue:
        show_queue_info(args.jsondb)<commit_msg>more info on queued items when listing<commit_after>
from csxj.db import Provider
from csxj.datasources import rtlinfo, sudpresse, lesoir, lalibre, dhnet


def show_queue_info(json_db):
    sources = [rtlinfo, sudpresse, lesoir, lalibre, dhnet]
    res = dict()
    for source in sources:
        p = Provider(json_db, source.SOURCE_NAME)
        print source.SOURCE_NAME
        batches_by_day = p.get_queued_batches_by_day()
        total_item_count = 0

        for day, batches in batches_by_day:
            print "\tDay:", day
            queued_item_count = 0
            for batch, items in batches:
                queued_item_count += len(items['articles'])

            print day, queued_item_count
            total_item_count += queued_item_count
        res[source.SOURCE_NAME] = (total_item_count, len(batches_by_day))


    for name, (item_count, day_count) in res.items():
        print "{0}: {1} items for {2} days".format(name, item_count, day_count)

def try_download_queue(json_db):
    sources = [rtlinfo, sudpresse, lesoir, lalibre, dhnet]
    for source in sources:
        p = Provider(json_db, source.SOURCE_NAME)
        batches_by_day = p.get_queued_batches_by_day()
        print source.SOURCE_NAME
        for day, batches in batches_by_day:
            print "\tDay:", day
            for batch, items in batches:
                print "\t\tBatch:", batch
                articles = items['articles']
                for title, url in articles:
                    print "\t\t\tDownloading {0}".format(url)
                    art, html = source.extract_article_data(url)
                    print "\t\t\t\t got {0} links".format(len(art.links))



if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Utility functions to troubleshoot queue management')
    parser.add_argument('--jsondb', type=str, dest='jsondb', required=True, help='json db root directory')

    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--download-queue',  action='store_true')
    group.add_argument('--show-queue', action='store_true')

    args = parser.parse_args()

    if args.download_queue:
        try_download_queue(args.jsondb)
    elif args.show_queue:
        show_queue_info(args.jsondb)
{'fraction_non_alphanumeric': 0.10256410256410256, 'fraction_numerical': 0.002352387673488591, 'mean_word_length': 2.726555652936021, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7134252', 'n_tokens_mistral': 1375, 'n_tokens_neox': 1313, 'n_words': 307}