<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu

from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import datetime

from docopt import docopt

from rapidpro_tools import logger, change_logging_level
from rapidpro_tools.mongo import meta, contacts, relayers, messages
from rapidpro_tools.utils import get_api_data

help = ("""Usage: dump-rapidpro.py [-v] [-h] [-z] [-a after] [--messages] """
        """[--contacts] [--relayers]

-h --help                       Display this help message
-v --verbose                    Display DEBUG messages
-a --after=<datetime_str>       rapidpro datetime formatted string.
-z --noresume                   Do NOT download where it left (messages)

--relayers                      Dumps all relayers
--contacts                      Dumps all contacts
--messages                      Dumps all messages

This script dumps JSON data from a rapidpro instance into mongo """)


def update_meta(endpoint, updated_on):
    db_item = meta.find_one({'endpoint': endpoint})
    if db_item:
        db_item.update({'updated_on': updated_on})
        meta.save(db_item)
    else:
        meta.insert({'endpoint': endpoint,
                     'updated_on': updated_on})


def update_collection(collection, data, id_field='uuid'):
    if not data['count']:
        return
    for item in data['results']:
        if collection.find({id_field: item.get(id_field)}).count():
            # item exist. update?
            db_item = collection.find_one(
                {id_field: item.get(id_field)})
            db_item.update(item)
            collection.save(db_item)
        else:
            collection.insert(item)
    return


def dump_contacts(**options):
    logger.info("Updating Contacts. Currently have {} contacts in DB."
                .format(contacts.count()))

    # call contacts API
    contacts_list = get_api_data('/contacts.json')
    update_collection(collection=contacts,
                      data=contacts_list,
                      id_field='uuid')

    # loop through potential next pages
    while contacts_list.get('next'):
        contacts_list = get_api_data(contacts_list.get('next'))
        update_collection(collection=contacts,
                          data=contacts_list,
                          id_field='uuid')

    logger.info("Updated Contacts completed. Now have {} contacts in DB."
                .format(contacts.count()))


def dump_relayers(**options):
    logger.info("Updating Relayers. Currently have {} relayers in DB."
                .format(relayers.count()))

    # call relayers API
    relayers_list = get_api_data('/relayers.json')
    update_collection(collection=relayers,
                      data=relayers_list,
                      id_field='relayer')

    # loop through potential next pages
    while relayers_list.get('next'):
        relayers_list = get_api_data(relayers_list.get('next'))
        update_collection(collection=relayers,
                          data=relayers_list,
                          id_field='relayer')

    logger.info("Updated Contacts completed. Now have {} relayers in DB."
                .format(relayers.count()))


def dump_messages(**options):
    logger.info("Updating Messages. Currently have {} messages in DB."
                .format(messages.count()))

    params = {}
    if options.get('after'):
        params.update({'after': options.get('after')})
    elif options.get('resume'):
        if meta.find_one({'endpoint': 'messages'}) is not None:
            params.update({
                'after': meta.find_one({
                    'endpoint': 'messages'}).get('updated_on')})

    # call messages API
    messages_list = get_api_data('/messages.json', **params)
    update_collection(collection=messages,
                      data=messages_list,
                      id_field='id')

    # loop through potential next pages
    while messages_list.get('next'):
        messages_list = get_api_data(messages_list.get('next'))
        update_collection(collection=messages,
                          data=messages_list,
                          id_field='id')

    logger.info("Updated Contacts completed. Now have {} messages in DB."
                .format(messages.count()))

<commit_msg>fix info messages<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu

from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import datetime

from docopt import docopt

from rapidpro_tools import logger, change_logging_level
from rapidpro_tools.mongo import meta, contacts, relayers, messages
from rapidpro_tools.utils import get_api_data

help = ("""Usage: dump-rapidpro.py [-v] [-h] [-z] [-a after] [--messages] """
        """[--contacts] [--relayers]

-h --help                       Display this help message
-v --verbose                    Display DEBUG messages
-a --after=<datetime_str>       rapidpro datetime formatted string.
-z --noresume                   Do NOT download where it left (messages)

--relayers                      Dumps all relayers
--contacts                      Dumps all contacts
--messages                      Dumps all messages

This script dumps JSON data from a rapidpro instance into mongo """)


def update_meta(endpoint, updated_on):
    db_item = meta.find_one({'endpoint': endpoint})
    if db_item:
        db_item.update({'updated_on': updated_on})
        meta.save(db_item)
    else:
        meta.insert({'endpoint': endpoint,
                     'updated_on': updated_on})


def update_collection(collection, data, id_field='uuid'):
    if not data['count']:
        return
    for item in data['results']:
        if collection.find({id_field: item.get(id_field)}).count():
            # item exist. update?
            db_item = collection.find_one(
                {id_field: item.get(id_field)})
            db_item.update(item)
            collection.save(db_item)
        else:
            collection.insert(item)
    return


def dump_contacts(**options):
    logger.info("Updating Contacts. Currently have {} contacts in DB."
                .format(contacts.count()))

    # call contacts API
    contacts_list = get_api_data('/contacts.json')
    update_collection(collection=contacts,
                      data=contacts_list,
                      id_field='uuid')

    # loop through potential next pages
    while contacts_list.get('next'):
        contacts_list = get_api_data(contacts_list.get('next'))
        update_collection(collection=contacts,
                          data=contacts_list,
                          id_field='uuid')

    logger.info("Updated Contacts completed. Now have {} contacts in DB."
                .format(contacts.count()))


def dump_relayers(**options):
    logger.info("Updating Relayers. Currently have {} relayers in DB."
                .format(relayers.count()))

    # call relayers API
    relayers_list = get_api_data('/relayers.json')
    update_collection(collection=relayers,
                      data=relayers_list,
                      id_field='relayer')

    # loop through potential next pages
    while relayers_list.get('next'):
        relayers_list = get_api_data(relayers_list.get('next'))
        update_collection(collection=relayers,
                          data=relayers_list,
                          id_field='relayer')

    logger.info("Updated Relayers completed. Now have {} relayers in DB."
                .format(relayers.count()))


def dump_messages(**options):
    logger.info("Updating Messages. Currently have {} messages in DB."
                .format(messages.count()))

    params = {}
    if options.get('after'):
        params.update({'after': options.get('after')})
    elif options.get('resume'):
        if meta.find_one({'endpoint': 'messages'}) is not None:
            params.update({
                'after': meta.find_one({
                    'endpoint': 'messages'}).get('updated_on')})

    # call messages API
    messages_list = get_api_data('/messages.json', **params)
    update_collection(collection=messages,
                      data=messages_list,
                      id_field='id')

    # loop through potential next pages
    while messages_list.get('next'):
        messages_list = get_api_data(messages_list.get('next'))
        update_collection(collection=messages,
                          data=messages_list,
                          id_field='id')

    logger.info("Updated Messages completed. Now have {} messages in DB."
                .format(messages.count()))


{'fraction_non_alphanumeric': 0.11146936895439889, 'fraction_numerical': 0.0009212344541685859, 'mean_word_length': 2.184818481848185, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6839068', 'n_tokens_mistral': 2418, 'n_tokens_neox': 2311, 'n_words': 560}