<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu

from __future__ import (unicode_literals, absolute_import,
                        division, print_function)

from rapidpro_tools.mongo import contacts
from rapidpro_mali import update_groups


def main():
    print(contacts.find().count())
    for contact in contacts.find({}):
        update_groups(contact, remove_others=True)

if __name__ == '__main__':
    main()<commit_msg>two-steps loop to avoid mongodb cursor expiration<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu

from __future__ import (unicode_literals, absolute_import,
                        division, print_function)

from rapidpro_tools import logger
from rapidpro_tools.mongo import contacts
from rapidpro_mali import update_groups


def main():
    uuids = [c['uuid'] for c in contacts.find({})]
    logger.info(len(uuids))
    for uuid in uuids:
        logger.info(uuid)
        update_groups(contacts.find_one({'uuid': uuid}), remove_others=True)

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.1164695177434031, 'fraction_numerical': 0.007279344858962694, 'mean_word_length': 3.5454545454545454, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6572453', 'n_tokens_mistral': 360, 'n_tokens_neox': 350, 'n_words': 95}