<commit_before>import datetime
from corehq.util.couch import IterDB
from dimagi.utils.couch.database import iter_docs
from dimagi.utils.chunked import chunked
import logging


def filter_doc_ids_by_doc_type(db, doc_ids, doc_types):
    for doc_ids_chunk in chunked(doc_ids, 100):
        keys = [[doc_type, doc_id]
                for doc_id in doc_ids_chunk
                for doc_type in doc_types]
        results = db.view('all_docs/by_doc_type', keys=keys, reduce=False)
        for result in results:
            yield result['id']


def copy_docs(source_db, target_db, doc_ids):
    """
    copy docs from source_db to target_db
    by doc_id

    """
    if not doc_ids:
        return
    with IterDB(target_db, new_edits=False) as iter_db:
        for doc in iter_docs(source_db, doc_ids, attachments=True):
            iter_db.save(doc)
    if iter_db.errors_by_type:
        logging.error('errors bulk saving in copy_docs: {!r}'
                      .format(iter_db.errors_by_type))


def _bulk_get_revs(target_db, doc_ids):
    """
    return (_id, _rev) for every existing doc in doc_ids

    if a doc id is not found in target_db, it is excluded from the result<commit_msg>make doctype continuous migration include attachments<commit_after>import datetime
from corehq.doctypemigrations.bulk_migrate import _insert_attachments
from corehq.util.couch import IterDB
from dimagi.utils.couch.database import iter_docs
from dimagi.utils.chunked import chunked
import logging


def filter_doc_ids_by_doc_type(db, doc_ids, doc_types):
    for doc_ids_chunk in chunked(doc_ids, 100):
        keys = [[doc_type, doc_id]
                for doc_id in doc_ids_chunk
                for doc_type in doc_types]
        results = db.view('all_docs/by_doc_type', keys=keys, reduce=False)
        for result in results:
            yield result['id']


def copy_docs(source_db, target_db, doc_ids):
    """
    copy docs from source_db to target_db
    by doc_id

    """
    if not doc_ids:
        return
    with IterDB(target_db, new_edits=False) as iter_db:
        for doc in iter_docs(source_db, doc_ids, attachments=True):
            doc = _insert_attachments(source_db, doc)
            iter_db.save(doc)
    if iter_db.errors_by_type:
        logging.error('errors bulk saving in copy_docs: {!r}'
                      .format(iter_db.errors_by_type))


def _bulk_get_revs(target_db, doc_ids):
    """
    return (_id, _rev) for every existing doc in doc_ids

    if a doc id is not found in target_db, it is excluded from the result
{'fraction_non_alphanumeric': 0.07547169811320754, 'fraction_numerical': 0.0023584905660377358, 'mean_word_length': 2.9765625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5208481', 'n_tokens_mistral': 843, 'n_tokens_neox': 817, 'n_words': 231}