<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals
from django.core.management.base import BaseCommand
from corehq.blobs import get_blob_db
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from corehq.form_processor.models import XFormInstanceSQL
from corehq.util.log import with_progress_bar
from couchforms.const import ATTACHMENT_NAME
from couchforms.models import XFormInstance as CouchForm
import datetime
import logging

_logger = logging.getLogger(__name__)


class Command(BaseCommand):
    help = "Check if there is form.xml in all forms for a domain."

    def add_arguments(self, parser):
        parser.add_argument('domain', help='Domain to check.')

    def handle(self, domain, **options):
        blob_db = get_blob_db()
        form_db = FormAccessors(domain)
        form_ids = form_db.get_all_form_ids_in_domain()
        bad_form_id_string = "%s %s: %%s" % (domain, datetime.datetime.utcnow())
        for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)):
            if isinstance(form, CouchForm):
                meta = form.blobs.get(ATTACHMENT_NAME)
                if not meta or not blob_db.exists(
                        meta.id, form._blobdb_bucket()):  # pylint: disable=protected-access
                    _logger.info(bad_form_id_string, form.form_id)
            elif isinstance(form, XFormInstanceSQL):
                meta = form.get_attachment_meta(ATTACHMENT_NAME)
                if not meta or not blob_db.exists(meta.blob_id, meta.blobdb_bucket()):
                    _logger.info(bad_form_id_string, form.form_id)
            else:
                raise Exception("not sure how we got here")<commit_msg>writing to file, not logging (#20071)<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals
from django.core.management.base import BaseCommand
from corehq.blobs import get_blob_db
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from corehq.form_processor.models import XFormInstanceSQL
from corehq.util.log import with_progress_bar
from couchforms.const import ATTACHMENT_NAME
from couchforms.models import XFormInstance as CouchForm


class Command(BaseCommand):
    help = "Check if there is form.xml in all forms for a domain."

    def add_arguments(self, parser):
        parser.add_argument('domain', help='Domain to check.')
        parser.add_argument('file_name', help='Location to put the output.')

    def handle(self, domain, file_name, **options):
        blob_db = get_blob_db()
        form_db = FormAccessors(domain)
        form_ids = form_db.get_all_form_ids_in_domain()

        with open(file_name, 'w') as open_file:
            for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)):
                if isinstance(form, CouchForm):
                    meta = form.blobs.get(ATTACHMENT_NAME)
                    if not meta or not blob_db.exists(
                            meta.id, form._blobdb_bucket()):  # pylint: disable=protected-access
                        open_file.write(form.form_id + "\n")
                elif isinstance(form, XFormInstanceSQL):
                    meta = form.get_attachment_meta(ATTACHMENT_NAME)
                    if not meta or not blob_db.exists(meta.blob_id, meta.blobdb_bucket()):
                        open_file.write(form.form_id + "\n")
                else:
                    raise Exception("not sure how we got here")
{'fraction_non_alphanumeric': 0.08071876782658301, 'fraction_numerical': 0.001426126640045636, 'mean_word_length': 2.922818791946309, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6614404', 'n_tokens_mistral': 1031, 'n_tokens_neox': 993, 'n_words': 254}