<commit_before>import datetime
from operator import itemgetter
from itertools import zip_longest
import logging
from django.core.files.uploadedfile import InMemoryUploadedFile

from . import filestorage
from .file_upload import FileUpload, wrap_gridfs_data_for_form
from .models import Registry
from .mongo_client import construct_mongo_client
from .utils import get_code, mongo_db_name, models_from_mongo_key, is_delimited_key, mongo_key, is_multisection
from .utils import is_file_cde, is_multiple_file_cde, is_uploaded_file

logger = logging.getLogger(__name__)


class FormParsingError(Exception):
    pass


class KeyValueMissing(Exception):
    pass


def find_sections(doc, form_name=None, section_code=None,
                  formp=None, sectionp=None,
                  multisection=False):
    """
    Iterates through form sections of a mongo doc.
      form_name: Filter by form
      section_code: Filter by code
      formp: predicate function(form_dict, index)
      sectionp: predicate function(section_dict, index | (index, index))
      multisection: whether to find multisections or single sections
    """
    formp = formp or (lambda f, i: True)
    sectionp = sectionp or (lambda s, i: True)

    for f, form in enumerate(doc.get("forms") or []):
        if (form_name is None or
                (form.get("name") == form_name and formp(form, f))):
            for s, section in enumerate(form.get("sections") or []):
                if ((section_code is None or
                     section.get("code") == section_code) and
                        bool(multisection) == bool(section.get("allow_multiple"))):
                    if multisection:
                        for s2, section2 in enumerate(section.get("cdes") or []):
                            product = dict(section)
                            product["cdes"] = section2
                            if sectionp(product, (s, s2)):
                                yield product
                    elif sectionp(section, s):
                        yield section


def find_cdes(doc, form_name=None, section_code=None, cde_code=None,
              formp=None, sectionp=None, cdep=None,
              multisection=False):
    """
    Iterates through CDEs stored in a mongo doc.
    """
    cdep = cdep or (lambda c, i: True)

    sections = find_sections(doc, form_name, section_code, formp, sectionp,
                             multisection=multisection)
    for section in sections:
        for c, cde in enumerate(section.get("cdes") or []):
            if (cde_code is None or
                    (cde.get("code") == cde_code and cdep(cde, c))):
                yield cde

section_allow_multiple = lambda s, i: bool(s.get("allow_multiple"))
section_not_allow_multiple = lambda s, i: not s.get("allow_multiple")


def get_mongo_value(registry_code, nested_data, delimited_key,
                    multisection_index=None):
    """
    Grabs a CDE value out of the mongo document.
      nested_data: mongo document dict
      delimited_key: form_name____section_code____cde_code
    """
    registry_model = Registry.objects.get(code=registry_code)
    form_model, section_model, cde_model = models_from_mongo_key(registry_model, delimited_key)

    if multisection_index is None:
        sectionp = None
    else:
        sectionp = lambda s, (i, j): j == multisection_index

    cdes = find_cdes(nested_data, form_model.name, section_model.code,
                     cde_model.code, sectionp=sectionp,
                     multisection=multisection_index is not None)
    for cde in cdes:
        return cde["value"]
    return None


def update_multisection_file_cdes(gridfs_filestore, registry_code,
                                  multisection_code, form_section_items,
                                  form_model, existing_nested_data, index_map):

    logger.debug("****************** START UPDATE MULTISECTION FILE CDES ****************************")
    logger.debug("updating any gridfs files in multisection %s" % multisection_code)

    updates = []

    for item_index, section_item_dict in enumerate(form_section_items):<commit_msg>python3: no more tuple unpacking in dynamic_data<commit_after>import datetime
from operator import itemgetter
from itertools import zip_longest
import logging
from django.core.files.uploadedfile import InMemoryUploadedFile

from . import filestorage
from .file_upload import FileUpload, wrap_gridfs_data_for_form
from .models import Registry
from .mongo_client import construct_mongo_client
from .utils import get_code, mongo_db_name, models_from_mongo_key, is_delimited_key, mongo_key, is_multisection
from .utils import is_file_cde, is_multiple_file_cde, is_uploaded_file

logger = logging.getLogger(__name__)


class FormParsingError(Exception):
    pass


class KeyValueMissing(Exception):
    pass


def find_sections(doc, form_name=None, section_code=None,
                  formp=None, sectionp=None,
                  multisection=False):
    """
    Iterates through form sections of a mongo doc.
      form_name: Filter by form
      section_code: Filter by code
      formp: predicate function(form_dict, index)
      sectionp: predicate function(section_dict, index | (index, index))
      multisection: whether to find multisections or single sections
    """
    formp = formp or (lambda f, i: True)
    sectionp = sectionp or (lambda s, i: True)

    for f, form in enumerate(doc.get("forms") or []):
        if (form_name is None or
                (form.get("name") == form_name and formp(form, f))):
            for s, section in enumerate(form.get("sections") or []):
                if ((section_code is None or
                     section.get("code") == section_code) and
                        bool(multisection) == bool(section.get("allow_multiple"))):
                    if multisection:
                        for s2, section2 in enumerate(section.get("cdes") or []):
                            product = dict(section)
                            product["cdes"] = section2
                            if sectionp(product, (s, s2)):
                                yield product
                    elif sectionp(section, s):
                        yield section


def find_cdes(doc, form_name=None, section_code=None, cde_code=None,
              formp=None, sectionp=None, cdep=None,
              multisection=False):
    """
    Iterates through CDEs stored in a mongo doc.
    """
    cdep = cdep or (lambda c, i: True)

    sections = find_sections(doc, form_name, section_code, formp, sectionp,
                             multisection=multisection)
    for section in sections:
        for c, cde in enumerate(section.get("cdes") or []):
            if (cde_code is None or
                    (cde.get("code") == cde_code and cdep(cde, c))):
                yield cde

section_allow_multiple = lambda s, i: bool(s.get("allow_multiple"))
section_not_allow_multiple = lambda s, i: not s.get("allow_multiple")


def get_mongo_value(registry_code, nested_data, delimited_key,
                    multisection_index=None):
    """
    Grabs a CDE value out of the mongo document.
      nested_data: mongo document dict
      delimited_key: form_name____section_code____cde_code
    """
    registry_model = Registry.objects.get(code=registry_code)
    form_model, section_model, cde_model = models_from_mongo_key(registry_model, delimited_key)

    if multisection_index is None:
        sectionp = None
    else:
        sectionp = lambda s, ij: ij[1] == multisection_index

    cdes = find_cdes(nested_data, form_model.name, section_model.code,
                     cde_model.code, sectionp=sectionp,
                     multisection=multisection_index is not None)
    for cde in cdes:
        return cde["value"]
    return None


def update_multisection_file_cdes(gridfs_filestore, registry_code,
                                  multisection_code, form_section_items,
                                  form_model, existing_nested_data, index_map):

    logger.debug("****************** START UPDATE MULTISECTION FILE CDES ****************************")
    logger.debug("updating any gridfs files in multisection %s" % multisection_code)

    updates = []

    for item_index, section_item_dict in enumerate(form_section_items):
{'fraction_non_alphanumeric': 0.09146708388494404, 'fraction_numerical': 0.0012035142616440004, 'mean_word_length': 2.5726569217540844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '484596', 'n_tokens_mistral': 2459, 'n_tokens_neox': 2363, 'n_words': 636}