<commit_before>from couchdbkit import ResourceNotFound
from django.contrib import messages
from corehq.apps.fixtures.exceptions import FixtureUploadError, ExcelMalformatException, \
    DuplicateFixtureTagException, FixtureAPIException
from django.core.validators import ValidationError
from django.utils.translation import ugettext as _
from corehq.apps.fixtures.models import FixtureTypeField, FixtureDataType, FixtureDataItem, FixtureItemField, \
    FieldList
from corehq.apps.users.bulkupload import GroupMemoizer
from corehq.apps.users.models import CommCareUser
from corehq.apps.users.util import normalize_username
from dimagi.utils.couch.bulk import CouchTransaction
from dimagi.utils.excel import WorkbookJSONReader, WorksheetNotFound


DELETE_HEADER = "Delete(Y/N)"


def do_fixture_upload(request, domain, file_ref, replace):
    try:
        workbook = WorkbookJSONReader(file_ref.get_filename())
    except AttributeError:
        raise FixtureUploadError(_("Error processing your Excel (.xlsx) file"))
    except Exception:
        raise FixtureUploadError(_("Invalid file-format. Please upload a valid xlsx file."))

    try:
        return run_upload(request, domain, workbook, replace=replace)

    except WorksheetNotFound as e:
        raise FixtureUploadError(_("Workbook does not contain a sheet called '%(title)s'") % {'title': e.title})

    except ExcelMalformatException as e:
        raise FixtureUploadError(_("Uploaded excel file has following formatting-problems: '%(e)s'") % {'e': e})
    except (DuplicateFixtureTagException, FixtureAPIException) as e:
        raise FixtureUploadError(unicode(e))
    except Exception:
        raise FixtureUploadError(_("Fixture upload failed for some reason and we have noted this failure. "
                                   "Please make sure the excel file is correctly formatted and try again."))


def run_upload(request, domain, workbook, replace=False):
    return_val = {
        "unknown_groups": [],
        "unknown_users": [],
        "number_of_fixtures": 0,
    }
    failure_messages = {
        "has_no_column": "Workbook 'types' has no column '{column_name}'.",
        "has_no_field_column": "Excel-sheet '{tag}' does not contain the column '{field}' "
                               "as specified in its 'types' definition",
        "has_extra_column": "Excel-sheet '{tag}' has an extra column" +
                            "'{field}' that's not defined in its 'types' definition",
        "wrong_property_syntax": "Properties should be specified as 'field 1: property 1'. In 'types' sheet, " +
                            "'{prop_key}' for field '{field}' is not correctly formatted",
        "sheet_has_no_property": "Excel-sheet '{tag}' does not contain property " +
                            "'{property}' of the field '{field}' as specified in its 'types' definition",
        "sheet_has_extra_property": "Excel-sheet '{tag}'' has an extra property " +
                            "'{property}' for the field '{field}' that's not defined in its 'types' definition. Re-check the formatting",
        "invalid_field_with_property": "Fields with attributes should be numbered as 'field: {field} integer",
        "invalid_property": "Attribute should be written as '{field}: {prop} interger'",<commit_msg>split out function<commit_after>from couchdbkit import ResourceNotFound
from django.contrib import messages
from corehq.apps.fixtures.exceptions import FixtureUploadError, ExcelMalformatException, \
    DuplicateFixtureTagException, FixtureAPIException
from django.core.validators import ValidationError
from django.utils.translation import ugettext as _
from corehq.apps.fixtures.models import FixtureTypeField, FixtureDataType, FixtureDataItem, FixtureItemField, \
    FieldList
from corehq.apps.users.bulkupload import GroupMemoizer
from corehq.apps.users.models import CommCareUser
from corehq.apps.users.util import normalize_username
from dimagi.utils.couch.bulk import CouchTransaction
from dimagi.utils.excel import WorkbookJSONReader, WorksheetNotFound


DELETE_HEADER = "Delete(Y/N)"


def do_fixture_upload(request, domain, file_ref, replace):
    workbook = _get_workbook(file_ref)

    try:
        return run_upload(request, domain, workbook, replace=replace)

    except WorksheetNotFound as e:
        raise FixtureUploadError(_("Workbook does not contain a sheet called '%(title)s'") % {'title': e.title})

    except ExcelMalformatException as e:
        raise FixtureUploadError(_("Uploaded excel file has following formatting-problems: '%(e)s'") % {'e': e})
    except (DuplicateFixtureTagException, FixtureAPIException) as e:
        raise FixtureUploadError(unicode(e))
    except Exception:
        raise FixtureUploadError(_("Fixture upload failed for some reason and we have noted this failure. "
                                   "Please make sure the excel file is correctly formatted and try again."))

def _get_workbook(download_ref):
    try:
        return WorkbookJSONReader(download_ref.get_filename())
    except AttributeError:
        raise FixtureUploadError(_("Error processing your Excel (.xlsx) file"))
    except Exception:
        raise FixtureUploadError(_("Invalid file-format. Please upload a valid xlsx file."))

def run_upload(request, domain, workbook, replace=False):
    return_val = {
        "unknown_groups": [],
        "unknown_users": [],
        "number_of_fixtures": 0,
    }
    failure_messages = {
        "has_no_column": "Workbook 'types' has no column '{column_name}'.",
        "has_no_field_column": "Excel-sheet '{tag}' does not contain the column '{field}' "
                               "as specified in its 'types' definition",
        "has_extra_column": "Excel-sheet '{tag}' has an extra column" +
                            "'{field}' that's not defined in its 'types' definition",
        "wrong_property_syntax": "Properties should be specified as 'field 1: property 1'. In 'types' sheet, " +
                            "'{prop_key}' for field '{field}' is not correctly formatted",
        "sheet_has_no_property": "Excel-sheet '{tag}' does not contain property " +
                            "'{property}' of the field '{field}' as specified in its 'types' definition",
        "sheet_has_extra_property": "Excel-sheet '{tag}'' has an extra property " +
                            "'{property}' for the field '{field}' that's not defined in its 'types' definition. Re-check the formatting",
        "invalid_field_with_property": "Fields with attributes should be numbered as 'field: {field} integer",
        "invalid_property": "Attribute should be written as '{field}: {prop} interger'",
{'fraction_non_alphanumeric': 0.09728676360365762, 'fraction_numerical': 0.000899415380002998, 'mean_word_length': 3.480859637340497, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1053143', 'n_tokens_mistral': 1824, 'n_tokens_neox': 1758, 'n_words': 567}