<commit_before>from django.core.management.base import BaseCommand, CommandError
from cat.models import MuseumObject, FunctionalCategory
from cat.models import ArtefactType, CulturalBloc
from cat.models import AccessStatus, LoanStatus
from cat.models import AcquisitionMethod, PhotoType, PhotoRecord
from cat.models import Person, Place, Maker, Obtained
from importcat import set_category
import os
import sys

from openpyxl.reader.excel import load_workbook


def prepare_stdout():
    unbuffered = os.fdopen(sys.stdout.fileno(), 'w', 0)
    sys.stdout = unbuffered


def import_xlsx(filename):
    prepare_stdout()
    wb = load_workbook(filename=filename)
    sheet = wb.get_sheet_by_name(name='Sheet1')
    sheet.get_highest_column()
    sheet.get_highest_row()
    sheet.cell(row=1, column=1)
    data = sheet.range(sheet.calculate_dimension())
    headers = [v.value for v in data[0]]
    for row in data[1:]:
        vals = [str(v.value).strip() for v in row]
        sys.stdout.write('\r ID=%s' % vals[0])
        m = MuseumObject()
        m.id = vals[0]
        m.registration_number = vals[0]
        m.old_registration_number = vals[1]
        m.other_number = vals[2]
        m.functional_category, c = FunctionalCategory.objects.get_or_create(
                name=vals[3])
        m.artefact_type, c = ArtefactType.objects.get_or_create(name=vals[4])
        m.storage_section = vals[5]
        m.storage_unit = vals[6]
        m.storage_bay = vals[7]
        m.storage_shelf_box_drawer = vals[8]
        if vals[9] != 'None':
#            if vals[0] == '40357':
#                import pdb; pdb.set_trace()
            sys.stdout.write(' Acq_date=%s' % vals[9])
            m.acquisition_date = vals[9].split(' ')[0]
        m.access_status, c = AccessStatus.objects.get_or_create(
                status=vals[10])
        m.description = vals[11]
        m.comment = vals[12]
        region = vals[13]
        state = vals[14]
        m.acquisition_method, c = AcquisitionMethod.objects.get_or_create(
                method=vals[15])
        m.loan_status, created = LoanStatus.objects.get_or_create(
                status=vals[16])
        country = vals[17]
        place = vals[18]
        m.place, created = Place.objects.get_or_create(name=place,
                                              australian_state=state,
                                              region=region,
                                              country=country)
        m.cultural_bloc, c = CulturalBloc.objects.get_or_create(name=vals[19])

        photographic_record = vals[20]
        m.collector, created = Person.objects.get_or_create(name=vals[21])
        m.when_collector_obtained = vals[22]
        m.how_collector_obtained, created = Obtained.objects.get_or_create(<commit_msg>Fix xls import.<commit_after>from django.core.management.base import BaseCommand, CommandError
from cat.models import MuseumObject, FunctionalCategory
from cat.models import ArtefactType, CulturalBloc
from cat.models import AccessStatus, LoanStatus
from cat.models import AcquisitionMethod, PhotoType, PhotoRecord
from cat.models import Person, Place, Maker, Obtained
from importcat import set_category
import os
import sys

from openpyxl.reader.excel import load_workbook


def prepare_stdout():
    unbuffered = os.fdopen(sys.stdout.fileno(), 'w', 0)
    sys.stdout = unbuffered


def import_xlsx(filename):
    prepare_stdout()
    wb = load_workbook(filename=filename)
    sheet = wb.get_sheet_by_name(name='Sheet1')
    sheet.get_highest_column()
    sheet.get_highest_row()
    sheet.cell(row=1, column=1)
    data = sheet.range(sheet.calculate_dimension())
    headers = [v.value for v in data[0]]
    for row in data[1:]:
        vals = [str(v.value).strip() for v in row]
        sys.stdout.write('\r ID=%s' % vals[0])
        m = MuseumObject()
        m.id = vals[0]
        m.registration_number = vals[0]
        m.old_registration_number = vals[1]
        m.other_number = vals[2]
        m.functional_category, c = FunctionalCategory.objects.get_or_create(
                name=vals[3])
        m.artefact_type, c = ArtefactType.objects.get_or_create(name=vals[4])
        m.storage_section = vals[5]
        m.storage_unit = vals[6]
        m.storage_bay = vals[7]
        m.storage_shelf_box_drawer = vals[8]
        if vals[9] != 'None':
            sys.stdout.write(' Acq_date=%s' % vals[9])
            m.acquisition_date = vals[9].split(' ')[0]
        m.access_status, c = AccessStatus.objects.get_or_create(
                status=vals[10])
        m.description = vals[11]
        m.comment = vals[12]
        region = vals[13]
        state = vals[14]
        m.acquisition_method, c = AcquisitionMethod.objects.get_or_create(
                method=vals[15])
        m.loan_status, created = LoanStatus.objects.get_or_create(
                status=vals[16])
        country = vals[17]
        place = vals[18]
        m.place, created = Place.objects.get_or_create(name=place,
                                              australian_state=state,
                                              region=region,
                                              country=country)
        m.cultural_bloc, c = CulturalBloc.objects.get_or_create(name=vals[19])

        photographic_record = vals[20]
        m.collector, created = Person.objects.get_or_create(name=vals[21])
        m.when_collector_obtained = vals[22]
        m.how_collector_obtained, created = Obtained.objects.get_or_create(
{'fraction_non_alphanumeric': 0.10648737052516809, 'fraction_numerical': 0.018171906232963837, 'mean_word_length': 2.465994962216625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5983975', 'n_tokens_mistral': 1814, 'n_tokens_neox': 1718, 'n_words': 331}