<commit_before>from apps.botanycollection.models import Accession, WoodFeatures
from bulkimport import (
    BulkDataImportHandler, BulkImportForm,
    BulkImporterException
)
from django.shortcuts import render
from django import forms
from django.contrib import messages

import logging
logger = logging.getLogger(__name__)

def setup_accessions_importer():
    bi = BulkDataImportHandler()
    bi.add_mapping(Accession, {
        'UQ Accession': 'uq_accession',
        'Material': 'material',
        'Source': 'source',
        'State': 'preservation_state',
        'Family': 'family',
        'SUBFAM': 'subfam',
        'TRIBE': 'tribe',
        'Genus': 'genus',
        'Species': 'species',
        'AUTHOR': 'species_author',
        'SSPNA': 'sspna',
        'SSPAU': 'sspau',
        'VARNA': 'varna',
        'VARAU': 'varau',
        'CULTIVAR': 'cultivar',
        'Common Name': 'common_name',
        'Biological Synonym': 'biological_synonym',
        'DETNA': 'detna',
        'DETDATE': 'detdate',
        'Collector': 'collector',
        'Collector Serial No.': 'collector_serial_no',
        'Collection Date': 'collection_date',
        'SOURCE': 'source',
        'SOURCE NUMBER': 'source_number',
        'id level flag': 'id_level_flag',
        'Country': 'country',
        'Site Name': 'site_name',
        'Lat/Long.': 'lat_long',
        'Altitude': 'altitude',
        'Notes': 'location_notes',
        'Related Accession': 'related_accession',
        'GRIN & Seed Atlas?': 'weblinks',
        })

    bi.header_row = 1
    bi.first_data_row = 2

    return bi


def setup_wood_importer():
    bi = BulkDataImportHandler()
    bi.add_mapping(Accession, {
        'family': 'family',
        'genus': 'genus',
        'Species': 'species',
        'Common Name': 'common_name',
        'Indigenous name': 'indigenous_name',
        'UQM Accession Number': 'uq_accession',
        'Specimen Collection Date': 'collection_date',
        'State': 'preservation_state',
        'Contributor': 'contributor',
        'DATE': 'date_contributed',
        'notes': 'accession_notes',
        'sample number': 'sample_number'
        }, 'UQM Accession Number', 'uq_accession')
    bi.add_mapping(WoodFeatures, {
        'aggregate rays': 'aggregate_rays',
        'axial canals': 'axial_canals',
        'axial parenchyma arrangement (1)': 'axial_parenchyma_arrangement1',
        'axial parenchyma arrangement (2)': 'axial_parenchyma_arrangement2',
        'axial parenchyma arrangement (3)': 'axial_parenchyma_arrangement3',
        'axial parenchyma arrangement (4)': 'axial_parenchyma_arrangement4',
        'axial parenchyma arrangement (5)': 'axial_parenchyma_arrangement5',
        'axial parenchyma bands': 'axial_parenchyma_bands',
        'axial parenchyma present': 'axial_parenchyma_present',
        'cambial variants': 'cambial_variants',
        'druses': 'druses',
        'Early/Late wood transition': 'early_late_wood_transition',
        'fibre helical thickenings': 'fibre_helical_thickenings',
        'fibre pits': 'fibre_pits',
        'fibre wall thickness': 'fibre_wall_thickness',
        'fusiform parenchyma cells': 'fusiform_parenchyma_cells',
        'helical thickenings': 'helical_thickenings',
        'included phloem': 'included_phloem',
        'Intervessel/tracheid pit arrangement': 'intervessel_pit_arrangement',
        'intervessels pits size': 'intervessel_pit_size',
        'Intervessel/tracheid pit shapes': 'intervessel_tracheid_pit_shapes',
        'lactifer tanniferous tubes': 'lactifer_tanniferous_tubes',
        'parenchyma like fibres present': 'parenchyma_like_fibres_present',
        'perforation plates types': 'perforation_plates_types',
        'prismatic crystals': 'prismatic_crystals',
        'radial secretory canals': 'radial_secretory_canals',
        'radial tracheids for gymnosperms': 'radial_tracheids_for_gymnosperms',
        'rays': 'rays',
        'rays cellular composition': 'rays_cellular_composition',
        'ray height': 'ray_height',
        'rays sheath cells': 'rays_sheath_cells',
        'rays structure': 'rays_structure',
        'ray width': 'ray_width',
        'ray type': 'ray_type',
        'Reference Specimens': 'reference_specimens',
        'silica': 'silica',<commit_msg>fix import of intervessel pit size field<commit_after>from apps.botanycollection.models import Accession, WoodFeatures
from bulkimport import (
    BulkDataImportHandler, BulkImportForm,
    BulkImporterException
)
from django.shortcuts import render
from django import forms
from django.contrib import messages

import logging
logger = logging.getLogger(__name__)

def setup_accessions_importer():
    bi = BulkDataImportHandler()
    bi.add_mapping(Accession, {
        'UQ Accession': 'uq_accession',
        'Material': 'material',
        'Source': 'source',
        'State': 'preservation_state',
        'Family': 'family',
        'SUBFAM': 'subfam',
        'TRIBE': 'tribe',
        'Genus': 'genus',
        'Species': 'species',
        'AUTHOR': 'species_author',
        'SSPNA': 'sspna',
        'SSPAU': 'sspau',
        'VARNA': 'varna',
        'VARAU': 'varau',
        'CULTIVAR': 'cultivar',
        'Common Name': 'common_name',
        'Biological Synonym': 'biological_synonym',
        'DETNA': 'detna',
        'DETDATE': 'detdate',
        'Collector': 'collector',
        'Collector Serial No.': 'collector_serial_no',
        'Collection Date': 'collection_date',
        'SOURCE': 'source',
        'SOURCE NUMBER': 'source_number',
        'id level flag': 'id_level_flag',
        'Country': 'country',
        'Site Name': 'site_name',
        'Lat/Long.': 'lat_long',
        'Altitude': 'altitude',
        'Notes': 'location_notes',
        'Related Accession': 'related_accession',
        'GRIN & Seed Atlas?': 'weblinks',
        })

    bi.header_row = 1
    bi.first_data_row = 2

    return bi


def setup_wood_importer():
    bi = BulkDataImportHandler()
    bi.add_mapping(Accession, {
        'family': 'family',
        'genus': 'genus',
        'Species': 'species',
        'Common Name': 'common_name',
        'Indigenous name': 'indigenous_name',
        'UQM Accession Number': 'uq_accession',
        'Specimen Collection Date': 'collection_date',
        'State': 'preservation_state',
        'Contributor': 'contributor',
        'DATE': 'date_contributed',
        'notes': 'accession_notes',
        'sample number': 'sample_number'
        }, 'UQM Accession Number', 'uq_accession')
    bi.add_mapping(WoodFeatures, {
        'aggregate rays': 'aggregate_rays',
        'axial canals': 'axial_canals',
        'axial parenchyma arrangement (1)': 'axial_parenchyma_arrangement1',
        'axial parenchyma arrangement (2)': 'axial_parenchyma_arrangement2',
        'axial parenchyma arrangement (3)': 'axial_parenchyma_arrangement3',
        'axial parenchyma arrangement (4)': 'axial_parenchyma_arrangement4',
        'axial parenchyma arrangement (5)': 'axial_parenchyma_arrangement5',
        'axial parenchyma bands': 'axial_parenchyma_bands',
        'axial parenchyma present': 'axial_parenchyma_present',
        'cambial variants': 'cambial_variants',
        'druses': 'druses',
        'Early/Late wood transition': 'early_late_wood_transition',
        'fibre helical thickenings': 'fibre_helical_thickenings',
        'fibre pits': 'fibre_pits',
        'fibre wall thickness': 'fibre_wall_thickness',
        'fusiform parenchyma cells': 'fusiform_parenchyma_cells',
        'helical thickenings': 'helical_thickenings',
        'included phloem': 'included_phloem',
        'Intervessel/tracheid pit arrangement': 'intervessel_pit_arrangement',
        'intervessel pit size': 'intervessel_pit_size',
        'Intervessel/tracheid pit shapes': 'intervessel_tracheid_pit_shapes',
        'lactifer tanniferous tubes': 'lactifer_tanniferous_tubes',
        'parenchyma like fibres present': 'parenchyma_like_fibres_present',
        'perforation plates types': 'perforation_plates_types',
        'prismatic crystals': 'prismatic_crystals',
        'radial secretory canals': 'radial_secretory_canals',
        'radial tracheids for gymnosperms': 'radial_tracheids_for_gymnosperms',
        'rays': 'rays',
        'rays cellular composition': 'rays_cellular_composition',
        'ray height': 'ray_height',
        'rays sheath cells': 'rays_sheath_cells',
        'rays structure': 'rays_structure',
        'ray width': 'ray_width',
        'ray type': 'ray_type',
        'Reference Specimens': 'reference_specimens',
        'silica': 'silica',
{'fraction_non_alphanumeric': 0.12788673552280377, 'fraction_numerical': 0.0027851920622026225, 'mean_word_length': 3.2141809290953547, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3612226', 'n_tokens_mistral': 2911, 'n_tokens_neox': 2651, 'n_words': 583}