<commit_before>#!/usr/bin/env python3
#
# Program for importing patient records from CSV files into PhenoTips
#
# Copyright 2015 University of Utah
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
# USA

import csv
import sys
import time
from datetime import timedelta
from dateutil.parser import parse as parsedate
from getopt import getopt
from getpass import getpass
from phenotipsbot import PhenoTipsBot
from sys import stdout
from traceback import print_exc

KNOWN_FIELDS = {
    #phenotips
    'external_id'                          : 'String',
    'exam_date'                            : 'Date',
    'first_name'                           : 'String',
    'last_name'                            : 'String',
    'date_of_birth'                        : 'Date',
    'gender'                               : 'Sex',
    'global_age_of_onset'                  : 'Database List',
    'global_mode_of_inheritance'           : 'Database List',
    'indication_for_referral'              : 'String',
    'health_card'                          : 'String',
    'proband'                              : 'Boolean',
    'family_of'                            : 'Database List',
    'maternal_ethnicity'                   : 'Race',
    'paternal_ethnicity'                   : 'Race',
    'ivf'                                  : 'Boolean',
    'consanguinity'                        : 'Boolean',
    'miscarriages'                         : 'Boolean',
    'omim_id'                              : 'Database List',
    'unaffected'                           : 'Boolean',
    'phenotype'                            : 'Database List',
    'negative_phenotype'                   : 'Database List',
    'prenatal_phenotype'                   : 'Database List',
    'negative_prenatal_phenotype'          : 'Database List',
    'prenatal_development'                 : 'String',
    'gestation'                            : 'Integer',
    'family_history'                       : 'String',
    'pedigree'                             : 'Database List',
    'diagnosis_notes'                      : 'String',
    'medical_history'                      : 'String',
    'reports_history'                      : 'Database List',
    'extended_phenotype'                   : 'Database List',
    'extended_prenatal_phenotype'          : 'Database List',
    'extended_negative_phenotype'          : 'Database List',
    'extended_negative_prenatal_phenotype' : 'Database List',
    'apgar1'                               : 'Static List',
    'apgar5'                               : 'Static List',
    'assistedReproduction_fertilityMeds'   : 'Boolean',
    'assistedReproduction_surrogacy'       : 'Boolean',
    'solved'                               : 'Boolean',
    'solved__pubmed_id'                    : 'String',
    'solved__gene_id'                      : 'String',
    'date_of_death'                        : 'Date',
    'solved__notes'                        : 'String',
    'assistedReproduction_donoregg'        : 'Boolean',
    'date_of_death_unknown'                : 'Boolean',
    'assistedReproduction_donorsperm'      : 'Boolean',
    'icsi'                                 : 'Boolean',
    'affectedRelatives'                    : 'Boolean',
    'multipleGestation'                    : 'Boolean',
    'assistedReproduction_iui'             : 'Boolean',
    #phenotype core
    'enrollment_date'                      : 'Date String',
    'consent_signed_date'                  : 'Date String',
    'subject_mmi'                          : 'Integer',
    'lab_id'                               : 'Integer',
    'kindred_id'                           : 'Integer',
    'subject_data_relationship'            : 'String',
    'investigator'                         : 'String',
    'home_zip_code'                        : 'Integer',
    #IBD
    'age_at_diagnosis'                     : 'Float',
    'age_at_enrollment'                    : 'Float',
    'lymphoblast_transformation_date'      : 'Date',
}

def normalize(field_name, value):
    field_type = KNOWN_FIELDS[field_name]
    try:
        if field_type in ('Date', 'Date String'):
            return parsedate(value).strftime('%Y-%m-%d')
        elif field_type == 'Sex':
            value = value.strip().lower()
            if value in ('m', 'male'):
                return 'M'
            elif value in ('f', 'female'):
                return 'F'
            elif value in ('o', 'other'):
                return 'O'
            else:
                return None
        elif field_type == 'Boolean':
            value = value.strip().lower()
            if value in ('t', 'true', 'y', 'yes', '1'):
                return '1'
            elif value in ('f', 'false', 'n', 'no', '0'):
                return '0'
            else:
                return None
        elif field_type == 'Integer':
            return int(value)
        elif field_type == 'Float':
            return float(value)
        else:
            return value
    except ValueError:
        return None

def parse_csv_file(file_name, unrecognized_column_callback, unrecognized_value_callback):
    reader = csv.reader(open(file_name, 'r'))
    column_map = dict()
    patients = list()

    i = -1
    for row in reader:
        i += 1

        #skip first row
        if i == 0:
            j = 0
            for cell in row:
                if cell in KNOWN_FIELDS:
                    column_map[cell] = j
                else:
                    unrecognized_column_callback(cell)
                j += 1
            continue
<commit_msg>Don't crash when importing CSV files with some empty rows<commit_after>#!/usr/bin/env python3
#
# Program for importing patient records from CSV files into PhenoTips
#
# Copyright 2015 University of Utah
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
# USA

import csv
import sys
import time
from datetime import timedelta
from dateutil.parser import parse as parsedate
from getopt import getopt
from getpass import getpass
from phenotipsbot import PhenoTipsBot
from sys import stdout
from traceback import print_exc

KNOWN_FIELDS = {
    #phenotips
    'external_id'                          : 'String',
    'exam_date'                            : 'Date',
    'first_name'                           : 'String',
    'last_name'                            : 'String',
    'date_of_birth'                        : 'Date',
    'gender'                               : 'Sex',
    'global_age_of_onset'                  : 'Database List',
    'global_mode_of_inheritance'           : 'Database List',
    'indication_for_referral'              : 'String',
    'health_card'                          : 'String',
    'proband'                              : 'Boolean',
    'family_of'                            : 'Database List',
    'maternal_ethnicity'                   : 'Race',
    'paternal_ethnicity'                   : 'Race',
    'ivf'                                  : 'Boolean',
    'consanguinity'                        : 'Boolean',
    'miscarriages'                         : 'Boolean',
    'omim_id'                              : 'Database List',
    'unaffected'                           : 'Boolean',
    'phenotype'                            : 'Database List',
    'negative_phenotype'                   : 'Database List',
    'prenatal_phenotype'                   : 'Database List',
    'negative_prenatal_phenotype'          : 'Database List',
    'prenatal_development'                 : 'String',
    'gestation'                            : 'Integer',
    'family_history'                       : 'String',
    'pedigree'                             : 'Database List',
    'diagnosis_notes'                      : 'String',
    'medical_history'                      : 'String',
    'reports_history'                      : 'Database List',
    'extended_phenotype'                   : 'Database List',
    'extended_prenatal_phenotype'          : 'Database List',
    'extended_negative_phenotype'          : 'Database List',
    'extended_negative_prenatal_phenotype' : 'Database List',
    'apgar1'                               : 'Static List',
    'apgar5'                               : 'Static List',
    'assistedReproduction_fertilityMeds'   : 'Boolean',
    'assistedReproduction_surrogacy'       : 'Boolean',
    'solved'                               : 'Boolean',
    'solved__pubmed_id'                    : 'String',
    'solved__gene_id'                      : 'String',
    'date_of_death'                        : 'Date',
    'solved__notes'                        : 'String',
    'assistedReproduction_donoregg'        : 'Boolean',
    'date_of_death_unknown'                : 'Boolean',
    'assistedReproduction_donorsperm'      : 'Boolean',
    'icsi'                                 : 'Boolean',
    'affectedRelatives'                    : 'Boolean',
    'multipleGestation'                    : 'Boolean',
    'assistedReproduction_iui'             : 'Boolean',
    #phenotype core
    'enrollment_date'                      : 'Date String',
    'consent_signed_date'                  : 'Date String',
    'subject_mmi'                          : 'Integer',
    'lab_id'                               : 'Integer',
    'kindred_id'                           : 'Integer',
    'subject_data_relationship'            : 'String',
    'investigator'                         : 'String',
    'home_zip_code'                        : 'Integer',
    #IBD
    'age_at_diagnosis'                     : 'Float',
    'age_at_enrollment'                    : 'Float',
    'lymphoblast_transformation_date'      : 'Date',
}

def normalize(field_name, value):
    field_type = KNOWN_FIELDS[field_name]
    try:
        if field_type in ('Date', 'Date String'):
            return parsedate(value).strftime('%Y-%m-%d')
        elif field_type == 'Sex':
            value = value.strip().lower()
            if value in ('m', 'male'):
                return 'M'
            elif value in ('f', 'female'):
                return 'F'
            elif value in ('o', 'other'):
                return 'O'
            else:
                return None
        elif field_type == 'Boolean':
            value = value.strip().lower()
            if value in ('t', 'true', 'y', 'yes', '1'):
                return '1'
            elif value in ('f', 'false', 'n', 'no', '0'):
                return '0'
            else:
                return None
        elif field_type == 'Integer':
            return int(value)
        elif field_type == 'Float':
            return float(value)
        else:
            return value
    except ValueError:
        return None

def parse_csv_file(file_name, unrecognized_column_callback, unrecognized_value_callback):
    reader = csv.reader(open(file_name, 'r'))
    column_map = dict()
    patients = list()

    i = -1
    for row in reader:
        i += 1

        #skip first row
        if i == 0:
            j = 0
            for cell in row:
                if cell in KNOWN_FIELDS:
                    column_map[cell] = j
                else:
                    unrecognized_column_callback(cell)
                j += 1
            continue

        #skip empty rows
        if len(row) == 0:
            continue

{'fraction_non_alphanumeric': 0.09550606324799873, 'fraction_numerical': 0.004676230482682096, 'mean_word_length': 1.4429816069699903, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7268047', 'n_tokens_mistral': 3469, 'n_tokens_neox': 3235, 'n_words': 904}