<commit_before>import csv
import sys, getopt
import uuid
import copy
from datetime import datetime
def date(datestr="", format="%Y%m%d"):
    if not datestr:
        return datetime.today().date()
    return datetime.strptime(datestr, format).date()

def coded(row, writer, name, datatype, wcount, concept_list, concepts_dict_list):
    answers  = [row['answer.1'].strip(),row['answer.2'].strip(),row['answer.3'].strip(),row['answer.4'].strip(),row['answer.5'].strip(),row['answer.6'].strip(),row['answer.7'].strip(),row['answer.8'].strip(),row['answer.9'].strip(),row['answer.10'].strip(),row['answer.12'].strip(),row['answer.12'].strip()] 
    answer_1  = row['answer.1']
    answer_2  = row['answer.2']
    answer_3  = row['answer.3']
    answer_4  = row['answer.4']
    answer_5  = row['answer.5']
    answer_6  = row['answer.6']
    answer_7  = row['answer.7']
    answer_8  = row['answer.8']
    answer_9  = row['answer.9']
    answer_10  = row['answer.10']
    answer_11  = row['answer.11']
    answer_12  = row['answer.12']
    reference_term_source  = row['reference-term-source']
    reference_term_code  = row['reference-term-code']
    reference_term_relationship  = row['reference-term-relationship']
    if name != '':
        for answer in answers:
            if answer != '':
                if answer not in concepts_dict_list:
                    if answer not in concept_list:
                        wcount = wcount + 1
                        concept_list.append(answer)
                        writer.writerow({'uuid':uuid.uuid1(),'name':answer,'class':'Misc','datatype':'N/A'})
        if name not in concept_list:
            wcount = wcount + 1
            concept_list.append(name)
            writer.writerow({'uuid':uuid.uuid1(),'name':name,'class':'Misc','datatype':datatype,'answer.1':answer_1,'answer.2':answer_2,'answer.3':answer_3,'answer.4':answer_4,'answer.5':answer_5,'answer.6':answer_6,'answer.7':answer_7,'answer.8':answer_8,'answer.9':answer_9,'answer.10':answer_10})        
    return wcount, concept_list

def single(row, writer, name, datatype, wcount, concept_list, concepts_dict_list):
    units  = row['units']
    High_Normal  = row['High Normal']
    Low_Normal  = row['Low Normal']
    Allow_Decimal  = row['Allow Decimal']
    locale  = row['locale']
    synonym_1  = row['synonym.1']
    reference_term_source  = row['reference-term-source']
    reference_term_code  = row['reference-term-code']
    reference_term_relationship  = row['reference-term-relationship']
    if name != '':
        if name not in concepts_dict_list:
            if name not in concept_list :
                wcount = wcount + 1
                concept_list.append(name)
                writer.writerow({'uuid':uuid.uuid1(),'name':name,'class':'Misc','datatype':datatype,'High Normal':High_Normal,'Low Normal':Low_Normal})        
    return wcount, concept_list

def get_concepts_list(dictionary_file):
    # file_data is the text of the file, not the filename
    concepts_dict_list = []
    with open(dictionary_file, encoding="utf8") as in_csvfile:
        reader = csv.DictReader(in_csvfile, quotechar='"')    
#        reader = csv.DictReader(dictionary_file, ('Concept Id','Name','Description','Synonyms','Answers','Set Members','Class','Datatype','Changed By','Creator'))
        for row in reader:
            concepts_dict_list.append(row['Name'])

    return concepts_dict_list

def main(argv):
    inputfile = ''
    outputfile = ''
    concept_dictionaryfile = ''
    try:
        opts, args = getopt.getopt(argv,"hi:o:d:",["ifile=","ofile=","dfile="])
    except getopt.GetoptError:<commit_msg>bug fixes<commit_after>import csv
import sys, getopt
import uuid
import copy
from datetime import datetime
def date(datestr="", format="%Y%m%d"):
    if not datestr:
        return datetime.today().date()
    return datetime.strptime(datestr, format).date()

def coded(row, writer, name, datatype, wcount, concept_list, concepts_dict_list):
    answers  = [row['answer.1'].strip(),row['answer.2'].strip(),row['answer.3'].strip(),row['answer.4'].strip(),row['answer.5'].strip(),row['answer.6'].strip(),row['answer.7'].strip(),row['answer.8'].strip(),row['answer.9'].strip(),row['answer.10'].strip(),row['answer.12'].strip(),row['answer.12'].strip()] 
    answer_1  = row['answer.1']
    answer_2  = row['answer.2']
    answer_3  = row['answer.3']
    answer_4  = row['answer.4']
    answer_5  = row['answer.5']
    answer_6  = row['answer.6']
    answer_7  = row['answer.7']
    answer_8  = row['answer.8']
    answer_9  = row['answer.9']
    answer_10  = row['answer.10']
    answer_11  = row['answer.11']
    answer_12  = row['answer.12']
    reference_term_source  = row['reference-term-source']
    reference_term_code  = row['reference-term-code']
    reference_term_relationship  = row['reference-term-relationship']
    if name != '':
        for answer in answers:
            if answer != '':
                if answer not in concepts_dict_list:
                    if answer not in concept_list:
                        wcount = wcount + 1
                        concept_list.append(answer)
                        writer.writerow({'uuid':uuid.uuid1(),'name':answer,'class':'Misc','datatype':'N/A'})
        if name not in concept_list:
            wcount = wcount + 1
            concept_list.append(name)
            writer.writerow({'uuid':uuid.uuid1(),'name':name,'class':'Misc','datatype':datatype,'answer.1':answer_1,'answer.2':answer_2,'answer.3':answer_3,'answer.4':answer_4,'answer.5':answer_5,'answer.6':answer_6,'answer.7':answer_7,'answer.8':answer_8,'answer.9':answer_9,'answer.10':answer_10})        
    return wcount, concept_list

def single(row, writer, name, datatype, wcount, concept_list, concepts_dict_list):
    units  = row['units']
    High_Normal  = row['High Normal']
    Low_Normal  = row['Low Normal']
    Allow_Decimal  = row['Allow Decimal']
    locale  = row['locale']
    synonym_1  = row['synonym.1']
    reference_term_source  = row['reference-term-source']
    reference_term_code  = row['reference-term-code']
    reference_term_relationship  = row['reference-term-relationship']
    if datatype == '':
        datatype = 'Text'
    if name != '':
        if name not in concepts_dict_list:
            if name not in concept_list :
                wcount = wcount + 1
                concept_list.append(name)
                writer.writerow({'uuid':uuid.uuid1(),'name':name,'class':'Misc','datatype':datatype,'High Normal':High_Normal,'Low Normal':Low_Normal})        
    return wcount, concept_list

def get_concepts_list(dictionary_file_list):
    # file_data is the text of the file, not the filename
    concepts_dict_list = []
    for dictionary_file in dictionary_file_list.split(':'):
        with open(dictionary_file, encoding="utf8") as in_csvfile:
            reader = csv.DictReader(in_csvfile, quotechar='"')    
    #        reader = csv.DictReader(dictionary_file, ('Concept Id','Name','Description','Synonyms','Answers','Set Members','Class','Datatype','Changed By','Creator'))
            for row in reader:
                concepts_dict_list.append(row['Name'] if 'Name' in row else row['name'])

    return concepts_dict_list

def main(argv):
    inputfile = ''
    outputfile = ''
    concept_dictionaryfile = ''
    try:
        opts, args = getopt.getopt(argv,"hi:o:d:",["ifile=","ofile=","dfile="])
    except getopt.GetoptError:
{'fraction_non_alphanumeric': 0.16061827956989247, 'fraction_numerical': 0.02043010752688172, 'mean_word_length': 3.3565573770491803, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4773184', 'n_tokens_mistral': 2407, 'n_tokens_neox': 2334, 'n_words': 406}