<commit_before>"""
    The goal is to build JSON dictionaries that can be used to search for codes in
    sources and map to elements in the CDM vocabulary.
"""

import csv
import json
import os
import argparse
import sys


def open_csv_file(file_name, mode="r"):

    ver_info = sys.version_info[0]
    if ver_info == 2:
        return open(file_name, mode=mode + "b")
    else:
        return open(file_name, newline="", mode=mode)


def main(source_vocabulary_directory, output_json_directory=None, delimiter="\t"):
    """Build files for needed vocabulary"""
    if output_json_directory is None:
        output_json_directory = source_vocabulary_directory

    concept_csv = os.path.join(source_vocabulary_directory, "CONCEPT.csv")
    vocabularies = []

    # Determine which vocabularies are in the concept file
    print("Scanning '%s'" % os.path.abspath(concept_csv))
    with open_csv_file(concept_csv, "r") as f:
        dict_reader = csv.DictReader(f, delimiter=delimiter)
        i = 0<commit_msg>Enforce utf8 encoding in reading CSV files<commit_after>"""
    The goal is to build JSON dictionaries that can be used to search for codes in
    sources and map to elements in the CDM vocabulary.
"""

import csv
import json
import os
import argparse
import sys


def open_csv_file(file_name, mode="r"):

    ver_info = sys.version_info[0]
    if ver_info == 2:
        return open(file_name, mode=mode + "b")
    else:
        return open(file_name, newline="", mode=mode, encoding="utf8")


def main(source_vocabulary_directory, output_json_directory=None, delimiter="\t"):
    """Build files for needed vocabulary"""
    if output_json_directory is None:
        output_json_directory = source_vocabulary_directory

    concept_csv = os.path.join(source_vocabulary_directory, "CONCEPT.csv")
    vocabularies = []

    # Determine which vocabularies are in the concept file
    print("Scanning '%s'" % os.path.abspath(concept_csv))
    with open_csv_file(concept_csv, "r") as f:
        dict_reader = csv.DictReader(f, delimiter=delimiter)
        i = 0
{'fraction_non_alphanumeric': 0.09486931268151017, 'fraction_numerical': 0.003872216844143272, 'mean_word_length': 3.852112676056338, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4039480', 'n_tokens_mistral': 669, 'n_tokens_neox': 641, 'n_words': 192}