<commit_before>#!/usr/bin/env python3
''' Script to build all dictionaries with all formats
    Usage:
    ./bin/convert_all.py --input_folder=./ext-dict --output_folder=./ext-output --extension=tab
'''

import argparse
from fileinput import filename
import glob
import os.path
import os
import subprocess
from iso_language_codes import language_name
import shlex

def readDicInfo(filepath):
    ''' Read metadata of dictionary with the following format

            Name = Dictionary of xyz
            Description = Description of this dictionary
            Source = en
            Target = vi
            Inflections = "NoInflections.txt"
            Version = 1.1

        Target and Source are the ISO 2-character codes of the language.
        Name, Source and Target are mandatory fields.
    '''

    valuemap = {}

    try:
        file = open(filepath, encoding='utf-8')

        lines = file.readlines()

        for line in lines:
            line = line.strip()

            if not line:
                continue
         
            key, value = line.split('=')

            valuemap[key.strip()] = value.strip()

        valuemap['FullSource'] = language_name(valuemap['Source'])
        valuemap['FullTarget'] = language_name(valuemap['Target'])

        keys = ['Name', 'Source', 'Target'] # Mandatory keys

        for i, k in enumerate(keys):
            if not k in valuemap:
                print(f'Missing key: "{keys[i]}"')

                return None
            
    except IOError as err:
        print(err)
        return None

#    print(valuemap)

    return valuemap


def main() -> None:
    parser = argparse.ArgumentParser(description='Convert all dictionaries in a folder',
        usage='Usage: python convert_all.py --input-folder ./mydictdata --output-folder ./myoutputdict --extension tsv')
    parser.add_argument('-i', '--input_folder', help='Input folder containing .tsv and .dfo files')
    parser.add_argument('-o', '--output_folder', help='Output folder containing dictionary files')
    parser.add_argument('-e', '--extension', default='tsv', help='Filename extention for input dictionary files. Default is .tsv')
    parser.add_argument('-m', '--metadata', default='dfo', help='Filename extention for input metadata for dictionary. Default is .dfo')

    args, array = parser.parse_known_args()

    input_folder = args.input_folder.replace(' ', '\\ ') 
    output_folder = args.output_folder.replace(' ', '\\ ') 
    extension = args.extension
    metadata = args.metadata<commit_msg>Add required argument<commit_after>#!/usr/bin/env python3
''' Script to build all dictionaries with all formats
    Usage:
    ./bin/convert_all.py --input_folder=./ext-dict --output_folder=./ext-output --extension=tab
'''

import argparse
from fileinput import filename
import glob
import os.path
import os
import subprocess
from iso_language_codes import language_name
import shlex

def readDicInfo(filepath):
    ''' Read metadata of dictionary with the following format

            Name = Dictionary of xyz
            Description = Description of this dictionary
            Source = en
            Target = vi
            Inflections = "NoInflections.txt"
            Version = 1.1

        Target and Source are the ISO 2-character codes of the language.
        Name, Source and Target are mandatory fields.
    '''

    valuemap = {}

    try:
        file = open(filepath, encoding='utf-8')

        lines = file.readlines()

        for line in lines:
            line = line.strip()

            if not line:
                continue
         
            key, value = line.split('=')

            valuemap[key.strip()] = value.strip()

        valuemap['FullSource'] = language_name(valuemap['Source'])
        valuemap['FullTarget'] = language_name(valuemap['Target'])

        keys = ['Name', 'Source', 'Target'] # Mandatory keys

        for i, k in enumerate(keys):
            if not k in valuemap:
                print(f'Missing key: "{keys[i]}"')

                return None
            
    except IOError as err:
        print(err)
        return None

#    print(valuemap)

    return valuemap


def main() -> None:
    parser = argparse.ArgumentParser(description='Convert all dictionaries in a folder',
        usage='Usage: python convert_all.py --input-folder=./mydictdata --output-folder=./myoutputdict --extension=tsv')
    parser.add_argument('-i', '--input_folder', required=True, help='Input folder containing .tsv and .dfo files')
    parser.add_argument('-o', '--output_folder', required=True, help='Output folder containing dictionary files')
    parser.add_argument('-e', '--extension', default='tsv', help='Filename extention for input dictionary files. Default is .tsv')
    parser.add_argument('-m', '--metadata', default='dfo', help='Filename extention for input metadata for dictionary. Default is .dfo')

    args, array = parser.parse_known_args()

    input_folder = args.input_folder.replace(' ', '\\ ') 
    output_folder = args.output_folder.replace(' ', '\\ ') 
    extension = args.extension
    metadata = args.metadata
{'fraction_non_alphanumeric': 0.11496278887583235, 'fraction_numerical': 0.0019584802193497847, 'mean_word_length': 2.5564066852367686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3554398', 'n_tokens_mistral': 1452, 'n_tokens_neox': 1393, 'n_words': 416}