<commit_before>import argparse
import logging
import sys


# Set up logging
logger = logging.getLogger('combotron')
logger.setLevel(logging.DEBUG)

# Create console handler and set level to debug
log_console_handler = logging.StreamHandler()
log_console_handler.setLevel(logging.DEBUG)

# Prepare formatter for log messages
log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
log_console_handler.setFormatter(log_formatter)
logger.addHandler(log_console_handler)

# Set up the gear data
lenses = [{'name': '<NAME>', 'tag': 'JohnS'},
          {'name': 'Jane',   'tag': 'Jimmy'},
          {'name': 'Jimmy',  'tag': 'Jimmy'},
          {'name': '<NAME>', 'tag': 'KaimalMarkII'}]

films = [{'name': 'Blanko', 'tag': 'Blanko'},
         {'name': 'Ina''s 1969', 'tag': 'Inas1969'},
         {'name': '<NAME>', 'tag': 'KodotXGrizzled'},
         {'name': 'Ina''s 1982', 'tag': 'Inas1982'}]

flashes = [{'name': 'Standard', 'tag': 'Standard'},
           {'name': 'Dreampop', 'tag': 'Dreampop'},
           {'name': '<NAME>', 'tag': 'CherryShine'},
           {'name': '<NAME>', 'tag': 'CadetBlueGel'}]

localized_no_flash_strings = {'en': 'No Flash',
                              'zh-Hans': '没开闪光灯',
                              'zh-Hant': '無閃光燈',
                              'nl': 'geen flits',
                              'fr': 'sans flash',
                              'de': 'Kein Blitz',
                              'id': 'Tanpa Blitz',
                              'it': 'Nessun Flash',
                              'ja': 'フラッシなし',
                              'ko': '플래시 없음',
                              'ms': 'Tiada Lampu Kilat',
                              'pt-BR': 'Sem flash',
                              'pt-PT': 'sem flash',
                              'ru': 'без вспышки',
                              'es': 'sin flash',
                              'sv': 'No Blixt'}

def lens_part(parts):
    return parts[0]
    
def film_part(parts):
    return parts[1]
    
def flash_part(parts):
    return parts[2]

def metadata_language(md):
    lang = None
    if ' Lens, ' in md:
        lang = 'en'
    elif '-lens, ' in md:
        lang = 'nl'
    elif 'Objectif' in md:
        lang = 'fr'
    elif ' Lins, ' in md:
        lang = 'sv'
    elif ' Lentes, ' in md:
        lang = 'pt-BR'
    elif ' レンズ, ' in md:
        lang = 'ja'
    elif ' объектив, ' in md:
        lang = 'ru'
    elif ' Lensa, ' in md:
        lang = 'ms'
    elif ' 렌즈, ' in md:
        lang = 'ko'
    elif ' 镜头，' in md:
        lang = 'zh-Hans'
    elif ' 鏡頭, ' in md:
        lang = 'zh-Hant'
    else:
        if md.startswith('Lente '):
            if ', Pellicola ' in md:
                lang = 'it'
            elif ', rolo ' in md:
                lang = 'pt-PT'
            else:
                lang = 'es'
                
    return lang

part_info = {'en': [' Lens', ' Film', ' Flash'],
             'nl': ['-lens', '-filmrol', '-flitser']}<commit_msg>Added rest of the part information.<commit_after>import argparse
import logging
import sys


# Set up logging
logger = logging.getLogger('combotron')
logger.setLevel(logging.DEBUG)

# Create console handler and set level to debug
log_console_handler = logging.StreamHandler()
log_console_handler.setLevel(logging.DEBUG)

# Prepare formatter for log messages
log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
log_console_handler.setFormatter(log_formatter)
logger.addHandler(log_console_handler)

# Set up the gear data
lenses = [{'name': '<NAME>', 'tag': 'JohnS'},
          {'name': 'Jane',   'tag': 'Jimmy'},
          {'name': 'Jimmy',  'tag': 'Jimmy'},
          {'name': '<NAME>', 'tag': 'KaimalMarkII'}]

films = [{'name': 'Blanko', 'tag': 'Blanko'},
         {'name': 'Ina''s 1969', 'tag': 'Inas1969'},
         {'name': 'Kod<NAME>', 'tag': 'KodotXGrizzled'},
         {'name': 'Ina''s 1982', 'tag': 'Inas1982'}]

flashes = [{'name': 'Standard', 'tag': 'Standard'},
           {'name': 'Dreampop', 'tag': 'Dreampop'},
           {'name': '<NAME>', 'tag': 'CherryShine'},
           {'name': 'Cadet Blue Gel', 'tag': 'CadetBlueGel'}]

localized_no_flash_strings = {'en': 'No Flash',
                              'zh-Hans': '没开闪光灯',
                              'zh-Hant': '無閃光燈',
                              'nl': 'geen flits',
                              'fr': 'sans flash',
                              'de': 'Kein Blitz',
                              'id': 'Tanpa Blitz',
                              'it': 'Nessun Flash',
                              'ja': 'フラッシなし',
                              'ko': '플래시 없음',
                              'ms': 'Tiada Lampu Kilat',
                              'pt-BR': 'Sem flash',
                              'pt-PT': 'sem flash',
                              'ru': 'без вспышки',
                              'es': 'sin flash',
                              'sv': 'No Blixt'}

def lens_part(parts):
    return parts[0]
    
def film_part(parts):
    return parts[1]
    
def flash_part(parts):
    return parts[2]

def metadata_language(md):
    lang = None
    if ' Lens, ' in md:
        lang = 'en'
    elif '-lens, ' in md:
        lang = 'nl'
    elif 'Objectif' in md:
        lang = 'fr'
    elif ' Lins, ' in md:
        lang = 'sv'
    elif ' Lentes, ' in md:
        lang = 'pt-BR'
    elif ' レンズ, ' in md:
        lang = 'ja'
    elif ' объектив, ' in md:
        lang = 'ru'
    elif ' Lensa, ' in md:
        lang = 'ms'
    elif ' 렌즈, ' in md:
        lang = 'ko'
    elif ' 镜头，' in md:
        lang = 'zh-Hans'
    elif ' 鏡頭, ' in md:
        lang = 'zh-Hant'
    else:
        if md.startswith('Lente '):
            if ', Pellicola ' in md:
                lang = 'it'
            elif ', rolo ' in md:
                lang = 'pt-PT'
            else:
                lang = 'es'
                
    return lang


part_info = {'en': [' Lens', ' Film', ' Flash'],
             'nl': ['-lens', '-filmrol', '-flitser'],
             'fr': ['Objectif ', 'Film ', 'Flash '],
             'de': [' Linse', ' Film', ' Blitz'],
             'it': ['Lente ', 'Pellicola ', 'Flash '],
             'es': ['Lente ', 'Película ', 'Flash '],
             'pt-PT': ['Lente ', 'rolo ', 'flash '],
             'sv': [' Lins', ' Film', ' Blixt'],
             'pt-BR': [' Lentes', ' Filme', ' Flash'],
             'ja': [' レンズ', ' フィルム', ' フラッシュ'],
             'ru': [' объектив', ' пленка', ' вспышка'],
             'id': ['Lensa ', 'Film ', 'Blitz '],      
             'ms': [' Lensa', ' Filem', ' Lampu Kilat'],
             'ko': [' 렌즈', ' 필름', ' 플래시'],  
             'zh-Hans': [' 镜头', ' 胶卷', ' 闪光灯'],
             'zh-Hant': [' 鏡頭', ' 膠片', ' 閃光燈']}
{'fraction_non_alphanumeric': 0.17700029524653085, 'fraction_numerical': 0.005609684086211987, 'mean_word_length': 1.5575688939222347, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '284779', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2340, 'n_words': 565}