<commit_before>#!/usr/bin/python

import os, os.path, sys
from shutil import copyfile
from PIL import Image
from collections import defaultdict

__pretend__ = False
extensions = ['.jpg', '.jpeg', '.png']

def process_file(input_file, out_path, name, image_type):
    if not os.path.isdir(out_path):
        os.makedirs(out_path)

    output_file = "%s/%s.png" % (out_path, image_type)
    print("Copying file: %s to %s" % (input_file, output_file))
    if not __pretend__ : copyfile(input_file, output_file)

def process_directory(directory, files, out_dir):
    valid_zreps = defaultdict(int)
    dashes_counter = defaultdict(int)

    for name in files:
        file_name, file_extension = os.path.splitext(name)
        splitted_name = file_name.split("_")
        material_zrep = splitted_name[-1]

        if ( file_extension in extensions and 
                  material_zrep.isdigit() and 
                  len(material_zrep) == 6 and
                  valid_zreps[material_zrep] < 2 and
                  file_name[:2] != '._' ) :
            valid_zreps[material_zrep] += 1

            if(dashes_counter[material_zrep] < len(splitted_name)):
                image_type = 'package'
            else:
                image_type = 'box'

            out_path = "%s/%s" % (out_dir, material_zrep)
            input_file = "%s/%s" % (directory, name)
            process_file(input_file, out_path, name, image_type)
            dashes_counter[material_zrep] = len(splitted_name)

def main():
    if(len(sys.argv) != 3):
        print("Usage: %s <input directory> <output directory>" % (sys.argv[0]))<commit_msg>Image type detection minor refactor<commit_after>#!/usr/bin/python

import os, os.path, sys
from shutil import copyfile
from PIL import Image
from collections import defaultdict

__pretend__ = False
extensions = ['.jpg', '.jpeg', '.png']

def process_file(input_file, out_path, name, image_type):
    if not os.path.isdir(out_path):
        os.makedirs(out_path)

    output_file = "%s/%s.png" % (out_path, image_type)
    print("Copying file: %s to %s" % (input_file, output_file))
    if not __pretend__ : copyfile(input_file, output_file)

def process_directory(directory, files, out_dir):
    valid_zreps = defaultdict(int)
    dashes_counter = defaultdict(int)

    for name in files:
        file_name, file_extension = os.path.splitext(name)
        splitted_name = file_name.split("_")
        material_zrep = splitted_name[-1]

        if ( file_extension in extensions and 
                  material_zrep.isdigit() and 
                  len(material_zrep) == 6 and
                  valid_zreps[material_zrep] < 2 and
                  file_name[:2] != '._' ) :
            valid_zreps[material_zrep] += 1

            if ( len(splitted_name) < dashes_counter[material_zrep] or
                  dashes_counter[material_zrep] == 0 ):
                image_type = 'package'
            else:
                image_type = 'box'

            out_path = "%s/%s" % (out_dir, material_zrep)
            input_file = "%s/%s" % (directory, name)
            process_file(input_file, out_path, name, image_type)
            dashes_counter[material_zrep] = len(splitted_name)

def main():
    if(len(sys.argv) != 3):
        print("Usage: %s <input directory> <output directory>" % (sys.argv[0]))
{'fraction_non_alphanumeric': 0.11644042232277527, 'fraction_numerical': 0.004524886877828055, 'mean_word_length': 2.6043478260869564, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2794916', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1061, 'n_words': 237}