<commit_before>import argparse
import json
import logging
import subprocess
import sys
import hashlib

from wand.image import Image
import coloredlogs

from basset.exceptions import *


class Converter:
    def __init__(self):
        coloredlogs.install()
        self.input_dir = ""
        self.output_dir = ""
        self.force_convert = False
        self.converted_files_hashes = {}

    @staticmethod
    def allowed_image_types():
        return ["eps", "pdf", "svg", "psd"]

    @staticmethod
    def sha1_of_file(file_path):
        sha = hashlib.sha1()
        with open(file_path, 'rb') as f:
            for line in f:
                sha.update(line)
            return sha.hexdigest()

    def convert_single_file(self, source_file, destination_file, target_resolution):
        sha1_of_original_file = self.sha1_of_file(source_file)

        self.converted_files_hashes[destination_file] = sha1_of_original_file
        convert_string = "convert \"{0}\" " \
                         "-resize {1}x{2} " \
                         "-density {3}x{4} " \
                         "\"{5}\"".format(source_file,
                                          target_resolution[0],
                                          target_resolution[1],
                                          target_resolution[0],
                                          target_resolution[1],
                                          destination_file)

        os.system(convert_string)

    def check_if_file_needs_reconverting(self, source_file, destination_file):
        sha1_of_original_file = self.sha1_of_file(source_file)

        destination_file_missing = not os.path.isfile(destination_file)
        destination_file_was_generated_from_the_different_file = destination_file in self.converted_files_hashes and \
                                                                 self.converted_files_hashes[
                                                                     destination_file] != sha1_of_original_file

        return self.force_convert or destination_file_missing or destination_file_was_generated_from_the_different_file

    @staticmethod
    def size_of_image(path):
        raw = subprocess.check_output("identify -verbose \"{0}\" | grep Resolution:".format(path),
                                      shell=True).decode("utf-8")
        resolution = raw.replace("Resolution::", "").strip(" \t\n\r")

        resolution_parts = resolution.split("x")

        return resolution_parts[0], resolution_parts[1]

    def convert(self):
        self.input_dir = self.input_dir.rstrip('\\/')
        self.output_dir = self.output_dir.rstrip('\\/')

        logging.info("Converting vector files from {0} to {1}".format(self.input_dir, self.output_dir))

        temp_file = os.path.join(self.output_dir, ".basset_temp")
        if os.path.isfile(temp_file):
            with open(temp_file, "r") as data_file:
                self.converted_files_hashes = json.load(data_file)

        self.check_if_input_dir_contains_vector_assets()
        self.check_if_input_dir_contains_xcassets()

        converted_files_count = 0
        for original_base_path, subdirectories, files in os.walk(self.input_dir):

            for filename in files:
                if "." in filename and filename[0] is not ".":
                    basename = filename.split(".")[0]
                    extension = filename.split(".")[1].lower()

                    if extension in Converter.allowed_image_types():
                        new_base_path = original_base_path.replace(self.input_dir, self.output_dir)
                        if not os.path.exists(new_base_path):
                            os.makedirs(new_base_path)
                        original_full_path = os.path.join(original_base_path, filename)

                        destination_templates = [(1, ".png")]
                        if extension is not "png":
                            destination_templates.append((2, "@2x.png"))
                            destination_templates.append((3, "@3x.png"))

                        selected_destination_templates = []
                        for template in destination_templates:
                            destination_path = os.path.join(new_base_path, basename + template[1])
                            if self.check_if_file_needs_reconverting(original_full_path, destination_path):
                                selected_destination_templates.append(template)

                        if len(selected_destination_templates) > 0:
                            with Image(filename=original_full_path) as img:
                                original_size = img.size

                            for template in selected_destination_templates:
                                converted_files_count += 1
                                new_image_size = (original_size[0] * template[0], original_size[1] * template[0])
                                destination_path = os.path.join(new_base_path, basename + template[1])

                                self.convert_single_file(original_full_path, destination_path, new_image_size)
                                logging.info("Converted {0} to {1}".format(original_full_path, destination_path))

        with open(temp_file, "w+") as data_file:
            json.dump(self.converted_files_hashes, data_file, indent=1)

        logging.info("Images conversion finished. Processed " + str(converted_files_count) + " images")

    def check_if_input_dir_contains_xcassets(self):
        for original_base_path, subdirectories, files in os.walk(self.input_dir):
            if original_base_path.endswith(".imageset"):
                raise AssetsDirContainsImagesetDirectoryException(original_base_path, self.input_dir)

    def check_if_input_dir_contains_vector_assets(self):
        directories_with_vector_files = {}
        if not os.path.isdir(self.input_dir):
            for path, subdirectories, files in os.walk(os.getcwd()):
                path = os.path.relpath(path, os.getcwd())
                for filename in files:
                    if "." in filename and filename[0] is not ".":
                        extension = filename.split(".")[1]

                        if extension.lower() in Converter.allowed_image_types():
                            top_dir_in_path = path.split(os.sep)[0]
                            if top_dir_in_path in directories_with_vector_files:
                                directories_with_vector_files[top_dir_in_path] += 1
                            else:
                                directories_with_vector_files[top_dir_in_path] = 1

            if directories_with_vector_files is not None:
                max_vector_files_count = -1
                directory_with_max_vector_files = None
                for path in directories_with_vector_files.keys():
                    if not path.endswith(".xcassets"):
                        if directories_with_vector_files[path] > max_vector_files_count:
                            max_vector_files_count = directories_with_vector_files[path]
                            directory_with_max_vector_files = path

                raise AssetsDirNotFoundException(directory_with_max_vector_files)


def main(args_to_parse):
    parser = argparse.ArgumentParser(description='Converts raw assets to proper PNG(s).')
    parser.add_argument('-i', '--input_dir', default="./Assets", help='directory with raw assets')
    parser.add_argument('-f', '--force_convert', default="False",
                        help='should regenerate assets even when they were generated before')
    parser.add_argument('-o', '--output_dir', default="./GeneratedAssets",
                        help='directory where generated PNG(s) will be stored')
    parsed_args = parser.parse_args(args_to_parse)

    converter = Converter()
    converter.input_dir = parsed_args.input_dir
    converter.output_dir = parsed_args.output_dir
    converter.output_dir = parsed_args.force_convert
    converter.convert()


if __name__ == '__main__':
    args = sys.argv[1:]
    main(args)
<commit_msg>Fixed image quality<commit_after>import argparse
import json
import logging
import subprocess
import sys
import hashlib

from wand.image import Image
import coloredlogs

from basset.exceptions import *


class Converter:
    def __init__(self):
        coloredlogs.install()
        self.input_dir = ""
        self.output_dir = ""
        self.force_convert = False
        self.converted_files_hashes = {}

    @staticmethod
    def allowed_image_types():
        return ["eps", "pdf", "svg", "psd"]

    @staticmethod
    def sha1_of_file(file_path):
        sha = hashlib.sha1()
        with open(file_path, 'rb') as f:
            for line in f:
                sha.update(line)
            return sha.hexdigest()

    def convert_single_file(self, source_file, destination_file, target_resolution, scale_factor):
        sha1_of_original_file = self.sha1_of_file(source_file)

        self.converted_files_hashes[destination_file] = sha1_of_original_file

        convert_string = "convert " \
                         "-density {0}00% " \
                         "\"{1}\" " \
                         "-resize {2}x{3} " \
                         "\"{4}\"".format(scale_factor,
                                          source_file,
                                          target_resolution[0],
                                          target_resolution[1],
                                          destination_file)

        os.system(convert_string)

    def check_if_file_needs_reconverting(self, source_file, destination_file):
        sha1_of_original_file = self.sha1_of_file(source_file)

        destination_file_missing = not os.path.isfile(destination_file)
        destination_file_was_generated_from_the_different_file = destination_file in self.converted_files_hashes and \
                                                                 self.converted_files_hashes[
                                                                     destination_file] != sha1_of_original_file

        return self.force_convert or destination_file_missing or destination_file_was_generated_from_the_different_file

    @staticmethod
    def size_of_image(path):
        raw = subprocess.check_output("identify -verbose \"{0}\" | grep Resolution:".format(path),
                                      shell=True).decode("utf-8")
        resolution = raw.replace("Resolution::", "").strip(" \t\n\r")

        resolution_parts = resolution.split("x")

        return resolution_parts[0], resolution_parts[1]

    def convert(self):
        self.input_dir = self.input_dir.rstrip('\\/')
        self.output_dir = self.output_dir.rstrip('\\/')

        logging.info("Converting vector files from {0} to {1}".format(self.input_dir, self.output_dir))

        temp_file = os.path.join(self.output_dir, ".basset_temp")
        if os.path.isfile(temp_file):
            with open(temp_file, "r") as data_file:
                self.converted_files_hashes = json.load(data_file)

        self.check_if_input_dir_contains_vector_assets()
        self.check_if_input_dir_contains_xcassets()

        converted_files_count = 0
        for original_base_path, subdirectories, files in os.walk(self.input_dir):

            for filename in files:
                if "." in filename and filename[0] is not ".":
                    basename = filename.split(".")[0]
                    extension = filename.split(".")[1].lower()

                    if extension in Converter.allowed_image_types():
                        new_base_path = original_base_path.replace(self.input_dir, self.output_dir)
                        if not os.path.exists(new_base_path):
                            os.makedirs(new_base_path)
                        original_full_path = os.path.join(original_base_path, filename)

                        destination_templates = [(1, ".png")]
                        if extension is not "png":
                            destination_templates.append((2, "@2x.png"))
                            destination_templates.append((3, "@3x.png"))

                        selected_destination_templates = []
                        for template in destination_templates:
                            destination_path = os.path.join(new_base_path, basename + template[1])
                            if self.check_if_file_needs_reconverting(original_full_path, destination_path):
                                selected_destination_templates.append(template)

                        if len(selected_destination_templates) > 0:
                            with Image(filename=original_full_path) as img:
                                original_size = img.size

                            for template in selected_destination_templates:
                                converted_files_count += 1
                                new_image_size = (original_size[0] * template[0], original_size[1] * template[0])
                                destination_path = os.path.join(new_base_path, basename + template[1])

                                self.convert_single_file(original_full_path, destination_path, new_image_size,
                                                         template[0])
                                logging.info("Converted {0} to {1}".format(original_full_path, destination_path))

        with open(temp_file, "w+") as data_file:
            json.dump(self.converted_files_hashes, data_file, indent=1)

        logging.info("Images conversion finished. Processed " + str(converted_files_count) + " images")

    def check_if_input_dir_contains_xcassets(self):
        for original_base_path, subdirectories, files in os.walk(self.input_dir):
            if original_base_path.endswith(".imageset"):
                raise AssetsDirContainsImagesetDirectoryException(original_base_path, self.input_dir)

    def check_if_input_dir_contains_vector_assets(self):
        directories_with_vector_files = {}
        if not os.path.isdir(self.input_dir):
            for path, subdirectories, files in os.walk(os.getcwd()):
                path = os.path.relpath(path, os.getcwd())
                for filename in files:
                    if "." in filename and filename[0] is not ".":
                        extension = filename.split(".")[1]

                        if extension.lower() in Converter.allowed_image_types():
                            top_dir_in_path = path.split(os.sep)[0]
                            if top_dir_in_path in directories_with_vector_files:
                                directories_with_vector_files[top_dir_in_path] += 1
                            else:
                                directories_with_vector_files[top_dir_in_path] = 1

            if directories_with_vector_files is not None:
                max_vector_files_count = -1
                directory_with_max_vector_files = None
                for path in directories_with_vector_files.keys():
                    if not path.endswith(".xcassets"):
                        if directories_with_vector_files[path] > max_vector_files_count:
                            max_vector_files_count = directories_with_vector_files[path]
                            directory_with_max_vector_files = path

                raise AssetsDirNotFoundException(directory_with_max_vector_files)


def main(args_to_parse):
    parser = argparse.ArgumentParser(description='Converts raw assets to proper PNG(s).')
    parser.add_argument('-i', '--input_dir', default="./Assets", help='directory with raw assets')
    parser.add_argument('-f', '--force_convert', default="False",
                        help='should regenerate assets even when they were generated before')
    parser.add_argument('-o', '--output_dir', default="./GeneratedAssets",
                        help='directory where generated PNG(s) will be stored')
    parsed_args = parser.parse_args(args_to_parse)

    converter = Converter()
    converter.input_dir = parsed_args.input_dir
    converter.output_dir = parsed_args.output_dir
    converter.output_dir = parsed_args.force_convert
    converter.convert()


if __name__ == '__main__':
    args = sys.argv[1:]
    main(args)

{'fraction_non_alphanumeric': 0.08707658321060383, 'fraction_numerical': 0.0062592047128129605, 'mean_word_length': 1.9018874643874644, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5740224', 'n_tokens_mistral': 4429, 'n_tokens_neox': 4217, 'n_words': 789}