<commit_before># _*_ coding: utf-8 _*

__author__ = '<NAME>'

import os
import math
import json
import subprocess
import shutil
import tempfile
import shlex
import logging
from Tkinter import *
import re
import sys
import template
import gui


# 文字コード問題のためのおまじないです。　
# 実行時にインタプリタへのoption[ -S ]必須です。
ENC = sys.getfilesystemencoding()

# noinspection PyUnresolvedReferences
sys.setdefaultencoding(ENC)

ENV_TMP = os.environ.copy()
ENV_TMP['PYTHONIOENCODING'] = ENC

logger = logging.getLogger()

if getattr(sys, 'frozen', False):
    # frozen
    BASE_DIR = os.path.dirname(sys.executable)
    # noinspection PyProtectedMember
    if sys.platform == "win32" and sys._MEIPASS:
        # noinspection PyProtectedMember
        BASE_DIR = sys._MEIPASS
    else:
        BASE_DIR = os.path.dirname(sys.executable)

else:
    # unfrozen
    BASE_DIR = os.path.dirname(os.path.realpath(__file__))

TEMP_FOOTAGE_NAME = "TEMP_FOOTAGE_%05d"
RENAME_PATTERN = TEMP_FOOTAGE_NAME + "%s"
RAW_PAD_REG = re.compile('\d+$')
FMT_PAD_REG = re.compile('%\d+d$')
SEP_REG = re.compile('[\._]$')
ASCII_REG = re.compile(r'\A[\x00-\x7f]*\Z')
TEMPLATE = template.get_template()
config_f = open(template.get_config_path(), 'r')
CONFIG = json.load(config_f)
config_f.close()


def _get_sequence_path_dic(folder_path):
    """
    src_pathは_normalize_padding_file_nameで%4d等に正規化された状態ではいります。
    
    :param folder_path:
    :return: { src_path: { "minPad": int, "count": int }, ... }
    """
    dst_file_names_dic = {}
    for root, dirs, files in os.walk(folder_path):
        for file_name in files:
            normalize_name = _normalize_padding_file_name(file_name)
            if normalize_name is not None:
                base_name = os.path.splitext(file_name)[0]
                temp_path = os.path.join(root, normalize_name)
                if temp_path not in dst_file_names_dic:
                    dst_file_names_dic[temp_path] = {
                        "minPad": int(re.search(RAW_PAD_REG, base_name).group(0)),
                        "count": 0}
                else:
                    temp_pad = int(re.search(RAW_PAD_REG, base_name).group(0))
                    if dst_file_names_dic[temp_path]["minPad"] > temp_pad:
                        dst_file_names_dic[temp_path]["minPad"] = temp_pad
                    dst_file_names_dic[temp_path]["count"] += 1
    return dst_file_names_dic


def _get_unique_path_with_pad(path):
    """
    同名パスが存在したら[_#]でパディングしたパスを返す。
    :param path:
    :return:dst_path
    """
    if os.path.isdir(path):
        dst_path = path
        pad = 2
        while os.path.lexists(dst_path):
            dst_path = path + "_" + str(pad)
            pad += 1
        return dst_path
    else:
        (dir_path, base_name) = os.path.split(path)
        (name, ext) = os.path.splitext(base_name)
        dst_path = os.path.join(dir_path, name + ext)
        pad = 2
        while os.path.lexists(dst_path):
            dst_path = os.path.join(dir_path, name + "_" + str(pad) + ext)
            pad += 1
        return dst_path


def _get_movie_stats(src_path):
    if sys.platform == "win32":
        si = subprocess.STARTUPINFO()
        # noinspection PyUnresolvedReferences
        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
        # si.wShowWindow = subprocess.SW_HIDE # default
        p = subprocess.Popen([CONFIG['ffprobePath'], '-show_streams', '-print_format', 'json', src_path],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, startupinfo=si, env=ENV_TMP)
    else:
        p = subprocess.Popen([CONFIG['ffprobePath'], '-show_streams', '-print_format', 'json', src_path],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, env=ENV_TMP)
    out, err = p.communicate()
    print out
    # logger.debug("\n\tffprobe error:\n" + err)
    logger.debug("\n\tffprobe log:\n" + out)
    width = re.search('"width": (\d+)', out).group(1)
    height = re.search('"height": (\d+)', out).group(1)
    frame_rate = re.search('"r_frame_rate": "(\d+\.\d+|\d+)/', out).group(1)
    pix_fmt = re.search('"pix_fmt": "(.+)"', out).group(1)
    return dict({"width": width, "height": height, "frame_rate": frame_rate, "pix_fmt": pix_fmt})


def _make_dst_file_path(src_path):
    """
    paddingを消去して拡張子をmp4にしたパスを返します。
    :param src_path:
    :return dst_file_path:
    """
    (src_dir_path, src_file_name) = os.path.split(src_path)
    (base_name, ext) = os.path.splitext(src_file_name)<commit_msg>print 消し忘れ<commit_after># _*_ coding: utf-8 _*

__author__ = '<NAME>'

import os
import math
import json
import subprocess
import shutil
import tempfile
import shlex
import logging
from Tkinter import *
import re
import sys
import template
import gui


# 文字コード問題のためのおまじないです。　
# 実行時にインタプリタへのoption[ -S ]必須です。
ENC = sys.getfilesystemencoding()

# noinspection PyUnresolvedReferences
sys.setdefaultencoding(ENC)

ENV_TMP = os.environ.copy()
ENV_TMP['PYTHONIOENCODING'] = ENC

logger = logging.getLogger()

if getattr(sys, 'frozen', False):
    # frozen
    BASE_DIR = os.path.dirname(sys.executable)
    # noinspection PyProtectedMember
    if sys.platform == "win32" and sys._MEIPASS:
        # noinspection PyProtectedMember
        BASE_DIR = sys._MEIPASS
    else:
        BASE_DIR = os.path.dirname(sys.executable)

else:
    # unfrozen
    BASE_DIR = os.path.dirname(os.path.realpath(__file__))

TEMP_FOOTAGE_NAME = "TEMP_FOOTAGE_%05d"
RENAME_PATTERN = TEMP_FOOTAGE_NAME + "%s"
RAW_PAD_REG = re.compile('\d+$')
FMT_PAD_REG = re.compile('%\d+d$')
SEP_REG = re.compile('[\._]$')
ASCII_REG = re.compile(r'\A[\x00-\x7f]*\Z')
TEMPLATE = template.get_template()
config_f = open(template.get_config_path(), 'r')
CONFIG = json.load(config_f)
config_f.close()


def _get_sequence_path_dic(folder_path):
    """
    src_pathは_normalize_padding_file_nameで%4d等に正規化された状態ではいります。
    
    :param folder_path:
    :return: { src_path: { "minPad": int, "count": int }, ... }
    """
    dst_file_names_dic = {}
    for root, dirs, files in os.walk(folder_path):
        for file_name in files:
            normalize_name = _normalize_padding_file_name(file_name)
            if normalize_name is not None:
                base_name = os.path.splitext(file_name)[0]
                temp_path = os.path.join(root, normalize_name)
                if temp_path not in dst_file_names_dic:
                    dst_file_names_dic[temp_path] = {
                        "minPad": int(re.search(RAW_PAD_REG, base_name).group(0)),
                        "count": 0}
                else:
                    temp_pad = int(re.search(RAW_PAD_REG, base_name).group(0))
                    if dst_file_names_dic[temp_path]["minPad"] > temp_pad:
                        dst_file_names_dic[temp_path]["minPad"] = temp_pad
                    dst_file_names_dic[temp_path]["count"] += 1
    return dst_file_names_dic


def _get_unique_path_with_pad(path):
    """
    同名パスが存在したら[_#]でパディングしたパスを返す。
    :param path:
    :return:dst_path
    """
    if os.path.isdir(path):
        dst_path = path
        pad = 2
        while os.path.lexists(dst_path):
            dst_path = path + "_" + str(pad)
            pad += 1
        return dst_path
    else:
        (dir_path, base_name) = os.path.split(path)
        (name, ext) = os.path.splitext(base_name)
        dst_path = os.path.join(dir_path, name + ext)
        pad = 2
        while os.path.lexists(dst_path):
            dst_path = os.path.join(dir_path, name + "_" + str(pad) + ext)
            pad += 1
        return dst_path


def _get_movie_stats(src_path):
    if sys.platform == "win32":
        si = subprocess.STARTUPINFO()
        # noinspection PyUnresolvedReferences
        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
        # si.wShowWindow = subprocess.SW_HIDE # default
        p = subprocess.Popen([CONFIG['ffprobePath'], '-show_streams', '-print_format', 'json', src_path],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, startupinfo=si, env=ENV_TMP)
    else:
        p = subprocess.Popen([CONFIG['ffprobePath'], '-show_streams', '-print_format', 'json', src_path],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, env=ENV_TMP)
    out, err = p.communicate()
    # logger.debug("\n\tffprobe error:\n" + err)
    logger.debug("\n\tffprobe log:\n" + out)
    width = re.search('"width": (\d+)', out).group(1)
    height = re.search('"height": (\d+)', out).group(1)
    frame_rate = re.search('"r_frame_rate": "(\d+\.\d+|\d+)/', out).group(1)
    pix_fmt = re.search('"pix_fmt": "(.+)"', out).group(1)
    return dict({"width": width, "height": height, "frame_rate": frame_rate, "pix_fmt": pix_fmt})


def _make_dst_file_path(src_path):
    """
    paddingを消去して拡張子をmp4にしたパスを返します。
    :param src_path:
    :return dst_file_path:
    """
    (src_dir_path, src_file_name) = os.path.split(src_path)
    (base_name, ext) = os.path.splitext(src_file_name)
{'fraction_non_alphanumeric': 0.13187299480030976, 'fraction_numerical': 0.005531585352361987, 'mean_word_length': 2.6988543371522096, 'pattern_counts': {'":': 26, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6127485', 'n_tokens_mistral': 3374, 'n_tokens_neox': 3262, 'n_words': 554}