<commit_before>#!/usr/bin/python

import os
import sys
import re
import shutil
import subprocess
import argparse
import ffmpy
import json
import datetime

# Edit options here ##################################################
outmode = 'mp4'                                                     #Extension of output file
outformat = 'mp4'                                                   #Format of output file
remover = True                                                      #Delete original file after conversion complete
accept_ext = '3gp flv mov mp4 mkv avi divx m4v mpeg mpg wmv ts'     #Extensions of video files to convert

temp_path = "/tmp"                                          #Directory for encode prior to moving back
mkvextract_exe = "mkvextract"                               #Path to mkvextract executable
atomicparsely_exe = "AtomicParsley"                         #Path to AtomicParsley executable
mkvpropedit_exe = "mkvpropedit"                             #Path to mkvpropedit executable

strip_title = True                                          #Strip 'Title' field from metadata
video_codec = 'libx264'                                     #Video codec to use
video_type = 'h264'                                         #Name of video codec to check for remux
audio_codec = 'aac'                                         #Audio codec to use
audio_type = 'aac'                                          #Name of audio codec to check for remux
crf = "18"                                                  #Default crf
vbr = ""                                                    #Default audio vbr rate (CBR is used if this is blank)
extract_subtitle = True                                     #Extract subtitles?
subtitle_languages = "en eng english"                       #Codes for languages to extract
threads = 0                                                 #Number of threads to use in ffmpeg, 0 defaults to all
additional_ffmpeg = '-preset slow -movflags +faststart'     #Default Additional flags for ffmpeg, preset sets speed and compression, movflags to make file web optimized
deinterlace_ffmpeg = 'yadif'                                #Deinterlacing options

## END OPTIONS - DO NOT EDIT BELOW THIS LINE UNLESS YOU KNOW WHAT YOU ARE DOING ##

class color:
   PURPLE = '\033[95m'
   CYAN = '\033[96m'
   DARKCYAN = '\033[36m'
   BLUE = '\033[94m'
   GREEN = '\033[92m'
   YELLOW = '\033[93m'
   RED = '\033[91m'
   BOLD = '\033[1m'
   UNDERLINE = '\033[4m'
   END = '\033[0m'

parser = argparse.ArgumentParser(description='Simple mp4 converter')
parser.add_argument('-i','--input', help='Input file name/path', required=True)
parser.add_argument('-m','--mode',help='Processing mode', choices=['quality', 'speed'], required=True)
args = parser.parse_args()

if args.mode == 'speed':
    crf = "22"
    additional_ffmpeg = '-preset superfast -movflags +faststart'

if outmode == 'mp4':
    outformat = 'mp4'
elif outmode == 'mkv':
    outformat = 'matroska'

print("Validating FFMpeg configuration...\n")
        
format_resp = ffmpy.FFprobe(
    inputs={'': None},
    global_options=[
        '-v', 'quiet',
        '-formats', '2']
).run(stdout=subprocess.PIPE,stderr=subprocess.PIPE)

if ('E mp4' in format_resp[0]) and ('E matroska' in format_resp[0]):
    print("You have the suitable formats")
else:
    print("You do not have both the mkv and mp4 formats...Exiting!")
    exit(1)

codec_resp = ffmpy.FFprobe(
    inputs={'': None},
    global_options=[
        '-v', 'quiet',
        '-codecs', '2']
).run(stdout=subprocess.PIPE,stderr=subprocess.PIPE)

if video_codec in codec_resp[0]:
    print("Check " + video_codec + " Video Encoder ... OK")
else:
    print("Check " + video_codec + " Video Encoder ... NOK")
    exit(1)

if audio_codec in codec_resp[0]:
    print("Check " + audio_codec + " Audio Encoder ... OK")
else:
    print("Check " + audio_codec + " Audio Encoder ... NOK")
    exit(1)

print("Your FFMpeg is OK!\n\nEntering File Processing...\n")

subtitle_languages = subtitle_languages.lower()

def process_file(path, file):
    extension = os.path.splitext(file)[1].replace(".", "").lower()
    filename = os.path.splitext(file)[0]

    if extension in accept_ext:
        print(file + " is an acceptable extension. Checking file...")
    else:
        print(file + " is not an acceptable extension. Skipping...")
        return

    try:
        tup_resp = ffmpy.FFprobe(
            inputs={os.path.join(path, file): None},
            global_options=[
                '-v', 'quiet',
                '-print_format', 'json',
                '-show_format', '-show_streams']
        ).run(stdout=subprocess.PIPE)
    
        metaData = json.loads(tup_resp[0].decode('utf-8'))
    except Exception as file_info_ex:
        print("File " + file + " is unable to be converted. Adding .PROBE_FAIL to file")
        shutil.move(os.path.join(path, filename + "." + extension), os.path.join(path, filename + "." + extension + ".PROBE_FAIL"))
        return

    encode_crf = []
    for vs in metaData["streams"]:
        if "codec_type" in vs and vs["codec_type"] == "video":
            if vs["codec_name"] == video_type:
                vcodec = 'copy'
                print("Video in stream " + str(vs["index"]) + " is " + video_type + ", no conversion needed...")
            else:
                vcodec = video_codec
                if crf:
                    encode_crf = ["-crf", "" + crf]
                print("Video in stream " + str(vs["index"]) + " is currently " + color.BOLD + color.YELLOW + vs["codec_name"] + color.END + ". Converting to " + color.BOLD + color.YELLOW + video_type + color.END + "...")

    encode_dif = []
    for vs in metaData["streams"]:
        if "codec_type" in vs and vs["codec_type"] == "video":
            if "pix_fmt" in vs and vs["pix_fmt"].find("top first") != -1:
                if deinterlace_ffmpeg:
                    encode_dif = ["-vf", "" + deinterlace_ffmpeg]
                print("Video in stream " + str(vs["index"]) + " is interlaced, deinterlacing...")
            else:
                print("Video in stream " + str(vs["index"]) + " is not interlaced, no deinterlacing needed...")

    encode_vbr = []
    for vs in metaData["streams"]:
        if "codec_type" in vs and vs["codec_type"] == "audio":
            if vs["codec_name"] == audio_type:
                acodec = 'copy'
                print("Audio in stream " + str(vs["index"]) + " is " + audio_type + ", no conversion needed...")
            else:<commit_msg>Bug fixes mjpeg ignored as a video codec<commit_after>#!/usr/bin/python

import os
import sys
import re
import shutil
import subprocess
import argparse
import ffmpy
import json
import datetime

# Edit options here ##################################################
outmode = 'mp4'                                                     #Extension of output file
outformat = 'mp4'                                                   #Format of output file
remover = True                                                      #Delete original file after conversion complete
accept_ext = '3gp flv mov mp4 mkv avi divx m4v mpeg mpg wmv ts'     #Extensions of video files to convert

temp_path = "/tmp"                                          #Directory for encode prior to moving back
mkvextract_exe = "mkvextract"                               #Path to mkvextract executable
atomicparsely_exe = "AtomicParsley"                         #Path to AtomicParsley executable
mkvpropedit_exe = "mkvpropedit"                             #Path to mkvpropedit executable

strip_title = True                                          #Strip 'Title' field from metadata
video_codec = 'libx264'                                     #Video codec to use
video_type = 'h264'                                         #Name of video codec to check for remux
audio_codec = 'aac'                                         #Audio codec to use
audio_type = 'aac'                                          #Name of audio codec to check for remux
crf = "18"                                                  #Default crf
vbr = ""                                                    #Default audio vbr rate (CBR is used if this is blank)
extract_subtitle = True                                     #Extract subtitles?
subtitle_languages = "en eng english"                       #Codes for languages to extract
threads = 0                                                 #Number of threads to use in ffmpeg, 0 defaults to all
additional_ffmpeg = '-preset slow -movflags +faststart'     #Default Additional flags for ffmpeg, preset sets speed and compression, movflags to make file web optimized
deinterlace_ffmpeg = 'yadif'                                #Deinterlacing options

## END OPTIONS - DO NOT EDIT BELOW THIS LINE UNLESS YOU KNOW WHAT YOU ARE DOING ##

class color:
   PURPLE = '\033[95m'
   CYAN = '\033[96m'
   DARKCYAN = '\033[36m'
   BLUE = '\033[94m'
   GREEN = '\033[92m'
   YELLOW = '\033[93m'
   RED = '\033[91m'
   BOLD = '\033[1m'
   UNDERLINE = '\033[4m'
   END = '\033[0m'

parser = argparse.ArgumentParser(description='Simple mp4 converter')
parser.add_argument('-i','--input', help='Input file name/path', required=True)
parser.add_argument('-m','--mode',help='Processing mode', choices=['quality', 'speed'], required=True)
args = parser.parse_args()

if args.mode == 'speed':
    crf = "22"
    additional_ffmpeg = '-preset superfast -movflags +faststart'

if outmode == 'mp4':
    outformat = 'mp4'
elif outmode == 'mkv':
    outformat = 'matroska'

print("Validating FFMpeg configuration...\n")
        
format_resp = ffmpy.FFprobe(
    inputs={'': None},
    global_options=[
        '-v', 'quiet',
        '-formats', '2']
).run(stdout=subprocess.PIPE,stderr=subprocess.PIPE)

if ('E mp4' in format_resp[0]) and ('E matroska' in format_resp[0]):
    print("You have the suitable formats")
else:
    print("You do not have both the mkv and mp4 formats...Exiting!")
    exit(1)

codec_resp = ffmpy.FFprobe(
    inputs={'': None},
    global_options=[
        '-v', 'quiet',
        '-codecs', '2']
).run(stdout=subprocess.PIPE,stderr=subprocess.PIPE)

if video_codec in codec_resp[0]:
    print("Check " + video_codec + " Video Encoder ... OK")
else:
    print("Check " + video_codec + " Video Encoder ... NOK")
    exit(1)

if audio_codec in codec_resp[0]:
    print("Check " + audio_codec + " Audio Encoder ... OK")
else:
    print("Check " + audio_codec + " Audio Encoder ... NOK")
    exit(1)

print("Your FFMpeg is OK!\n\nEntering File Processing...\n")

subtitle_languages = subtitle_languages.lower()

def process_file(path, file):
    extension = os.path.splitext(file)[1].replace(".", "").lower()
    filename = os.path.splitext(file)[0]

    if extension in accept_ext:
        print(file + " is an acceptable extension. Checking file...")
    else:
        print(file + " is not an acceptable extension. Skipping...")
        return

    try:
        tup_resp = ffmpy.FFprobe(
            inputs={os.path.join(path, file): None},
            global_options=[
                '-v', 'quiet',
                '-print_format', 'json',
                '-show_format', '-show_streams']
        ).run(stdout=subprocess.PIPE)
    
        metaData = json.loads(tup_resp[0].decode('utf-8'))
    except Exception as file_info_ex:
        print("File " + file + " is unable to be converted. Adding .PROBE_FAIL to file")
        shutil.move(os.path.join(path, filename + "." + extension), os.path.join(path, filename + "." + extension + ".PROBE_FAIL"))
        return

    vcodec = ''
    acodec = ''
    encode_crf = []

    for vs in metaData["streams"]:
        if "codec_type" in vs and vs["codec_type"] == "video":
            if vs["codec_name"] == video_type:
                vcodec = 'copy'
                print("Video in stream " + str(vs["index"]) + " is " + video_type + ", no conversion needed...")
            elif vs["codec_name"] != 'mjpeg':
                vcodec = video_codec
                if crf:
                    encode_crf = ["-crf", "" + crf]
                print("Video in stream " + str(vs["index"]) + " is currently " + color.BOLD + color.YELLOW + vs["codec_name"] + color.END + ". Converting to " + color.BOLD + color.YELLOW + video_type + color.END + "...")

    encode_dif = []
    for vs in metaData["streams"]:
        if "codec_type" in vs and vs["codec_type"] == "video":
            if "field_order" in vs and vs["field_order"].find("progressive") == -1:
                if deinterlace_ffmpeg:
                    encode_dif = ["-vf", "" + deinterlace_ffmpeg]
                print("Video in stream " + str(vs["index"]) + " is interlaced, deinterlacing...")
            else:
                print("Video in stream " + str(vs["index"]) + " is not interlaced, no deinterlacing needed...")

    encode_vbr = []
    for vs in metaData["streams"]:
        if "codec_type" in vs and vs["codec_type"] == "audio":
            if vs["codec_name"] == audio_type:
                acodec = 'copy'
                print("Audio in stream " + str(vs["index"]) + " is " + audio_type + ", no conversion needed...")
            else:
{'fraction_non_alphanumeric': 0.12957062661204674, 'fraction_numerical': 0.012592929752693067, 'mean_word_length': 1.9792090395480226, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1486509', 'n_tokens_mistral': 3978, 'n_tokens_neox': 3767, 'n_words': 1097}