<commit_before>#!/usr/bin/env python3
import os.path
import shlex
import subprocess
import sys

import tqdm

from . import *
from .util import *

class FFmpegException(SplitterException):
	pass

debug("Loading modules...")
from .FFprobe import ffprobe as FFmpeg_probe

if sys.platform.startswith('win'):
	ffmpeg_executable = 'FFMPEG.EXE'
else:
	ffmpeg_executable = 'ffmpeg'
debug("FFmpeg is {}".format(ffmpeg_executable))

def FFmpeg_command(input_source, output_filename_pattern='{filepart}-%03d{output_ext}', **kwargs):
	dirname, basename = os.path.split(input_source)
	filepart, ext = os.path.splitext(basename)
	output_ext = kwargs.pop('output_ext', ext.upper())
	filters = kwargs.pop('filters', [])
	### arguments could go here before input source
	command = [ '-i', input_source ]
	if 'title' in kwargs:
		command += [ '-metadata', 'title='+kwargs.pop('title') ]
	if 'frames' in kwargs:
		command += '-f segment -map 0 -flags +global_header'.split()
		frame_splits = sorted(set( flatten(kwargs.pop('frames')) ), key=float)
		command += [ '-segment_frames', ','.join(frame_splits) ]
	elif 'splits' in kwargs: # these are decimal times
		command += '-f segment -map 0 -flags +global_header'.split()
		time_splits = sorted(set( flatten(kwargs.pop('splits')) ), key=float)
		command += [ '-segment_times', ','.join(time_splits) ]
	if ext.upper() in ['.ASF', '.WMV']:
		info("Direct stream copy disabled")
		output_ext = '.NUT'
	elif not filters:
		command += [ '-c:v', 'copy', '-c:a', 'copy' ]
	command.extend(filters)
	try:
		ofn = output_filename_pattern.format(**locals())
	except:
		warning("Output filename is {}, which is probably not what you want".format(output_filename_pattern))
		ofn = output_filename_pattern
	command += [ ofn ]
	for k, v in kwargs.items():
		debug("Extra parameter unused: {}={}".format(k, v))
	return [ffmpeg_executable, '-nostdin']+command
def parse_output(outs, errs='', returncode=None):
	warnings = [ 'deprecated pixel format used, make sure you did set range correctly',
				 'DTS discontinuity',
				 'Invalid timestamp',
				 'Non-increasing DTS',
				 'VBV buffer size not set, muxing may fail' ]
	def _parse(b, prefix='STDOUT', warnings=warnings, encoding=stream_encoding):
		lastframeline = ''
		line = b.decode(encoding).rstrip()
		if 'Unrecognized option' in line:
			raise FFmpegException(line)
		elif 'At least one output file must be specified' in line:
			raise FFmpegException(line)<commit_msg>Should not cut on 0 or end-of-file<commit_after>#!/usr/bin/env python3
import os.path
import shlex
import subprocess
import sys

import tqdm

from . import *
from .util import *

class FFmpegException(SplitterException):
	pass

debug("Loading modules...")
from .FFprobe import ffprobe as FFmpeg_probe

if sys.platform.startswith('win'):
	ffmpeg_executable = 'FFMPEG.EXE'
else:
	ffmpeg_executable = 'ffmpeg'
debug("FFmpeg is {}".format(ffmpeg_executable))

def FFmpeg_command(input_source, output_filename_pattern='{filepart}-%03d{output_ext}', **kwargs):
	dirname, basename = os.path.split(input_source)
	filepart, ext = os.path.splitext(basename)
	output_ext = kwargs.pop('output_ext', ext.upper())
	filters = kwargs.pop('filters', [])
	### arguments could go here before input source
	command = [ '-i', input_source ]
	if 'title' in kwargs:
		command += [ '-metadata', 'title='+kwargs.pop('title') ]
	if 'frames' in kwargs:
		command += '-f segment -map 0 -flags +global_header'.split()
		frame_splits = sorted(set(f for f in flatten(kwargs.pop('frames')) if f)-set([0, '0']), key=float)
		command += [ '-segment_frames', ','.join(frame_splits) ]
	elif 'splits' in kwargs: # these are decimal times
		command += '-f segment -map 0 -flags +global_header'.split()
		time_splits = sorted(set(t for t in flatten(kwargs.pop('splits')) if t)-set([0, '0']), key=float)
		command += [ '-segment_times', ','.join(time_splits) ]
	if ext.upper() in ['.ASF', '.WMV']:
		info("Direct stream copy disabled")
		output_ext = '.NUT'
	elif not filters:
		command += [ '-c:v', 'copy', '-c:a', 'copy' ]
	command.extend(filters)
	try:
		ofn = output_filename_pattern.format(**locals())
	except:
		warning("Output filename is {}, which is probably not what you want".format(output_filename_pattern))
		ofn = output_filename_pattern
	command += [ ofn ]
	for k, v in kwargs.items():
		debug("Extra parameter unused: {}={}".format(k, v))
	return [ffmpeg_executable, '-nostdin']+command
def parse_output(outs, errs='', returncode=None):
	warnings = [ 'deprecated pixel format used, make sure you did set range correctly',
				 'DTS discontinuity',
				 'Invalid timestamp',
				 'Non-increasing DTS',
				 'VBV buffer size not set, muxing may fail' ]
	def _parse(b, prefix='STDOUT', warnings=warnings, encoding=stream_encoding):
		lastframeline = ''
		line = b.decode(encoding).rstrip()
		if 'Unrecognized option' in line:
			raise FFmpegException(line)
		elif 'At least one output file must be specified' in line:
			raise FFmpegException(line)
{'fraction_non_alphanumeric': 0.15065458207452165, 'fraction_numerical': 0.0030211480362537764, 'mean_word_length': 4.933094384707288, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1605557', 'n_tokens_mistral': 1764, 'n_tokens_neox': 1667, 'n_words': 455}