<commit_before>#!/usr/bin/env python

import os.path
import subprocess

def get_usable_ffmpeg(cmd):
    try:
        p = subprocess.Popen([cmd, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = p.communicate()
        vers = str(out, 'utf-8').split('\n')[0].split()
        assert (vers[0] == 'ffmpeg' and vers[2][0] > '0') or (vers[0] == 'avconv')
        #if the version is strange like 'N-1234-gd1111', set version to 2.0
        try:
            version = [int(i) for i in vers[2].split('.')]
        except:
            version = [1, 0]
        return cmd, version
    except:
        return None

get_usable_ffmpeg_result = get_usable_ffmpeg('ffmpeg') or get_usable_ffmpeg('avconv')
if get_usable_ffmpeg_result:
    FFMPEG, FFMPEG_VERSION = get_usable_ffmpeg_result
else:
    FFMPEG, FFMPEG_VERSION = None, None

def has_ffmpeg_installed():
    return FFMPEG is not None

def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-i']
            params.append(file)
            params.append(output)
            subprocess.call(params)

    return

def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
    # Use concat demuxer on FFmpeg >= 1.1<commit_msg>set FFMPEG and FFMPEG_VERSION gracefully<commit_after>#!/usr/bin/env python

import os.path
import subprocess

def get_usable_ffmpeg(cmd):
    try:
        p = subprocess.Popen([cmd, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = p.communicate()
        vers = str(out, 'utf-8').split('\n')[0].split()
        assert (vers[0] == 'ffmpeg' and vers[2][0] > '0') or (vers[0] == 'avconv')
        #if the version is strange like 'N-1234-gd1111', set version to 2.0
        try:
            version = [int(i) for i in vers[2].split('.')]
        except:
            version = [1, 0]
        return cmd, version
    except:
        return None

FFMPEG, FFMPEG_VERSION = get_usable_ffmpeg('ffmpeg') or get_usable_ffmpeg('avconv') or (None, None)

def has_ffmpeg_installed():
    return FFMPEG is not None

def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-i']
            params.append(file)
            params.append(output)
            subprocess.call(params)

    return

def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
    # Use concat demuxer on FFmpeg >= 1.1
{'fraction_non_alphanumeric': 0.1310483870967742, 'fraction_numerical': 0.018548387096774192, 'mean_word_length': 2.776255707762557, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '498835', 'n_tokens_mistral': 908, 'n_tokens_neox': 874, 'n_words': 203}