<commit_before>import json
import logging
import logging.config
import pathlib
import sys
import pexpect
import yaml
import click
import collections
import io
import av

__author__ = 'cody'

cfg = {}


def configure_logging(logging_cfg):
    logging.config.dictConfig(logging_cfg)


def load_config(cfg_file):
    return yaml.load(cfg_file)


def get_config_value(key, default=None):
    if key in cfg:
        return cfg[key]
    return default


def get_video_paths():
    directories = [pathlib.Path(d) for d in get_config_value("directories", [])]
    globs = ["**/*.{0}".format(e) for e in get_config_value("extensions", ["avi"])]

    video_files = []
    for directory in directories:
        logging.info("Searching directory: {0}".format(directory))
        for glob in globs:
            video_files.extend(directory.glob(glob))

    return video_files


def get_video_info(path):
    options = [
        "-v", "quiet",
        "-print_format", "json",
        "-show_format",
        "-show_streams"
    ]

    child = pexpect.spawnu("ffprobe", options + [str(path)], logfile=sys.stdout)
    child.expect(pexpect.EOF)
    ffprobe_json = child.before
    return json.loads(ffprobe_json)


def get_audio_streams_for_file(path):
    video_info = get_video_info(path)

    audio_streams = []

    if "streams" in video_info:
        audio_streams = [s for s in video_info["streams"] if s["codec_type"] == "audio"]

    return audio_streams


with click.open_file("config.yml") as cfg_file:
    cfg.update(load_config(cfg_file))

if "logging" not in cfg:
    click.secho("No logging configuration found", fg="red")
else:
    configure_logging(cfg["logging"])

logging.getLogger(__name__).info("Hello")

video_files = get_video_paths()

logging.root.info("Found {0} files to check".format(len(video_files)))

# audio_streams = get_audio_streams_for_file(video_files[0])
#
# print(audio_streams)
# print([a["codec_name"] for a in audio_streams])


video_info = get_video_info(video_files[0])

print(video_info)

# 1) first audio stream is aac
<commit_msg>refactored and cleaned code<commit_after>import json
import logging
import logging.config
import pathlib
import sys
import pexpect
import yaml
import io
# import click
import collections
import io
# import av

__author__ = 'cody'

cfg = {}


def configure_logging(logging_cfg):
    logging.config.dictConfig(logging_cfg)


def load_config(cfg_file):
    return yaml.load(cfg_file)


def get_config_value(key, default=None):
    if key in cfg:
        return cfg[key]
    return default


def get_video_paths():
    directories = [pathlib.Path(d) for d in get_config_value("directories", [])]
    globs = ["**/*.{0}".format(e) for e in get_config_value("extensions", ["avi"])]

    video_files = []
    for directory in directories:
        logging.info("Searching directory: {0}".format(directory))
        for glob in globs:
            video_files.extend(directory.glob(glob))

    return video_files


def get_video_info(path):
    options = [
        "-v", "quiet",
        "-print_format", "json",
        "-show_format",
        "-show_streams"
    ]

    child = pexpect.spawnu("ffprobe", options + [str(path)], logfile=sys.stdout)
    child.expect(pexpect.EOF)
    ffprobe_json = child.before
    return json.loads(ffprobe_json)


def get_audio_streams_for_file(path):
    video_info = get_video_info(path)

    audio_streams = []

    if "streams" in video_info:
        audio_streams = [s for s in video_info["streams"] if s["codec_type"] == "audio"]

    return audio_streams


def build_ffmpeg_args(video_info):
    # http://ffmpeg.org/ffmpeg.html#Advanced-options
    # https://trac.ffmpeg.org/wiki/How%20to%20use%20-map%20option

    # build output map

    pass

with io.open("config.yml") as cfg_file:
    cfg.update(load_config(cfg_file))

if "logging" not in cfg:
    # click.secho("No logging configuration found", fg="red")
    print("No logging configuration found")
else:
    configure_logging(cfg["logging"])

logging.getLogger(__name__).info("Hello")

video_files = get_video_paths()

logging.root.info("Found {0} files to check".format(len(video_files)))

# audio_streams = get_audio_streams_for_file(video_files[0])
#
# print(audio_streams)
# print([a["codec_name"] for a in audio_streams])


video_info = get_video_info(video_files[0])

print(video_info)

# collect audio streams
audio_streams = [s for s in video_info.get("streams", []) if s.get("codec_type", "").lower() == "audio"]

# 1) first audio stream is aac
if not len(audio_streams) or audio_streams[0].get("codec_name", "").lower() == "aac":
    # TODO: exit?
    pass



{'fraction_non_alphanumeric': 0.12797747942832394, 'fraction_numerical': 0.0045474231268947595, 'mean_word_length': 4.026115342763874, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4677450', 'n_tokens_mistral': 1598, 'n_tokens_neox': 1548, 'n_words': 348}