<commit_before>#!/usr/bin/env python3
# file: vid2webm.py
# vim:fileencoding=utf-8:fdm=marker:ft=python
#
# Copyright © 2018 <NAME> <<EMAIL>>.
# SPDX-License-Identifier: MIT
# Created: 2018-12-16T22:45:15+0100
# Last modified: 2019-03-09T00:59:43+0100
"""
Convert videos to webm files, using 2-pass constrained rate VP9
encoding for video and libvorbis for audio.
"""

from datetime import datetime, timedelta
import argparse
import logging
import math
import os
import re
import subprocess as sp
import sys

__version__ = '0.3'


def main(argv):
    """Entry point for vid2webm.py."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--log',
        default='info',
        choices=['debug', 'info', 'warning', 'error'],
        help="logging level (defaults to 'info')"
    )
    parser.add_argument('-v', '--version', action='version', version=__version__)
    parser.add_argument(
        '-s', '--start', type=str, default=None, help="time (hh:mm:ss) at which to start encoding"
    )
    parser.add_argument(
        '-d', '--dummy', action="store_true", help="print commands but do not run them"
    )
    parser.add_argument("files", metavar='files', nargs='+', help="one or more files to process")
    args = parser.parse_args(argv)
    logging.basicConfig(
        level=getattr(logging, args.log.upper(), None), format='%(levelname)s: %(message)s'
    )
    logging.debug(f'command line arguments = {argv}')
    logging.debug(f'parsed arguments = {args}')
    if not check_ffmpeg():
        return 1
    for fn in args.files:
        logging.info(f"processing '{fn}'.")
        t1, t2, t3 = expectedtime(fn)
        starttime = datetime.now()
        startstr = str(starttime)[:-7]
        tc = get_tc(fn)
        logging.info(f'started at {startstr}.')
        t1 = str(starttime+t1)[:-10]
        t2 = str(starttime+t2)[:-10]
        t3 = str(starttime+t3)[:-10]
        logging.info(f'encoding is expected to take until {t2} on average')
        logging.info(f'but it could be anywhere between {t1} and {t3}')
        a1 = mkargs(
            fn,
            1,
            tc,
            start=args.start,
        )
        a2 = mkargs(
            fn,
            2,
            tc,
            start=args.start,
        )
        if not args.dummy:
            origbytes, newbytes = encode(a1, a2)
        else:
            logging.basicConfig(level='INFO')
            logging.info('first pass: ' + ' '.join(a1))
            logging.info('second pass: ' + ' '.join(a2))
            return
        stoptime = datetime.now()
        stopstr = str(stoptime)[:-7]
        logging.info(f'ended at {stopstr}.')
        runtime = stoptime - starttime
        runstr = str(runtime)[:-7]
        logging.info(f'total running time {runstr}.')
        encspeed = origbytes / (runtime.seconds * 1000)<commit_msg>Print all commands when using dummy flag.<commit_after>#!/usr/bin/env python3
# file: vid2webm.py
# vim:fileencoding=utf-8:fdm=marker:ft=python
#
# Copyright © 2018 <NAME> <<EMAIL>>.
# SPDX-License-Identifier: MIT
# Created: 2018-12-16T22:45:15+0100
# Last modified: 2019-03-09T16:01:49+0100
"""
Convert videos to webm files, using 2-pass constrained rate VP9
encoding for video and libvorbis for audio.
"""

from datetime import datetime, timedelta
import argparse
import logging
import math
import os
import re
import subprocess as sp
import sys

__version__ = '0.3'


def main(argv):
    """Entry point for vid2webm.py."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--log',
        default='info',
        choices=['debug', 'info', 'warning', 'error'],
        help="logging level (defaults to 'info')"
    )
    parser.add_argument('-v', '--version', action='version', version=__version__)
    parser.add_argument(
        '-s', '--start', type=str, default=None, help="time (hh:mm:ss) at which to start encoding"
    )
    parser.add_argument(
        '-d', '--dummy', action="store_true", help="print commands but do not run them"
    )
    parser.add_argument("files", metavar='files', nargs='+', help="one or more files to process")
    args = parser.parse_args(argv)
    logging.basicConfig(
        level=getattr(logging, args.log.upper(), None), format='%(levelname)s: %(message)s'
    )
    logging.debug(f'command line arguments = {argv}')
    logging.debug(f'parsed arguments = {args}')
    if not check_ffmpeg():
        return 1
    for fn in args.files:
        logging.info(f"processing '{fn}'.")
        t1, t2, t3 = expectedtime(fn)
        starttime = datetime.now()
        startstr = str(starttime)[:-7]
        tc = get_tc(fn)
        logging.info(f'started at {startstr}.')
        t1 = str(starttime+t1)[:-10]
        t2 = str(starttime+t2)[:-10]
        t3 = str(starttime+t3)[:-10]
        logging.info(f'encoding is expected to take until {t2} on average')
        logging.info(f'but it could be anywhere between {t1} and {t3}')
        a1 = mkargs(
            fn,
            1,
            tc,
            start=args.start,
        )
        a2 = mkargs(
            fn,
            2,
            tc,
            start=args.start,
        )
        if not args.dummy:
            origbytes, newbytes = encode(a1, a2)
        else:
            logging.basicConfig(level='INFO')
            logging.info('first pass: ' + ' '.join(a1))
            logging.info('second pass: ' + ' '.join(a2))
            continue
        stoptime = datetime.now()
        stopstr = str(stoptime)[:-7]
        logging.info(f'ended at {stopstr}.')
        runtime = stoptime - starttime
        runstr = str(runtime)[:-7]
        logging.info(f'total running time {runstr}.')
        encspeed = origbytes / (runtime.seconds * 1000)
{'fraction_non_alphanumeric': 0.1430309350017379, 'fraction_numerical': 0.028501911713590546, 'mean_word_length': 2.487878787878788, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3159903', 'n_tokens_mistral': 1925, 'n_tokens_neox': 1845, 'n_words': 453}