<commit_before>import json
from pathlib import Path
from subprocess import run

dataset = Path('~/tthl-dataset/').expanduser()
video_dirs = sorted(dataset.glob('video*/'))
for i, video_dir in enumerate(video_dirs):
    print('{} ({} / {})'.format(video_dir, i, len(video_dirs)))
    url = json.load((video_dir / 'info.json').open())['video_src']
    run(['youtube-dl', '--no-playlist', '-f', '18', '-o', str(video_dir / 'video.mp4'), url])
    print('-' * 50)<commit_msg>Update at 2017-07-20 16-44-03<commit_after>import json
from pathlib import Path
from subprocess import run

dataset = Path('~/tthl-dataset/').expanduser()
video_dirs = sorted(dataset.glob('video*/'))

# run(['rm', '-rf', '{}/**/*.mp4'.format(dataset)])

for i, video_dir in enumerate(video_dirs):
    print('{} ({} / {})'.format(video_dir, i, len(video_dirs)))
    url = json.load((video_dir / 'info.json').open())['video_src']
    run(['youtube-dl', '--no-playlist', '-f', '18', '-o', str(video_dir / 'video.mp4'), url])
    print('-' * 60)
{'fraction_non_alphanumeric': 0.2341897233201581, 'fraction_numerical': 0.024703557312252964, 'mean_word_length': 4.596685082872928, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5340554', 'n_tokens_mistral': 386, 'n_tokens_neox': 364, 'n_words': 77}