<commit_before>#!/usr/bin/env python
from __future__ import print_function
import errno
import json
import os
import re
import shutil
import string
import sys

import requests


CONFIG_FILE = 'scripts.conf'
EXTENSIONS = ['avi', 'm4v', 'mkv', 'mp4']
SUB_EXTENSIONS = ['idx', 'sub', 'srt']
PATTERN = re.compile('^(.*)(\d{4})\.(.*)', re.I)
REPLACEMENTS = {
    ':': '',
}
ENDPOINT = 'https://api.themoviedb.org/3/search/movie'


def get_config(cfgfile):
    if not cfgfile.startswith('/'):
        cfgfile = os.path.join(os.path.dirname(sys.argv[0]), '..', cfgfile)

    config = {}
    with open(cfgfile) as fh:
        for line in fh:
            k, v = re.split('\s+=\s+', line, 1)
            config[k] = v.strip()
    return config


CONFIG = get_config(CONFIG_FILE)


def get_matching_files(dirname):
    # Loop through the files in ``job_dir``, trying to find one which
    # matches PATTERN. Also build a list of files and their sizes.
    #
    # If no files match PATTERN but we do find one or more files with
    # a suitable extension we assume the largest file is the one we
    # want. We take this file and use the ``job_dir`` as the base for
    # the new file name.
    files = {
        'match': (),
        'by_size': [],
        'subs': [],
    }
    for root, _, fns in os.walk(dirname):
        for fn in fns:
            ext = fn.rsplit('.', 1)[-1].lower()
            if ext in SUB_EXTENSIONS:
                files['subs'].append(fn)

            if ext not in EXTENSIONS:
                continue

            abs = os.path.abspath(os.path.join(root, fn))
            files['by_size'].append((fn, os.path.getsize(abs)))
            match = PATTERN.match(fn)<commit_msg>Fix bug where subtitles are ignored if they're not at the root of the job directory.<commit_after>#!/usr/bin/env python
from __future__ import print_function
import errno
import json
import os
import re
import shutil
import string
import sys

import requests


CONFIG_FILE = 'scripts.conf'
EXTENSIONS = ['avi', 'm4v', 'mkv', 'mp4']
SUB_EXTENSIONS = ['idx', 'sub', 'srt']
PATTERN = re.compile('^(.*)(\d{4})\.(.*)', re.I)
REPLACEMENTS = {
    ':': '',
}
ENDPOINT = 'https://api.themoviedb.org/3/search/movie'


def get_config(cfgfile):
    if not cfgfile.startswith('/'):
        cfgfile = os.path.join(os.path.dirname(sys.argv[0]), '..', cfgfile)

    config = {}
    with open(cfgfile) as fh:
        for line in fh:
            k, v = re.split('\s+=\s+', line, 1)
            config[k] = v.strip()
    return config


CONFIG = get_config(CONFIG_FILE)


def get_matching_files(dirname):
    # Loop through the files in ``dirname``, trying to find one which
    # matches PATTERN. Also build a list of files and their sizes.
    #
    # If no files match PATTERN but we do find one or more files with
    # a suitable extension we assume the largest file is the one we
    # want. We take this file and use the ``job_dir`` as the base for
    # the new file name.
    files = {
        'match': (),
        'by_size': [],
        'subs': [],
    }
    for root, _, fns in os.walk(dirname):
        for fn in fns:
            ext = fn.rsplit('.', 1)[-1].lower()
            if ext in SUB_EXTENSIONS:
                files['subs'].append(os.path.join(root, fn))

            if ext not in EXTENSIONS:
                continue

            abs = os.path.abspath(os.path.join(root, fn))
            files['by_size'].append((fn, os.path.getsize(abs)))
            match = PATTERN.match(fn)
{'fraction_non_alphanumeric': 0.14129496402877698, 'fraction_numerical': 0.00460431654676259, 'mean_word_length': 2.421259842519685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1759332', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1133, 'n_words': 351}