<commit_before>#!/usr/bin/env python
# encoding: utf-8
# yyets-subdl.py - command-line downloader for YYeTs subtitles

import os
import sys
import shutil
import zipfile

# pip install requests
import requests
try:
    # pip install rarfile
    # brew install unrar
    import rarfile
except ImportError, e:
    rarfile = None


def query(keyword):
    request_url = 'http://www.yyets.com/php/search/api'
    request_params = dict(keyword=keyword)
    response = requests.get(request_url, params=request_params)
    if response.status_code == requests.codes.ok:
        items = response.json()['data']
        choices = [x for x in items if x['type'] == 'subtitle']
        return choices
    else:
        raise Exception('HTTP status_code=%s' % response.status_code)
    return []


def ask(choices):
    for idx, choice in enumerate(choices):
        print '%2d. %s' % (idx+1, choice['title'])
        print '    %s' % choice['version']
    answers = raw_input('What items do you want? (seperated by commas) [1] ')
    if answers: return map(lambda x: int(x)-1, answers.split(r','))
    else: return [0]


def download(items):
    files = []
    for item in items:
        # item_url = 'http://www.yyets.com/%s/%s' % (item['type'], item['itemid'])
        download_url = 'http://www.yyets.com/%s/index/download?id=%s' % (item['type'], item['itemid'])
        download_filename = '%s.zip' % (item['version'].split(';')[0] or item['title'])
        command_line = "curl -L '%s' > '%s'" % (download_url, download_filename)
        print command_line
        os.system(command_line.encode('utf-8'))
        files.append(download_filename)
    return files


def extract(files):
    for filepath in files:
        print 'Examining %s' % filepath
        extract_success = False
        archive = None
        try:
            # first we assume it's a Zip archive
            archive = zipfile.ZipFile(filepath)
        except zipfile.BadZipfile:
            # seems not a Zip, let's try RAR
            # https://rarfile.readthedocs.org/en/latest/api.html
            archive = rarfile.RarFile(filepath)
        if archive:
            for item in archive.namelist():
                filename = os.path.split(item)[1]
                extension = os.path.splitext(filename)[1].lower()

                try: filename = filename.decode('gb2312')
                except: pass

                try: filename = filename.encode('utf-8')
                except: pass

                found = False
                target_lang = u'繁体&英文'.encode('utf-8')
                if ('ass' in extension) and (target_lang in filename):
                    # zipfile.extract() makes filename escaped with '%xx'
                    with archive.open(item, 'r') as infile, open(filename, 'wb') as outfile:
                        shutil.copyfileobj(infile, outfile)
                        extract_success = True
                    print 'Extracted %s' % filename
            archive.close()
        if extract_success:
            os.remove(filepath)


if __name__ == '__main__':
    if len(sys.argv) < 2:
        print 'Usage: %s <keyword>' % sys.argv[0]
        print "Example: %s 'Person of Interest S04E01'" % sys.argv[0]
        sys.exit(1)
    keyword = sys.argv[1]
    choices = query(keyword)
    chosen_ids = ask(choices)
    chosens = [choices[idx] for idx in chosen_ids]
    files = download(chosens)
    extract(files)
<commit_msg>fix: error handling in extract()<commit_after>#!/usr/bin/env python
# encoding: utf-8
# yyets-subdl.py - command-line downloader for YYeTs subtitles

import os
import sys
import shutil
import zipfile

# pip install requests
import requests
try:
    # pip install rarfile
    # brew install unrar
    import rarfile
except ImportError, e:
    rarfile = None


def query(keyword):
    request_url = 'http://www.yyets.com/php/search/api'
    request_params = dict(keyword=keyword)
    response = requests.get(request_url, params=request_params)
    if response.status_code == requests.codes.ok:
        items = response.json()['data']
        choices = [x for x in items if x['type'] == 'subtitle']
        return choices
    else:
        raise Exception('HTTP status_code=%s' % response.status_code)
    return []


def ask(choices):
    for idx, choice in enumerate(choices):
        print '%2d. %s' % (idx+1, choice['title'])
        print '    %s' % choice['version']
    answers = raw_input('What items do you want? (seperated by commas) [1] ')
    if answers: return map(lambda x: int(x)-1, answers.split(r','))
    else: return [0]


def download(items):
    files = []
    for item in items:
        # item_url = 'http://www.yyets.com/%s/%s' % (item['type'], item['itemid'])
        download_url = 'http://www.yyets.com/%s/index/download?id=%s' % (item['type'], item['itemid'])
        download_filename = '%s.zip' % (item['version'].split(';')[0] or item['title'])
        command_line = "curl -L '%s' > '%s'" % (download_url, download_filename)
        print command_line
        os.system(command_line.encode('utf-8'))
        files.append(download_filename)
    return files


def extract(files):
    for filepath in files:
        print 'Examining %s' % filepath
        extract_success = False
        archive = None
        try:
            # first we assume it's a Zip archive
            archive = zipfile.ZipFile(filepath)
        except zipfile.BadZipfile:
            if rarfile:
                # seems not a Zip, let's try RAR
                # https://rarfile.readthedocs.org/en/latest/api.html
                archive = rarfile.RarFile(filepath)
        if archive:
            for item in archive.namelist():
                filename = os.path.split(item)[1]
                extension = os.path.splitext(filename)[1].lower()

                try: filename = filename.decode('gb2312')
                except: pass

                try: filename = filename.encode('utf-8')
                except: pass

                found = False
                target_lang = u'繁体&英文'.encode('utf-8')
                if ('ass' in extension) and (target_lang in filename):
                    # zipfile.extract() makes filename escaped with '%xx'
                    with archive.open(item, 'r') as infile, open(filename, 'wb') as outfile:
                        shutil.copyfileobj(infile, outfile)
                        extract_success = True
                    print 'Extracted %s' % filename
            archive.close()
        else:
            print 'Cannot unpack %s' % filename
        if extract_success:
            os.remove(filepath)


if __name__ == '__main__':
    if len(sys.argv) < 2:
        print 'Usage: %s <keyword>' % sys.argv[0]
        print "Example: %s 'Person of Interest S04E01'" % sys.argv[0]
        sys.exit(1)
    keyword = sys.argv[1]
    choices = query(keyword)
    chosen_ids = ask(choices)
    chosens = [choices[idx] for idx in chosen_ids]
    files = download(chosens)
    extract(files)

{'fraction_non_alphanumeric': 0.1252336448598131, 'fraction_numerical': 0.007189072609633357, 'mean_word_length': 2.2203703703703703, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1403803', 'n_tokens_mistral': 2161, 'n_tokens_neox': 2088, 'n_words': 545}