<commit_before>#!/usr/bin/env python

__all__ = ['tudou_download', 'tudou_download_playlist', 'tudou_download_by_id', 'tudou_download_by_iid']

from ..common import *
from xml.dom.minidom import parseString
import you_get.extractors.acfun

def tudou_download_by_iid(iid, title, output_dir = '.', merge = True, info_only = False):
    data = json.loads(get_decoded_html('http://www.tudou.com/outplay/goto/getItemSegs.action?iid=%s' % iid))
    temp = max([data[i] for i in data if 'size' in data[i][0]], key=lambda x:sum([part['size'] for part in x]))
    vids, size = [t["k"] for t in temp], sum([t["size"] for t in temp])

    urls = []
    for vid in vids:
        for i in parseString(get_html('http://ct.v2.tudou.com/f?id=%s' % vid)).getElementsByTagName('f'):
            urls.append(i.firstChild.nodeValue.strip())

    ext = r1(r'http://[\w.]*/(\w+)/[\w.]*', urls[0])

    print_info(site_info, title, ext, size)
    if not info_only:
        download_urls(urls, title, ext, size, output_dir=output_dir, merge = merge)

def tudou_download_by_id(id, title, output_dir = '.', merge = True, info_only = False):
    html = get_html('http://www.tudou.com/programs/view/%s/' % id)

    iid = r1(r'iid\s*[:=]\s*(\S+)', html)
    title = r1(r'kw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
    tudou_download_by_iid(iid, title, output_dir = output_dir, merge = merge, info_only = info_only)

def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
    if 'acfun.tudou.com' in url:  #wrong way!
        url = url.replace('acfun.tudou.com', 'www.acfun.tv')
        you_get.extractors.acfun.acfun_download(url, output_dir, 
                                               merge, 
                                               info_only)
        return  #throw you back
    
    # Embedded player
    id = r1(r'http://www.tudou.com/v/([^/]+)/', url)
    if id:
        return tudou_download_by_id(id, title="", info_only=info_only)

    html = get_decoded_html(url)

    title = r1(r'kw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
    assert title
    title = unescape_html(title)

    vcode = r1(r'vcode\s*[:=]\s*\'([^\']+)\'', html)
    if vcode:
        from .youku import youku_download_by_vid
        return youku_download_by_vid(vcode, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
<commit_msg>[tudou] fix #1526<commit_after>#!/usr/bin/env python

__all__ = ['tudou_download', 'tudou_download_playlist', 'tudou_download_by_id', 'tudou_download_by_iid']

from ..common import *
from xml.dom.minidom import parseString
import you_get.extractors.acfun

def tudou_download_by_iid(iid, title, output_dir = '.', merge = True, info_only = False):
    data = json.loads(get_decoded_html('http://www.tudou.com/outplay/goto/getItemSegs.action?iid=%s' % iid))
    temp = max([data[i] for i in data if 'size' in data[i][0]], key=lambda x:sum([part['size'] for part in x]))
    vids, size = [t["k"] for t in temp], sum([t["size"] for t in temp])

    urls = []
    for vid in vids:
        for i in parseString(get_html('http://ct.v2.tudou.com/f?id=%s' % vid)).getElementsByTagName('f'):
            urls.append(i.firstChild.nodeValue.strip())

    ext = r1(r'http://[\w.]*/(\w+)/[\w.]*', urls[0])

    print_info(site_info, title, ext, size)
    if not info_only:
        download_urls(urls, title, ext, size, output_dir=output_dir, merge = merge)

def tudou_download_by_id(id, title, output_dir = '.', merge = True, info_only = False):
    html = get_html('http://www.tudou.com/programs/view/%s/' % id)

    iid = r1(r'iid\s*[:=]\s*(\S+)', html)
    title = r1(r'kw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
    tudou_download_by_iid(iid, title, output_dir = output_dir, merge = merge, info_only = info_only)

def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
    if 'acfun.tudou.com' in url:  #wrong way!
        url = url.replace('acfun.tudou.com', 'www.acfun.tv')
        you_get.extractors.acfun.acfun_download(url, output_dir,
                                               merge,
                                               info_only)
        return  #throw you back

    # Embedded player
    id = r1(r'http://www.tudou.com/v/([^/]+)/', url)
    if id:
        return tudou_download_by_id(id, title="", info_only=info_only)

    html = get_decoded_html(url)

    title = r1(r'\Wkw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
    assert title
    title = unescape_html(title)

    vcode = r1(r'vcode\s*[:=]\s*\'([^\']+)\'', html)
    if vcode:
        from .youku import youku_download_by_vid
        return youku_download_by_vid(vcode, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)

{'fraction_non_alphanumeric': 0.1893021312160468, 'fraction_numerical': 0.00459674049310489, 'mean_word_length': 3.0844709897610922, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4905347', 'n_tokens_mistral': 1843, 'n_tokens_neox': 1832, 'n_words': 340}