<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ..common import *
from ..extractor import VideoExtractor

import time
import traceback
import json
import urllib.request
import urllib.parse


def fetch_cna():

    def quote_cna(val):
        if '%' in val:
            return val
        return urllib.parse.quote(val)

    if cookies:
        for cookie in cookies:
            if cookie.name == 'cna' and cookie.domain == '.youku.com':
                log.i('Found cna in imported cookies. Use it')
                return quote_cna(cookie.value)
    url = 'http://log.mmstat.com/eg.js'
    req = urllib.request.urlopen(url)
    headers = req.getheaders()
    for header in headers:
        if header[0].lower() == 'set-cookie':
            n_v = header[1].split(';')[0]
            name, value = n_v.split('=')
            if name == 'cna':
                return quote_cna(value)
    log.w('It seems that the client failed to fetch a cna cookie. Please load your own cookie if possible')
    return quote_cna('DOG4EdW4qzsCAbZyXbU+t7Jt')


class Youku(VideoExtractor):
    name = "优酷 (Youku)"
    mobile_ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
    dispatcher_url = 'vali.cp31.ott.cibntv.net'

    # Last updated: 2017-10-13
    stream_types = [
        {'id': 'hd3',      'container': 'flv', 'video_profile': '1080P'},
        {'id': 'hd3v2',    'container': 'flv', 'video_profile': '1080P'},
        {'id': 'mp4hd3',   'container': 'mp4', 'video_profile': '1080P'},
        {'id': 'mp4hd3v2', 'container': 'mp4', 'video_profile': '1080P'},

        {'id': 'hd2',      'container': 'flv', 'video_profile': '超清'},
        {'id': 'hd2v2',    'container': 'flv', 'video_profile': '超清'},
        {'id': 'mp4hd2',   'container': 'mp4', 'video_profile': '超清'},
        {'id': 'mp4hd2v2', 'container': 'mp4', 'video_profile': '超清'},

        {'id': 'mp4hd',    'container': 'mp4', 'video_profile': '高清'},
        # not really equivalent to mp4hd
        {'id': 'flvhd',    'container': 'flv', 'video_profile': '渣清'},
        {'id': '3gphd',    'container': 'mp4', 'video_profile': '渣清'},

        {'id': 'mp4sd',    'container': 'mp4', 'video_profile': '标清'},
        # obsolete?
        {'id': 'flv',      'container': 'flv', 'video_profile': '标清'},
        {'id': 'mp4',      'container': 'mp4', 'video_profile': '标清'},
    ]

    def __init__(self):
        super().__init__()

        self.ua = self.__class__.mobile_ua
        self.referer = 'http://v.youku.com'

        self.page = None
        self.video_list = None
        self.video_next = None
        self.password = <PASSWORD>
        self.api_data = None
        self.api_error_code = None
        self.api_error_msg = None

        self.ccode = '0513'
        self.utid = None

    def youku_ups(self):
        url = 'https://ups.youku.com/ups/get.json?vid={}&ccode={}'.format(self.vid, self.ccode)
        url += '&client_ip=192.168.1.1'
        url += '&utid=' + self.utid
        url += '&client_ts=' + str(int(time.time()))
        if self.password_protected:
            url += '&password=' + self.password
        headers = dict(Referer=self.referer)
        headers['User-Agent'] = self.ua
        api_meta = json.loads(get_content(url, headers=headers))

        self.api_data = api_meta['data']<commit_msg>Use 0513 when cookies are used<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ..common import *
from ..extractor import VideoExtractor

import time
import traceback
import json
import urllib.request
import urllib.parse


def fetch_cna():

    def quote_cna(val):
        if '%' in val:
            return val
        return urllib.parse.quote(val)

    if cookies:
        for cookie in cookies:
            if cookie.name == 'cna' and cookie.domain == '.youku.com':
                log.i('Found cna in imported cookies. Use it')
                return quote_cna(cookie.value)
    url = 'http://log.mmstat.com/eg.js'
    req = urllib.request.urlopen(url)
    headers = req.getheaders()
    for header in headers:
        if header[0].lower() == 'set-cookie':
            n_v = header[1].split(';')[0]
            name, value = n_v.split('=')
            if name == 'cna':
                return quote_cna(value)
    log.w('It seems that the client failed to fetch a cna cookie. Please load your own cookie if possible')
    return quote_cna('DOG4EdW4qzsCAbZyXbU+t7Jt')


class Youku(VideoExtractor):
    name = "优酷 (Youku)"
    mobile_ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
    dispatcher_url = 'vali.cp31.ott.cibntv.net'

    # Last updated: 2017-10-13
    stream_types = [
        {'id': 'hd3',      'container': 'flv', 'video_profile': '1080P'},
        {'id': 'hd3v2',    'container': 'flv', 'video_profile': '1080P'},
        {'id': 'mp4hd3',   'container': 'mp4', 'video_profile': '1080P'},
        {'id': 'mp4hd3v2', 'container': 'mp4', 'video_profile': '1080P'},

        {'id': 'hd2',      'container': 'flv', 'video_profile': '超清'},
        {'id': 'hd2v2',    'container': 'flv', 'video_profile': '超清'},
        {'id': 'mp4hd2',   'container': 'mp4', 'video_profile': '超清'},
        {'id': 'mp4hd2v2', 'container': 'mp4', 'video_profile': '超清'},

        {'id': 'mp4hd',    'container': 'mp4', 'video_profile': '高清'},
        # not really equivalent to mp4hd
        {'id': 'flvhd',    'container': 'flv', 'video_profile': '渣清'},
        {'id': '3gphd',    'container': 'mp4', 'video_profile': '渣清'},

        {'id': 'mp4sd',    'container': 'mp4', 'video_profile': '标清'},
        # obsolete?
        {'id': 'flv',      'container': 'flv', 'video_profile': '标清'},
        {'id': 'mp4',      'container': 'mp4', 'video_profile': '标清'},
    ]

    def __init__(self):
        super().__init__()

        self.ua = self.__class__.mobile_ua
        self.referer = 'http://v.youku.com'

        self.page = None
        self.video_list = None
        self.video_next = None
        self.password = <PASSWORD>
        self.api_data = None
        self.api_error_code = None
        self.api_error_msg = None

        if cookies:
            self.ccode = '0513'
        else:
            self.ccode = '0507'
        self.utid = None

    def youku_ups(self):
        url = 'https://ups.youku.com/ups/get.json?vid={}&ccode={}'.format(self.vid, self.ccode)
        url += '&client_ip=192.168.1.1'
        url += '&utid=' + self.utid
        url += '&client_ts=' + str(int(time.time()))
        if self.password_protected:
            url += '&password=' + self.password
        headers = dict(Referer=self.referer)
        headers['User-Agent'] = self.ua
        api_meta = json.loads(get_content(url, headers=headers))

        self.api_data = api_meta['data']
{'fraction_non_alphanumeric': 0.17364657814096016, 'fraction_numerical': 0.030643513789581207, 'mean_word_length': 2.4390366281986955, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '904831', 'n_tokens_mistral': 2518, 'n_tokens_neox': 2424, 'n_words': 535}