<commit_before>from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    find_xpath_attr,
    int_or_none,
    js_to_json,
    unescapeHTML,
    determine_ext,
)


class HowStuffWorksIE(InfoExtractor):
    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
    _TESTS = [
        {
            'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
            'info_dict': {
                'id': '450221',
                'ext': 'flv',
                'title': 'Cool Jobs - Iditarod Musher',
                'description': 'Cold sleds, freezing temps and warm dog breath... an Iditarod musher\'s dream. <NAME> jumps on a sled to find out what the big deal is.',
                'display_id': 'cool-jobs-iditarod-musher',
                'thumbnail': r're:^https?://.*\.jpg$',
                'duration': 161,
            },
            'skip': 'Video broken',
        },
        {
            'url': 'http://adventure.howstuffworks.com/7199-survival-zone-food-and-water-in-the-savanna-video.htm',
            'info_dict': {
                'id': '453464',
                'ext': 'mp4',
                'title': 'Survival Zone: Food and Water In the Savanna',
                'description': 'Learn how to find both food and water while trekking in the African savannah. In this video from the Discovery Channel.',
                'display_id': 'survival-zone-food-and-water-in-the-savanna',
                'thumbnail': r're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
            'info_dict': {
                'id': '440011',
                'ext': 'mp4',
                'title': 'Sword Swallowing #1 by <NAME>',
                'description': 'Video footage (1 of 3) used by permission of the owner <NAME> through Sword Swallowers Association International <www.swordswallow.org>',
                'display_id': 'sword-swallowing-1-by-dan-meyer',
                'thumbnail': r're:^https?://.*\.jpg$',
            },
        },
        {
            'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        clip_js = self._search_regex(
            r'(?s)var clip = ({.*?});', webpage, 'clip info')
        clip_info = self._parse_json(
            clip_js, display_id, transform_source=js_to_json)<commit_msg>[howstuffworks] add support for more domains<commit_after>from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    find_xpath_attr,
    int_or_none,
    js_to_json,
    unescapeHTML,
    determine_ext,
)


class HowStuffWorksIE(InfoExtractor):
    _VALID_URL = r'https?://[\da-z-]+\.(?:howstuffworks|stuff(?:(?:youshould|theydontwantyouto)know|toblowyourmind|momnevertoldyou)|(?:brain|car)stuffshow|fwthinking|geniusstuff)\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
    _TESTS = [
        {
            'url': 'http://www.stufftoblowyourmind.com/videos/optical-illusions-video.htm',
            'md5': '76646a5acc0c92bf7cd66751ca5db94d',
            'info_dict': {
                'id': '855410',
                'ext': 'mp4',
                'title': 'Your Trickster Brain: Optical Illusions -- Science on the Web',
                'description': 'md5:e374ff9561f6833ad076a8cc0a5ab2fb',
            },
        },
        {
            'url': 'http://shows.howstuffworks.com/more-shows/why-does-balloon-stick-to-hair-video.htm',
            'only_matching': True,
        }
    ]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        clip_js = self._search_regex(
            r'(?s)var clip = ({.*?});', webpage, 'clip info')
        clip_info = self._parse_json(
            clip_js, display_id, transform_source=js_to_json)
{'fraction_non_alphanumeric': 0.15286470871449206, 'fraction_numerical': 0.020462205103514684, 'mean_word_length': 2.126410835214447, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6423441', 'n_tokens_mistral': 1479, 'n_tokens_neox': 1355, 'n_words': 240}