<commit_before># coding: utf-8
from __future__ import unicode_literals

import itertools
import re
import random
import json

from .common import InfoExtractor
from ..compat import (
    compat_kwargs,
    compat_parse_qs,
    compat_str,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    clean_html,
    ExtractorError,
    int_or_none,
    orderedSet,
    parse_duration,
    parse_iso8601,
    qualities,
    str_or_none,
    try_get,
    unified_timestamp,
    update_url_query,
    url_or_none,
    urljoin,
)


class TwitchBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:(?:www|go|m)\.)?twitch\.tv'

    _API_BASE = 'https://api.twitch.tv'
    _USHER_BASE = 'https://usher.ttvnw.net'
    _LOGIN_FORM_URL = 'https://www.twitch.tv/login'
    _LOGIN_POST_URL = 'https://passport.twitch.tv/login'
    _CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
    _NETRC_MACHINE = 'twitch'

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                expected=True)

    def _call_api(self, path, item_id, *args, **kwargs):
        headers = kwargs.get('headers', {}).copy()
        headers.update({
            'Accept': 'application/vnd.twitchtv.v5+json; charset=UTF-8',
            'Client-ID': self._CLIENT_ID,
        })
        kwargs['headers'] = headers
        response = self._download_json(
            '%s/%s' % (self._API_BASE, path), item_id,
            *args, **compat_kwargs(kwargs))
        self._handle_error(response)
        return response

    def _real_initialize(self):
        self._login()

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return

        def fail(message):<commit_msg>[twitch:stream] Expect 400 and 410 HTTP errors from API<commit_after># coding: utf-8
from __future__ import unicode_literals

import itertools
import re
import random
import json

from .common import InfoExtractor
from ..compat import (
    compat_kwargs,
    compat_parse_qs,
    compat_str,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    clean_html,
    ExtractorError,
    int_or_none,
    orderedSet,
    parse_duration,
    parse_iso8601,
    qualities,
    str_or_none,
    try_get,
    unified_timestamp,
    update_url_query,
    url_or_none,
    urljoin,
)


class TwitchBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:(?:www|go|m)\.)?twitch\.tv'

    _API_BASE = 'https://api.twitch.tv'
    _USHER_BASE = 'https://usher.ttvnw.net'
    _LOGIN_FORM_URL = 'https://www.twitch.tv/login'
    _LOGIN_POST_URL = 'https://passport.twitch.tv/login'
    _CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
    _NETRC_MACHINE = 'twitch'

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                expected=True)

    def _call_api(self, path, item_id, *args, **kwargs):
        headers = kwargs.get('headers', {}).copy()
        headers.update({
            'Accept': 'application/vnd.twitchtv.v5+json; charset=UTF-8',
            'Client-ID': self._CLIENT_ID,
        })
        kwargs.update({
            'headers': headers,
            'expected_status': (400, 410),
        })
        response = self._download_json(
            '%s/%s' % (self._API_BASE, path), item_id,
            *args, **compat_kwargs(kwargs))
        self._handle_error(response)
        return response

    def _real_initialize(self):
        self._login()

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return

        def fail(message):
{'fraction_non_alphanumeric': 0.11648568608094768, 'fraction_numerical': 0.010365251727541954, 'mean_word_length': 2.555263157894737, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3843456', 'n_tokens_mistral': 1375, 'n_tokens_neox': 1340, 'n_words': 261}