<commit_before>'''
Copyright 2015 ohyou

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

import requests
import subprocess
import json
import sys
import multiprocessing
import time
import random

channel_url = "twitch.tv/"
processes = []


def get_channel():
    # Reading the channel name - passed as an argument to this script
    if len(sys.argv) >= 2:
        global channel_url
        channel_url += sys.argv[1]
    else:
        print "An error has occurred while trying to read arguments."
        sys.exit(1)


def get_proxies():
    # Reading the list of proxies
    try:
        lines = [line.rstrip("\n") for line in open("proxylist.txt")]
    except IOError as e:
        print "An error has occurred while trying to read the list of proxies: %s" % e.strerror
        sys.exit(1)

    return lines


def get_url():
    # Getting the json with all data regarding the stream
    try:
        response = subprocess.Popen(
            ["livestreamer", channel_url, "-j"], stdout=subprocess.PIPE).communicate()[0]
    except subprocess.CalledProcessError as e:
        print "An error has occurred while trying to get the stream data."
        sys.exit(1)

    # Decoding the url to the worst quality of the stream
    try:
        url = json.loads(response)['streams']['worst']['url']
    except (ValueError, KeyError):
        print "An error has occurred while trying to process the response."
        sys.exit(1)

    return url


def open_url(url, proxy):
    # Explicitly opening the session
    session = requests.Session()

    # Sending HEAD requests
    while True:
        try:
            response = session.head(url, proxies=proxy)
            print "Sent HEAD request with %s" % proxy["http"]
            time.sleep(20)
            response.connection.close()
        except requests.exceptions.Timeout:
            print "  Timeout error for %s" % proxy["http"]
        except requests.exceptions.ConnectionError:
            print "  Connection error for %s" % proxy["http"]


def prepare_processes():
    global processes
    proxies = get_proxies()

    if len(proxies) < 1:
        print "An error has occurred while preparing the process: Not enough proxy servers."
        sys.exit(1)

    for proxy in proxies:
        # Preparing the process and giving it its own proxy
        processes.append(<commit_msg>Tweak session management<commit_after>'''
Copyright 2015 ohyou

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

import requests
import subprocess
import json
import sys
import multiprocessing
import time
import random

channel_url = "twitch.tv/"
processes = []


def get_channel():
    # Reading the channel name - passed as an argument to this script
    if len(sys.argv) >= 2:
        global channel_url
        channel_url += sys.argv[1]
    else:
        print "An error has occurred while trying to read arguments."
        sys.exit(1)


def get_proxies():
    # Reading the list of proxies
    try:
        lines = [line.rstrip("\n") for line in open("proxylist.txt")]
    except IOError as e:
        print "An error has occurred while trying to read the list of proxies: %s" % e.strerror
        sys.exit(1)

    return lines


def get_url():
    # Getting the json with all data regarding the stream
    try:
        response = subprocess.Popen(
            ["livestreamer", channel_url, "-j"], stdout=subprocess.PIPE).communicate()[0]
    except subprocess.CalledProcessError as e:
        print "An error has occurred while trying to get the stream data."
        sys.exit(1)

    # Decoding the url to the worst quality of the stream
    try:
        url = json.loads(response)['streams']['worst']['url']
    except (ValueError, KeyError):
        print "An error has occurred while trying to process the response."
        sys.exit(1)

    return url


def open_url(url, proxy):
    # Sending HEAD requests
    while True:
        try:
            with requests.Session() as s:
                response = s.head(url, proxies=proxy)
            print "Sent HEAD request with %s" % proxy["http"]
            time.sleep(20)
        except requests.exceptions.Timeout:
            print "  Timeout error for %s" % proxy["http"]
        except requests.exceptions.ConnectionError:
            print "  Connection error for %s" % proxy["http"]


def prepare_processes():
    global processes
    proxies = get_proxies()

    if len(proxies) < 1:
        print "An error has occurred while preparing the process: Not enough proxy servers."
        sys.exit(1)

    for proxy in proxies:
        # Preparing the process and giving it its own proxy
        processes.append(
{'fraction_non_alphanumeric': 0.08192900681247758, 'fraction_numerical': 0.006812477590534241, 'mean_word_length': 2.7772511848341233, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '573555', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1505, 'n_words': 622}