<commit_before># ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
from __future__ import absolute_import
import sys
import time

from svtplay_dl.output import progress, output # FIXME use progressbar() instead
from svtplay_dl.log import log
from svtplay_dl.utils.urllib import urlopen, Request, HTTPError
from svtplay_dl.fetcher import VideoRetriever

class HTTP(VideoRetriever):
    def name(self):
        return "http"

    def download(self):
        """ Get the stream from HTTP """
        request = Request(self.url)
        request.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
        try:
            response = urlopen(request)
        except HTTPError as e:
            log.error("Something wrong with that url")
            return
        try:
            total_size = response.info()['Content-Length']
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 0

        file_d = output(self.options, self.options.output, "mp4")
        if hasattr(file_d, "read") is False:
            return

        lastprogress = 0
        while 1:
            chunk = response.read(8192)
            bytes_so_far += len(chunk)

            if not chunk:
                break

            file_d.write(chunk)
            if self.options.output != "-":
                now = time.time()
                if lastprogress + 1 < now:
                    lastprogress = now
                    progress(bytes_so_far, total_size)

        if self.options.output != "-":
            file_d.close()

<commit_msg>http: unused import<commit_after># ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
from __future__ import absolute_import
import time

from svtplay_dl.output import progress, output # FIXME use progressbar() instead
from svtplay_dl.log import log
from svtplay_dl.utils.urllib import urlopen, Request, HTTPError
from svtplay_dl.fetcher import VideoRetriever

class HTTP(VideoRetriever):
    def name(self):
        return "http"

    def download(self):
        """ Get the stream from HTTP """
        request = Request(self.url)
        request.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
        try:
            response = urlopen(request)
        except HTTPError as e:
            log.error("Something wrong with that url")
            return
        try:
            total_size = response.info()['Content-Length']
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 0

        file_d = output(self.options, self.options.output, "mp4")
        if hasattr(file_d, "read") is False:
            return

        lastprogress = 0
        while 1:
            chunk = response.read(8192)
            bytes_so_far += len(chunk)

            if not chunk:
                break

            file_d.write(chunk)
            if self.options.output != "-":
                now = time.time()
                if lastprogress + 1 < now:
                    lastprogress = now
                    progress(bytes_so_far, total_size)

        if self.options.output != "-":
            file_d.close()


{'fraction_non_alphanumeric': 0.104047976011994, 'fraction_numerical': 0.02158920539730135, 'mean_word_length': 1.954827280779451, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '793143', 'n_tokens_mistral': 1047, 'n_tokens_neox': 997, 'n_words': 254}