<commit_before># ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
from __future__ import absolute_import

from svtplay_dl.utils.output import ETA, progressbar, output
from svtplay_dl.fetcher import VideoRetriever


class HTTP(VideoRetriever):
    @property
    def name(self):
        return "http"

    def download(self):
        """ Get the stream from HTTP """
        self.output_extention = "mp4"  # this might be wrong..
        data = self.http.request("get", self.url, stream=True)
        try:
            total_size = data.headers['content-length']
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 0

        file_d = output(self.output, self.config, "mp4")
        if file_d is None:
            return

        eta = ETA(total_size)
        for i in data.iter_content(8192):
            bytes_so_far += len(i)
            file_d.write(i)<commit_msg>fetcher.http: if we can detect its mp3, use that as extension.<commit_after># ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
from __future__ import absolute_import
import os

from svtplay_dl.utils.output import ETA, progressbar, output
from svtplay_dl.fetcher import VideoRetriever


class HTTP(VideoRetriever):
    @property
    def name(self):
        return "http"

    def download(self):
        """ Get the stream from HTTP """
        _, ext = os.path.splitext(self.url)
        if ext == ".mp3":
            self.output_extention = "mp3"
        else:
            self.output_extention = "mp4"  # this might be wrong..
        data = self.http.request("get", self.url, stream=True)
        try:
            total_size = data.headers['content-length']
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 0

        file_d = output(self.output, self.config, self.output_extention)
        if file_d is None:
            return

        eta = ETA(total_size)
        for i in data.iter_content(8192):
            bytes_so_far += len(i)
            file_d.write(i)
{'fraction_non_alphanumeric': 0.11027686532144533, 'fraction_numerical': 0.01313937118723604, 'mean_word_length': 2.285053929121726, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4914012', 'n_tokens_mistral': 702, 'n_tokens_neox': 678, 'n_words': 169}