<commit_before># !/usr/bin/env python
# coding: utf-8
"""
arabic-toons-downloader - API-less arabic-toons movies and series downloader

Usage:
    downloader.py movie <movie_url> [<directory>]
    downloader.py episode <episode_url> [<directory>]
    downloader.py series <series_url> [<directory>] [options]
    downloader.py (-h | --help | --version | --usage)

Options:
    -e EPISODES, --episodes EPISODES        Maximum number of new files
                                            to download EPISODES must be
                                            in format "2 5 3-7"
    -h, --help                              Display this message and quit
    --version                               Show program version and quit
"""
import concurrent.futures
import os
import re
import subprocess
import sys

import bs4
import docopt
import m3u8
import requests

__author__ = "mohan3d"
__author_email__ = "<EMAIL>"
__version__ = "0.1.3"

ARABIC_TOONS_HOST = 'www.arabic-toons.com'
ARABIC_TOONS_USER_AGENT = {
    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/90.0.4430.93 Safari/537.36 '
}


class PageParser:
    def __init__(self):
        self.session = requests.session()
        self.user_agent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) " \
                          "Gecko/20100101 Firefox/50.0"

        self.session.headers.update({
            **ARABIC_TOONS_USER_AGENT,
            'Accept': 'text/html',
            'Accept-Encoding': 'gzip, deflate',
            'Connection': 'keep-alive',
            'Host': ARABIC_TOONS_HOST,
        })

    def _html(self, url):
        return self.session.get(url).content.decode('utf-8')

    def _parse(self, html):
        raise NotImplementedError()


class VideoParser(PageParser):
    _STREAM_URL = re.compile(r'<source src="(http://[^"]+)" type="application/x-mpegURL">')
    _STREAM_TITLE = re.compile(r'<h1 style="padding:15px">([^<]+)</h1>')

    def _parse(self, html):
        return self._extract(html, self._STREAM_URL)

    def _parse_title(self, html):
        return self._extract(html, self._STREAM_TITLE)

    def get_stream_info(self, url):
        html = self._html(url)
        url = self._parse(html)
        title = self._parse_title(html)
        title = ' '.join(title.split(' ')[1:]).strip()

        return url, title

    @staticmethod
    def _extract(html: str, regex: re.Pattern):
        return requests.utils.unquote(regex.search(html).group(1))


class SeriesParser(PageParser):
    def _parse(self, html):
        soup = bs4.BeautifulSoup(html, 'html.parser')
        series_list = soup.findAll('div', attrs={'class': 'col-sm-4 col-xs-6 col-md-2 col-lg-2'})

        return ['http://{}/{}'.format(ARABIC_TOONS_HOST, div.find('a')['href']) for div in series_list]

    def get_episodes_urls(self, url):
        html = self._html(url)
        return self._parse(html)


class StreamDownloader:
    def __init__(self, stream_url: str, workers: int = 1, ffmpeg: bool = False):
        self.url = stream_url
        self.workers = workers
        self.ffmpeg = ffmpeg

        self.session = requests.session()
        self.session.headers.update(ARABIC_TOONS_USER_AGENT)

    def download(self, filepath: str):
        segments = self._get_segments()

        if self.workers and self.workers > 1:
            with concurrent.futures.ThreadPoolExecutor(max_workers=self.workers) as executor:
                responses = executor.map(self.session.get, segments)
        else:
            responses = [self.session.get(segment_url) for segment_url in segments]

        with open(filepath, 'wb') as f:
            for response in responses:
                f.write(response.content)

        if self.ffmpeg:
            self._ffmpeg(filepath)

    def _get_segments(self):
        return m3u8.load(self.url, headers=self.session.headers).segments.uri

    @staticmethod
    def _ffmpeg(path: str, remove_input=True):
        output_path = path.replace('.ts', '.mp4')

        subprocess.run(["ffmpeg", "-i", path, "-acodec", "copy", "-vcodec", "copy", output_path],
                       stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, check=True)

        if remove_input:
            os.remove(path)


class ATDownloader:
    def __init__(self, directory):
        self.directory = directory

        if self.directory is not None and not os.path.exists(self.directory):
            os.makedirs(self.directory)

    def _download_video(self, url):
        stream_url, stream_title = VideoParser().get_stream_info(url)
        filepath = os.path.join(self.directory, self._get_file_name(stream_title))

        for path in [filepath, filepath.replace('.ts', '.mp4')]:
            if os.path.exists(path):
                print(f'{path} already exists!')
                return

        StreamDownloader(stream_url, workers=16, ffmpeg=True).download(filepath)

    def download_movie(self, movie_url):
        self._download_video(movie_url)

    def download_series(self, series_url, specific_episodes=None):
        episodes_urls = SeriesParser().get_episodes_urls(series_url)

        if specific_episodes:
            parsed_episodes = list(self._parse_episodes(specific_episodes))
            episodes_urls = [url for i, url in enumerate(episodes_urls, 1)
                             if i in parsed_episodes]

        for episode_url in episodes_urls:
            self._download_video(episode_url)

    @staticmethod
    def _parse_episodes(episodes_str):
        for episodes_range in episodes_str.split():
            p = episodes_range.split('-')

            start = end = p[0]
            end = p[1] if len(p) == 2 else end

            for episode in range(int(start), int(end) + 1):
                yield episode

    @staticmethod
    def _get_file_name(name, ext='ts'):
        return f'{name}.{ext}'


def main():
    args = docopt.docopt(__doc__, argv=sys.argv[1:], version=__version__)

    downloader = ATDownloader(
        directory=os.path.expanduser(args.get('<directory>') or os.getcwd()))

    try:
        if args.get('movie'):
            downloader.download_movie(args.get('<movie_url>'))
        elif args.get('episode'):
            downloader.download_movie(args.get('<episode_url>'))
        elif args.get('series'):
            downloader.download_series(args.get('<series_url>'),
                                       args.get('--episodes'))
    except KeyboardInterrupt:
        pass


if __name__ == "__main__":
    main()
<commit_msg>feat: add option to pass segments/processes to be used for download<commit_after># !/usr/bin/env python
# coding: utf-8
"""
arabic-toons-downloader - API-less arabic-toons movies and series downloader

Usage:
    downloader.py movie <movie_url> [<directory>]
    downloader.py episode <episode_url> [<directory>]
    downloader.py series <series_url> [<directory>] [options]
    downloader.py (-h | --help | --version | --usage)

Options:
    -e EPISODES, --episodes EPISODES        Maximum number of new files
                                            to download EPISODES must be
                                            in format "2 5 3-7"
    -s SEGMENTS, --segments SEGMENTS        Number of segments to be downloaded in parallel,
                                            for faster network and cpu set to 16 or 32
    -p PROCESSES, --processes PROCESSES     Maximum number of processes to be used to download series
                                            for faster network and cpu set to 4 or 8
                                            (n episodes will be downloaded in parallel)
    -h, --help                              Display this message and quit
    --version                               Show program version and quit
"""
import concurrent.futures
import os
import re
import subprocess
import sys

import bs4
import docopt
import m3u8
import requests

__author__ = "mohan3d"
__author_email__ = "<EMAIL>"
__version__ = "0.1.3"

ARABIC_TOONS_HOST = 'www.arabic-toons.com'
ARABIC_TOONS_USER_AGENT = {
    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/90.0.4430.93 Safari/537.36 '
}

DEFAULT_SIMULTANEOUS_SEGMENTS_COUNT = 4
DEFAULT_PROCESSES_COUNT = 1


class PageParser:
    def __init__(self):
        self.session = requests.session()

        self.session.headers.update({
            **ARABIC_TOONS_USER_AGENT,
            'Accept': 'text/html',
            'Accept-Encoding': 'gzip, deflate',
            'Connection': 'keep-alive',
            'Host': ARABIC_TOONS_HOST,
        })

    def _html(self, url):
        return self.session.get(url).content.decode('utf-8')

    def _parse(self, html):
        raise NotImplementedError()


class VideoParser(PageParser):
    _STREAM_URL = re.compile(r'<source src="(http://[^"]+)" type="application/x-mpegURL">')
    _STREAM_TITLE = re.compile(r'<h1 style="padding:15px">([^<]+)</h1>')

    def _parse(self, html):
        return self._extract(html, self._STREAM_URL)

    def _parse_title(self, html):
        return self._extract(html, self._STREAM_TITLE)

    def get_stream_info(self, url):
        html = self._html(url)
        url = self._parse(html)
        title = self._parse_title(html)
        title = ' '.join(title.split(' ')[1:]).strip()

        return url, title

    @staticmethod
    def _extract(html: str, regex: re.Pattern):
        return requests.utils.unquote(regex.search(html).group(1))


class SeriesParser(PageParser):
    def _parse(self, html):
        soup = bs4.BeautifulSoup(html, 'html.parser')
        series_list = soup.findAll('div', attrs={'class': 'col-sm-4 col-xs-6 col-md-2 col-lg-2'})

        return ['http://{}/{}'.format(ARABIC_TOONS_HOST, div.find('a')['href']) for div in series_list]

    def get_episodes_urls(self, url):
        html = self._html(url)
        return self._parse(html)


class StreamDownloader:
    def __init__(self, stream_url: str, workers: int = 1, ffmpeg: bool = False):
        self.url = stream_url
        self.workers = workers
        self.ffmpeg = ffmpeg

        self.session = requests.session()
        self.session.headers.update(ARABIC_TOONS_USER_AGENT)

    def download(self, filepath: str):
        segments = self._get_segments()

        if self.workers and self.workers > 1:
            with concurrent.futures.ThreadPoolExecutor(max_workers=self.workers) as executor:
                responses = executor.map(self.session.get, segments)
        else:
            responses = [self.session.get(segment_url) for segment_url in segments]

        with open(filepath, 'wb') as f:
            for response in responses:
                f.write(response.content)

        if self.ffmpeg:
            self._ffmpeg(filepath)

    def _get_segments(self):
        return m3u8.load(self.url, headers=self.session.headers).segments.uri

    @staticmethod
    def _ffmpeg(path: str, remove_input=True):
        output_path = path.replace('.ts', '.mp4')

        subprocess.run(["ffmpeg", "-i", path, "-acodec", "copy", "-vcodec", "copy", output_path],
                       stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, check=True)

        if remove_input:
            os.remove(path)


class ATDownloader:
    def __init__(self, directory, workers: int = 1):
        self.directory = directory
        self.workers = workers

        if self.directory is not None and not os.path.exists(self.directory):
            os.makedirs(self.directory)

    def _download_video(self, url):
        stream_url, stream_title = VideoParser().get_stream_info(url)
        filepath = os.path.join(self.directory, self._get_file_name(stream_title))

        for path in [filepath, filepath.replace('.ts', '.mp4')]:
            if os.path.exists(path):
                print(f'{path} already exists!')
                return

        StreamDownloader(stream_url, workers=self.workers, ffmpeg=True).download(filepath)

    def download_movie(self, movie_url):
        self._download_video(movie_url)

    def download_series(self, series_url, specific_episodes=None, nproc=1):
        episodes_urls = SeriesParser().get_episodes_urls(series_url)

        if specific_episodes:
            parsed_episodes = list(self._parse_episodes(specific_episodes))
            episodes_urls = [url for i, url in enumerate(episodes_urls, 1)
                             if i in parsed_episodes]

        if nproc > 1:
            with concurrent.futures.ProcessPoolExecutor(max_workers=nproc) as executor:
                executor.map(self._download_video, episodes_urls)
        else:
            for episode_url in episodes_urls:
                self._download_video(episode_url)

    @staticmethod
    def _parse_episodes(episodes_str):
        for episodes_range in episodes_str.split():
            p = episodes_range.split('-')

            start = end = p[0]
            end = p[1] if len(p) == 2 else end

            for episode in range(int(start), int(end) + 1):
                yield episode

    @staticmethod
    def _get_file_name(name, ext='ts'):
        return f'{name}.{ext}'


def main():
    args = docopt.docopt(__doc__, argv=sys.argv[1:], version=__version__)
    segments = args.get('--segments') or DEFAULT_SIMULTANEOUS_SEGMENTS_COUNT

    downloader = ATDownloader(
        directory=os.path.expanduser(args.get('<directory>') or os.getcwd()), workers=int(segments))

    try:
        if args.get('movie'):
            downloader.download_movie(args.get('<movie_url>'))
        elif args.get('episode'):
            downloader.download_movie(args.get('<episode_url>'))
        elif args.get('series'):
            processes = args.get('--processes') or DEFAULT_PROCESSES_COUNT

            downloader.download_series(args.get('<series_url>'),
                                       args.get('--episodes'),
                                       nproc=int(processes))
    except KeyboardInterrupt:
        pass


if __name__ == "__main__":
    main()

{'fraction_non_alphanumeric': 0.1179225202213708, 'fraction_numerical': 0.011210444160635732, 'mean_word_length': 2.350368433563109, 'pattern_counts': {'":': 2, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 72, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1490994', 'n_tokens_mistral': 4535, 'n_tokens_neox': 4311, 'n_words': 950}