<commit_before># -*- coding: utf-8 -*-

"""
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License,
    or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    See the GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.

    @author: RaNaN, spoob, mkaay
"""

from time import time, sleep
from random import randint

import os

if os.name != "nt":
    from module.utils.fs import chown
    from pwd import getpwnam
    from grp import getgrnam

from Base import Base, Fail, Retry
from module.utils import chunks as _chunks
from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\
    remove, makedirs, chmod, stat, exists, join

# Import for Hoster Plugins
chunks = _chunks

class Abort(Exception):
    """ raised when aborted """

class Reconnect(Exception):
    """ raised when reconnected """

class SkipDownload(Exception):
    """ raised when download should be skipped """

class Hoster(Base):
    """
    Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading.
    """

    @staticmethod
    def getInfo(urls):
        """This method is used to retrieve the online status of files for hoster plugins.
        It has to *yield* list of tuples with the result in this format (name, size, status, url),
        where status is one of API pyfile statusses.

        :param urls: List of urls
        :return: yield list of tuple with results (name, size, status, url)
        """
        pass

    def __init__(self, pyfile):
        Base.__init__(self, pyfile.m.core)

        self.wantReconnect = False
        #: enables simultaneous processing of multiple downloads
        self.limitDL = 0
        #: chunk limit
        self.chunkLimit = 1
        #: enables resume (will be ignored if server dont accept chunks)
        self.resumeDownload = False

        #: time() + wait in seconds
        self.waitUntil = 0
        self.waiting = False

        self.ocr = None  #captcha reader instance
        #: account handler instance, see :py:class:`Account`
        self.account = self.core.accountManager.getAccountForPlugin(self.__name__)

        #: premium status
        self.premium = False
        #: username/login
        self.user = None

        if self.account and not self.account.isUsable(): self.account = None
        if self.account:
            self.user = self.account.loginname
            #: Browser instance, see `network.Browser`
            self.req = self.account.getAccountRequest()
            # Default:  -1, True, True
            self.chunkLimit, self.resumeDownload, self.multiDL = self.account.getDownloadSettings()
            self.premium = self.account.isPremium()
        else:
            self.req = self.core.requestFactory.getRequest(self.__name__)

        #: associated pyfile instance, see `PyFile`
        self.pyfile = pyfile
        self.thread = None # holds thread in future

        #: location where the last call to download was saved
        self.lastDownload = ""
        #: re match of the last call to `checkDownload`
        self.lastCheck = None
        #: js engine, see `JsEngine`
        self.js = self.core.js
        self.cTask = None #captcha task

        self.retries = 0 # amount of retries already made
        self.html = None # some plugins store html code here

        self.init()

    def getMultiDL(self):
        self.logDebug("Deprectated attribute multiDL, use limitDL instead")
        return self.limitDL <= 0

    def setMultiDL(self, val):
        self.logDebug("Deprectated attribute multiDL, use limitDL instead")
        self.limitDL = 0 if val else 1

    multiDL = property(getMultiDL, setMultiDL)

    def getChunkCount(self):
        if self.chunkLimit <= 0:
            return self.config["download"]["chunks"]
        return min(self.config["download"]["chunks"], self.chunkLimit)

    def getDownloadLimit(self):
        if self.account:
            limit = self.account.options.get("limitDL", 0)
            if limit == "": limit = 0
            return int(limit)
        else:
            return self.limitDL


    def __call__(self):
        return self.__name__

    def init(self):
        """initialize the plugin (in addition to `__init__`)"""
        pass

    def setup(self):
        """ setup for enviroment and other things, called before downloading (possibly more than one time)"""
        pass

    def preprocessing(self, thread):
        """ handles important things to do before starting """
        self.thread = thread

        if self.account:
            # will force a relogin or reload of account info if necessary
            self.account.getAccountInfo()
        else:
            self.req.clearCookies()

        self.setup()

        self.pyfile.setStatus("starting")

        return self.process(self.pyfile)


    def process(self, pyfile):
        """the 'main' method of every plugin, you **have to** overwrite it"""
        raise NotImplementedError

    def resetAccount(self):
        """ dont use account and retry download """
        self.account = None
        self.req = self.core.requestFactory.getRequest(self.__name__)
        self.retry()

    def checksum(self, local_file=None):
        """
        return codes:
        0  - checksum ok
        1  - checksum wrong
        5  - can't get checksum
        10 - not implemented
        20 - unknown error
        """
        #@TODO checksum check hook

        return True, 10


    def setWait(self, seconds, reconnect=False):<commit_msg>Don't fail miserably on hoster's internal server error.<commit_after># -*- coding: utf-8 -*-

"""
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License,
    or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    See the GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.

    @author: RaNaN, spoob, mkaay
"""

from time import time, sleep
from random import randint

import os

if os.name != "nt":
    from module.utils.fs import chown
    from pwd import getpwnam
    from grp import getgrnam

from Base import Base, Fail, Retry
from module.utils import chunks as _chunks
from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\
    remove, makedirs, chmod, stat, exists, join

# Import for Hoster Plugins
chunks = _chunks

class Abort(Exception):
    """ raised when aborted """

class Reconnect(Exception):
    """ raised when reconnected """

class SkipDownload(Exception):
    """ raised when download should be skipped """

class Hoster(Base):
    """
    Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading.
    """

    @staticmethod
    def getInfo(urls):
        """This method is used to retrieve the online status of files for hoster plugins.
        It has to *yield* list of tuples with the result in this format (name, size, status, url),
        where status is one of API pyfile statusses.

        :param urls: List of urls
        :return: yield list of tuple with results (name, size, status, url)
        """
        pass

    def __init__(self, pyfile):
        Base.__init__(self, pyfile.m.core)

        self.wantReconnect = False
        #: enables simultaneous processing of multiple downloads
        self.limitDL = 0
        #: chunk limit
        self.chunkLimit = 1
        #: enables resume (will be ignored if server dont accept chunks)
        self.resumeDownload = False

        #: time() + wait in seconds
        self.waitUntil = 0
        self.waiting = False

        self.ocr = None  #captcha reader instance
        #: account handler instance, see :py:class:`Account`
        self.account = self.core.accountManager.getAccountForPlugin(self.__name__)

        #: premium status
        self.premium = False
        #: username/login
        self.user = None

        if self.account and not self.account.isUsable(): self.account = None
        if self.account:
            self.user = self.account.loginname
            #: Browser instance, see `network.Browser`
            self.req = self.account.getAccountRequest()
            # Default:  -1, True, True
            self.chunkLimit, self.resumeDownload, self.multiDL = self.account.getDownloadSettings()
            self.premium = self.account.isPremium()
        else:
            self.req = self.core.requestFactory.getRequest(self.__name__)

        #: associated pyfile instance, see `PyFile`
        self.pyfile = pyfile
        self.thread = None # holds thread in future

        #: location where the last call to download was saved
        self.lastDownload = ""
        #: re match of the last call to `checkDownload`
        self.lastCheck = None
        #: js engine, see `JsEngine`
        self.js = self.core.js
        self.cTask = None #captcha task

        self.retries = 0 # amount of retries already made
        self.html = None # some plugins store html code here

        self.init()

    def getMultiDL(self):
        self.logDebug("Deprectated attribute multiDL, use limitDL instead")
        return self.limitDL <= 0

    def setMultiDL(self, val):
        self.logDebug("Deprectated attribute multiDL, use limitDL instead")
        self.limitDL = 0 if val else 1

    multiDL = property(getMultiDL, setMultiDL)

    def getChunkCount(self):
        if self.chunkLimit <= 0:
            return self.config["download"]["chunks"]
        return min(self.config["download"]["chunks"], self.chunkLimit)

    def getDownloadLimit(self):
        if self.account:
            limit = self.account.options.get("limitDL", 0)
            if limit == "": limit = 0
            return int(limit)
        else:
            return self.limitDL


    def __call__(self):
        return self.__name__

    def init(self):
        """initialize the plugin (in addition to `__init__`)"""
        pass

    def setup(self):
        """ setup for enviroment and other things, called before downloading (possibly more than one time)"""
        pass

    def preprocessing(self, thread):
        """ handles important things to do before starting """
        self.thread = thread

        if self.account:
            # will force a relogin or reload of account info if necessary
            self.account.getAccountInfo()
        else:
            self.req.clearCookies()

        self.setup()

        self.pyfile.setStatus("starting")

        try:
            return self.process(self.pyfile)
        except Exception, e:
            # Can't seem to import BadHeader
            if e.__class__.__name__ == 'BadHeader' and e.code == 500:
                self.logInfo("Internal Server Error")
                self.tempOffline()
            raise e


    def process(self, pyfile):
        """the 'main' method of every plugin, you **have to** overwrite it"""
        raise NotImplementedError

    def resetAccount(self):
        """ dont use account and retry download """
        self.account = None
        self.req = self.core.requestFactory.getRequest(self.__name__)
        self.retry()

    def checksum(self, local_file=None):
        """
        return codes:
        0  - checksum ok
        1  - checksum wrong
        5  - can't get checksum
        10 - not implemented
        20 - unknown error
        """
        #@TODO checksum check hook

        return True, 10


    def setWait(self, seconds, reconnect=False):
{'fraction_non_alphanumeric': 0.08605486610058785, 'fraction_numerical': 0.00383736120182887, 'mean_word_length': 2.328532608695652, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '85017', 'n_tokens_mistral': 3356, 'n_tokens_neox': 3205, 'n_words': 1191}