<commit_before># -*- coding: utf-8 -*-

import re

from module.utils import html_unescape, parseFileSize

from module.plugins.Hoster import Hoster
from module.network.RequestFactory import getURL
from module.plugins.Plugin import chunks
from module.plugins.ReCaptcha import ReCaptcha

key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64')

def correctDownloadLink(url):
    url = re.sub("http://.*?/", "http://uploaded.to/",url, 1)
    url = re.sub("\\.to/.*?id=", ".to/file/", url, 1)

    if "/file/" not in url:
        url = url.replace("uploaded.to/", "uploaded.to/file/")

    parts = url.split("/")
    return "/".join(parts[:min(5,len(parts))]) + "/"

def getID(url):
    """ returns id of corrected url"""
    return re.search(r"uploaded.to/file/([^/]+)", url).group(1)

def getAPIData(urls):
        post = {"apikey" : key}

        idMap = {}

        for i, url in enumerate(urls):
            newUrl = correctDownloadLink(url)
            id = getID(newUrl)
            post["id_%s" % i] = id
            idMap[id] = url

        api = unicode(getURL("http://uploaded.to/api/filemultiple", post=post, decode=False), 'iso-8859-1')

        result = {}

        if api:
            for line in api.splitlines():
                data = line.split(",")
                if data[1] in idMap:
                    result[data[1]] = (data[0], data[2], data[4], data[3], idMap[data[1]])

        return result

def parseFileInfo(self, url = '', html = ''):
    if not html and hasattr(self, "html"): html = self.html
    name, size, status, found, fileid = url, 0, 3, None, None

    if re.search(self.FILE_OFFLINE_PATTERN, html):
        # File offline
        status = 1
    else:
        found = re.search(self.FILE_INFO_PATTERN, html)
        if found:
            name, fileid = html_unescape(found.group('N')), found.group('ID')
            size = parseFileSize(found.group('S'))
            status = 2

    return name, size, status, fileid

def getInfo(urls):
    for chunk in chunks(urls, 80):
        result = []

        api = getAPIData(chunk)

        for data in api.itervalues():
            if data[0] == "online":
                result.append((html_unescape(data[2]), data[1], 2, data[4]))

            elif data[0] == "offline":
                result.append((data[4], 0, 1, data[4]))

        yield result


class UploadedTo(Hoster):
    __name__ = "UploadedTo"
    __type__ = "hoster"
    __pattern__ = r"(http://[\w\.-]*?uploaded\.to/.*?(file/|\?id=|&id=)[\w]+/?)|(http://[\w\.]*?ul\.to/(\?id=|&id=)?[\w\-]+/.+)|(http://[\w\.]*?ul\.to/(\?id=|&id=)?[\w\-]+/?)"
    __version__ = "0.54"
    __description__ = """Uploaded.to Download Hoster"""
    __author_name__ = ("spoob", "mkaay")
    __author_mail__ = ("<EMAIL>", "<EMAIL>")

    FILE_INFO_PATTERN = r'<a href="file/(?P<ID>\w+)" id="filename">(?P<N>[^<]+)</a> &nbsp;\s*<small[^>]*>(?P<S>[^<]+)</small>'
    FILE_OFFLINE_PATTERN = r'<small class="cL">Error: 404</small>'

    def setup(self):
        self.html = None
        self.multiDL = False
        self.resumeDownload = False
        self.url = False
        self.chunkLimit = 1 # critical problems with more chunks
        if self.account:
            self.premium = self.account.getAccountInfo(self.user)["premium"]
            if self.premium:
                self.multiDL = True
                self.resumeDownload = True


        self.pyfile.url = correctDownloadLink(self.pyfile.url)
        self.fileID = getID(self.pyfile.url)

    def process(self, pyfile):
        self.req.cj.setCookie("uploaded.to", "lang", "en")

        api = getAPIData([pyfile.url])

        # TODO: fallback to parse from site, because api sometimes delivers wrong status codes

        if not api:
            self.logWarning("No response for API call")

            self.html = unicode(self.load(pyfile.url, decode = False), 'iso-8859-1')
            name, size, status, self.fileID = parseFileInfo(self)
            self.logDebug(name, size, status, self.fileID)
            if status == 1:
                self.offline()
            elif status == 2:
                pyfile.name, pyfile.size = name, size
            else:
                self.fail('Parse error - file info')
        elif api == 'Access denied':
            self.fail(_("API key invalid"))

        else:
            if self.fileID not in api:
                self.offline()

            self.data = api[self.fileID]
            if self.data[0] != "online":
                self.offline()

            pyfile.name = html_unescape(self.data[2])

        # self.pyfile.name = self.get_file_name()

        if self.premium:
            self.handlePremium()
        else:
            self.handleFree()


    def handlePremium(self):
        info = self.account.getAccountInfo(self.user, True)
        self.log.debug("%(name)s: Use Premium Account (%(left)sGB left)" % {"name" :self.__name__, "left" : info["trafficleft"]/1024/1024})
        if int(self.data[1])/1024 > info["trafficleft"]:
            self.log.info(_("%s: Not enough traffic left" % self.__name__))
            self.account.empty(self.user)
            self.resetAccount()
            self.fail(_("Traffic exceeded"))

        self.download("http://uploaded.to/file/%s/ddl" % self.fileID)


    def handleFree(self):
        self.html = self.load(self.pyfile.url, decode=True)
        
        if 'var free_enabled = false;' in self.html:
            self.logError("Free-download capacities exhausted.")
            self.retry(24, 300)

        wait = re.search(r"Current waiting period: <span>(\d+)</span> seconds", self.html).group(1)<commit_msg>closed #548<commit_after># -*- coding: utf-8 -*-

import re

from module.utils import html_unescape, parseFileSize

from module.plugins.Hoster import Hoster
from module.network.RequestFactory import getURL
from module.plugins.Plugin import chunks
from module.plugins.ReCaptcha import ReCaptcha

key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64')

def correctDownloadLink(url):
    url = re.sub("http://.*?/", "http://uploaded.to/",url, 1)
    url = re.sub("\\.to/.*?id=", ".to/file/", url, 1)

    if "/file/" not in url:
        url = url.replace("uploaded.to/", "uploaded.to/file/")

    parts = url.split("/")
    return "/".join(parts[:min(5,len(parts))]) + "/"

def getID(url):
    """ returns id of corrected url"""
    return re.search(r"uploaded.to/file/([^/]+)", url).group(1)

def getAPIData(urls):
        post = {"apikey" : key}

        idMap = {}

        for i, url in enumerate(urls):
            newUrl = correctDownloadLink(url)
            id = getID(newUrl)
            post["id_%s" % i] = id
            idMap[id] = url

        api = unicode(getURL("http://uploaded.to/api/filemultiple", post=post, decode=False), 'iso-8859-1')

        result = {}

        if api:
            for line in api.splitlines():
                data = line.split(",")
                if data[1] in idMap:
                    result[data[1]] = (data[0], data[2], data[4], data[3], idMap[data[1]])

        return result

def parseFileInfo(self, url = '', html = ''):
    if not html and hasattr(self, "html"): html = self.html
    name, size, status, found, fileid = url, 0, 3, None, None

    if re.search(self.FILE_OFFLINE_PATTERN, html):
        # File offline
        status = 1
    else:
        found = re.search(self.FILE_INFO_PATTERN, html)
        if found:
            name, fileid = html_unescape(found.group('N')), found.group('ID')
            size = parseFileSize(found.group('S'))
            status = 2

    return name, size, status, fileid

def getInfo(urls):
    for chunk in chunks(urls, 80):
        result = []

        api = getAPIData(chunk)

        for data in api.itervalues():
            if data[0] == "online":
                result.append((html_unescape(data[2]), data[1], 2, data[4]))

            elif data[0] == "offline":
                result.append((data[4], 0, 1, data[4]))

        yield result


class UploadedTo(Hoster):
    __name__ = "UploadedTo"
    __type__ = "hoster"
    __pattern__ = r"(http://[\w\.-]*?uploaded\.to/.*?(file/|\?id=|&id=)[\w]+/?)|(http://[\w\.]*?ul\.to/(\?id=|&id=)?[\w\-]+/.+)|(http://[\w\.]*?ul\.to/(\?id=|&id=)?[\w\-]+/?)"
    __version__ = "0.55"
    __description__ = """Uploaded.to Download Hoster"""
    __author_name__ = ("spoob", "mkaay", "zoidberg")
    __author_mail__ = ("<EMAIL>", "<EMAIL>", "<EMAIL>")

    FILE_INFO_PATTERN = r'<a href="file/(?P<ID>\w+)" id="filename">(?P<N>[^<]+)</a> &nbsp;\s*<small[^>]*>(?P<S>[^<]+)</small>'
    FILE_OFFLINE_PATTERN = r'<small class="cL">Error: 404</small>'

    def setup(self):
        self.html = None
        self.multiDL = False
        self.resumeDownload = False
        self.url = False
        self.chunkLimit = 1 # critical problems with more chunks
        if self.account:
            self.premium = self.account.getAccountInfo(self.user)["premium"]
            if self.premium:
                self.multiDL = True
                self.resumeDownload = True


        self.pyfile.url = correctDownloadLink(self.pyfile.url)
        self.fileID = getID(self.pyfile.url)

    def process(self, pyfile):
        self.req.cj.setCookie("uploaded.to", "lang", "en")

        api = getAPIData([pyfile.url])

        # TODO: fallback to parse from site, because api sometimes delivers wrong status codes

        if not api:
            self.logWarning("No response for API call")

            self.html = unicode(self.load(pyfile.url, decode = False), 'iso-8859-1')
            name, size, status, self.fileID = parseFileInfo(self)
            self.logDebug(name, size, status, self.fileID)
            if status == 1:
                self.offline()
            elif status == 2:
                pyfile.name, pyfile.size = name, size
            else:
                self.fail('Parse error - file info')
        elif api == 'Access denied':
            self.fail(_("API key invalid"))

        else:
            if self.fileID not in api:
                self.offline()

            self.data = api[self.fileID]
            if self.data[0] != "online":
                self.offline()

            pyfile.name = html_unescape(self.data[2])

        # self.pyfile.name = self.get_file_name()

        if self.premium:
            self.handlePremium()
        else:
            self.handleFree()


    def handlePremium(self):
        info = self.account.getAccountInfo(self.user, True)
        self.log.debug("%(name)s: Use Premium Account (%(left)sGB left)" % {"name" :self.__name__, "left" : info["trafficleft"]/1024/1024})
        if int(self.data[1])/1024 > info["trafficleft"]:
            self.log.info(_("%s: Not enough traffic left" % self.__name__))
            self.account.empty(self.user)
            self.resetAccount()
            self.fail(_("Traffic exceeded"))

        header = self.load("http://uploaded.to/file/%s" % self.fileID, just_header=True)
        if "location" in header:
            #Direct download
            print "Direct Download: " + header['location']
            self.download(header['location'])
        else:
            #Indirect download
            self.html = self.load("http://uploaded.to/file/%s" % self.fileID)
            found = re.search(r'<div class="tfree".*\s*<form method="post" action="(.*?)"', self.html)
            if not found:
                self.fail("Download URL not found. Try to enable direct downloads.")
            url = found.group(1)
            print "Premium URL: " + url
            self.download(url, post={})

    def handleFree(self):
        self.html = self.load(self.pyfile.url, decode=True)
        
        if 'var free_enabled = false;' in self.html:
            self.logError("Free-download capacities exhausted.")
            self.retry(24, 300)

        wait = re.search(r"Current waiting period: <span>(\d+)</span> seconds", self.html).group(1)
{'fraction_non_alphanumeric': 0.1634421265141319, 'fraction_numerical': 0.012954239569313594, 'mean_word_length': 2.201992997576084, 'pattern_counts': {'":': 6, '<': 36, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1347851', 'n_tokens_mistral': 4047, 'n_tokens_neox': 3824, 'n_words': 819}