<commit_before># -*- coding: utf-8 -*-

# Test links (random.bin):
# http://ul.to/044yug9o
# http://ul.to/gzfhd0xs

import re

from module.utils import html_unescape, parseFileSize

from module.plugins.Hoster import Hoster
from module.network.RequestFactory import getURL
from module.plugins.Plugin import chunks
from module.plugins.ReCaptcha import ReCaptcha

key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64')


def getID(url):
    """ returns id from file url"""
    m = re.match(UploadedTo.__pattern__, url)
    return m.group('ID')


def getAPIData(urls):
    post = {"apikey": key}

    idMap = {}

    for i, url in enumerate(urls):
        id = getID(url)
        post["id_%s" % i] = id
        idMap[id] = url

    api = unicode(getURL("http://uploaded.net/api/filemultiple", post=post, decode=False), 'iso-8859-1')

    result = {}

    if api:
        for line in api.splitlines():
            data = line.split(",", 4)
            if data[1] in idMap:
                result[data[1]] = (data[0], data[2], data[4], data[3], idMap[data[1]])

    return result


def parseFileInfo(self, url='', html=''):
    if not html and hasattr(self, "html"): html = self.html
    name, size, status, found, fileid = url, 0, 3, None, None

    if re.search(self.FILE_OFFLINE_PATTERN, html):
        # File offline
        status = 1
    else:
        found = re.search(self.FILE_INFO_PATTERN, html)
        if found:
            name, fileid = html_unescape(found.group('N')), found.group('ID')
            size = parseFileSize(found.group('S'))
            status = 2

    return name, size, status, fileid


def getInfo(urls):
    for chunk in chunks(urls, 80):
        result = []

        api = getAPIData(chunk)

        for data in api.itervalues():
            if data[0] == "online":
                result.append((html_unescape(data[2]), data[1], 2, data[4]))

            elif data[0] == "offline":
                result.append((data[4], 0, 1, data[4]))

        yield result


class UploadedTo(Hoster):
    __name__ = "UploadedTo"
    __type__ = "hoster"
    __pattern__ = r"https?://[\w\.-]*?(uploaded\.(to|net)|ul\.to)(/file/|/?\?id=|.*?&id=|/)(?P<ID>\w+)"
    __version__ = "0.68"
    __description__ = """Uploaded.net Download Hoster"""
    __author_name__ = ("spoob", "mkaay", "zoidberg", "netpok", "stickell")
    __author_mail__ = ("<EMAIL>", "<EMAIL>", "<EMAIL>", "<EMAIL>", "<EMAIL>")

    FILE_INFO_PATTERN = r'<a href="file/(?P<ID>\w+)" id="filename">(?P<N>[^<]+)</a> &nbsp;\s*<small[^>]*>(?P<S>[^<]+)</small>'
    FILE_OFFLINE_PATTERN = r'<small class="cL">Error: 404</small>'
    DL_LIMIT_PATTERN = "You have reached the max. number of possible free downloads for this hour"

    def setup(self):
        self.html = None
        self.multiDL = False
        self.resumeDownload = False<commit_msg>UploadedTo: sometimes API returns "can't find request". Trying three times.<commit_after># -*- coding: utf-8 -*-

# Test links (random.bin):
# http://ul.to/044yug9o
# http://ul.to/gzfhd0xs

import re

from module.utils import html_unescape, parseFileSize

from module.plugins.Hoster import Hoster
from module.network.RequestFactory import getURL
from module.plugins.Plugin import chunks
from module.plugins.ReCaptcha import ReCaptcha

key = "<KEY>('base64')


def getID(url):
    """ returns id from file url"""
    m = re.match(UploadedTo.__pattern__, url)
    return m.group('ID')


def getAPIData(urls):
    post = {"apikey": key}

    idMap = {}

    for i, url in enumerate(urls):
        id = getID(url)
        post["id_%s" % i] = id
        idMap[id] = url

    for i in xrange(3):
        api = unicode(getURL("http://uploaded.net/api/filemultiple", post=post, decode=False), 'iso-8859-1')
        if api != "can't find request":
            break

    result = {}

    if api:
        for line in api.splitlines():
            data = line.split(",", 4)
            if data[1] in idMap:
                result[data[1]] = (data[0], data[2], data[4], data[3], idMap[data[1]])

    return result


def parseFileInfo(self, url='', html=''):
    if not html and hasattr(self, "html"): html = self.html
    name, size, status, found, fileid = url, 0, 3, None, None

    if re.search(self.FILE_OFFLINE_PATTERN, html):
        # File offline
        status = 1
    else:
        found = re.search(self.FILE_INFO_PATTERN, html)
        if found:
            name, fileid = html_unescape(found.group('N')), found.group('ID')
            size = parseFileSize(found.group('S'))
            status = 2

    return name, size, status, fileid


def getInfo(urls):
    for chunk in chunks(urls, 80):
        result = []

        api = getAPIData(chunk)

        for data in api.itervalues():
            if data[0] == "online":
                result.append((html_unescape(data[2]), data[1], 2, data[4]))

            elif data[0] == "offline":
                result.append((data[4], 0, 1, data[4]))

        yield result


class UploadedTo(Hoster):
    __name__ = "UploadedTo"
    __type__ = "hoster"
    __pattern__ = r"https?://[\w\.-]*?(uploaded\.(to|net)|ul\.to)(/file/|/?\?id=|.*?&id=|/)(?P<ID>\w+)"
    __version__ = "0.69"
    __description__ = """Uploaded.net Download Hoster"""
    __author_name__ = ("spoob", "mkaay", "zoidberg", "netpok", "stickell")
    __author_mail__ = ("<EMAIL>", "<EMAIL>", "<EMAIL>", "<EMAIL>", "<EMAIL>")

    FILE_INFO_PATTERN = r'<a href="file/(?P<ID>\w+)" id="filename">(?P<N>[^<]+)</a> &nbsp;\s*<small[^>]*>(?P<S>[^<]+)</small>'
    FILE_OFFLINE_PATTERN = r'<small class="cL">Error: 404</small>'
    DL_LIMIT_PATTERN = "You have reached the max. number of possible free downloads for this hour"

    def setup(self):
        self.html = None
        self.multiDL = False
        self.resumeDownload = False
{'fraction_non_alphanumeric': 0.17096383457345563, 'fraction_numerical': 0.015919709292265098, 'mean_word_length': 2.8026315789473686, 'pattern_counts': {'":': 7, '<': 38, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2119390', 'n_tokens_mistral': 2104, 'n_tokens_neox': 2012, 'n_words': 468}