<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import tempfile
from time import time
from base64 import b64decode
import hashlib
import random
from time import sleep

from module.plugins.Hoster import Hoster

class ShareonlineBiz(Hoster):
    __name__ = "ShareonlineBiz"
    __type__ = "hoster"
    __pattern__ = r"(?:http://)?(?:www.)?share-online.biz/download.php\?id="
    __version__ = "0.1"
    __description__ = """Shareonline.biz Download Hoster"""
    __author_name__ = ("spoob")
    __author_mail__ = ("<EMAIL>")

    def __init__(self, parent):
        Hoster.__init__(self, parent)
        self.parent = parent
        self.html = [None, None]
        self.want_reconnect = False
        self.init_ocr()
        self.url = self.parent.url
        self.read_config()
        if self.config['premium']:
            self.multi_dl = True
        else:
            self.multi_dl = False

    def prepare(self, thread):
        pyfile = self.parent

        self.download_api_data()
        if self.api_data["status"]:
            for i in range(5):
                if self.download_html():
                    break
                else:
                    pyfile.status.waituntil = self.time_plus_wait
                    thread.wait(self.pyfile)
            pyfile.status.filename = self.api_data["filename"]
            pyfile.status.waituntil = self.time_plus_wait
            pyfile.status.url = self.get_file_url()
            pyfile.status.want_reconnect = self.want_reconnect
            return True
        else:
            return False

    def download_api_data(self):
        """
        http://images.rapidshare.com/apidoc.txt
        """
        api_url_base = "http://www.share-online.biz/linkcheck/linkcheck.php?md5=1"
        api_param_file = {"links": self.url}
        src = self.load(api_url_base, cookies=False, post=api_param_file)

        fields = src.split(";")
        self.api_data = {}
        self.api_data["fileid"] = fields[0]
        self.api_data["status"] = fields[1]
        if self.api_data["status"] == "NOTFOUND":
            return
        self.api_data["filename"] = fields[2]
        self.api_data["size"] = fields[3] # in bytes
        self.api_data["checksum"] = fields[4].strip().lower().replace("\n\n", "") # md5        

    def download_html(self):
        if self.config['premium']:
            post_vars = {"act": "login",
                         "location": "service.php",
                         "dieseid": "",
                         "user": self.config['username'],
                         "pass": self.config['password'],
                         "login":"Log+me+in",
                         "folder_autologin":"1"}
            self.load("http://www.share-online.biz/login.php", cookies=True, post=post_vars)
        url = self.parent.url
        self.html[0] = self.load(url, cookies=True)
        
        if not self.config['premium']:
            #captcha_image = tempfile.NamedTemporaryFile(suffix=".jpg").name

            for i in range(10):

                captcha_image = tempfile.NamedTemporaryFile(suffix=".jpg", delete=False)
                imgStr = self.req.load("http://www.share-online.biz/captcha.php?rand="+ "0." + str(random.randint(10**15,10**16)), cookies=True)
                captcha_image.write(imgStr)
                captcha_image.close()
                captcha = self.ocr.get_captcha(captcha_image.name)
                os.remove(captcha_image.name)
                self.logger.debug("%s Captcha %s: %s" % (self.__name__, i, captcha))
                sleep(3)
                self.html[1] = self.load(url, post={"captchacode": captcha}, cookies=True)
                if re.search(r"no slots available", self.html[1]):
                    self.time_plus_wait = time() + 120
                    self.logger.debug("%s: no free slots, waiting 120 seconds" % (self.__name__))
                    return False
                if re.search(r"Der Download ist Ihnen zu langsam", self.html[1]):
                    self.time_plus_wait = time() + 15
                    return True

            raise Exception("Captcha not decrypted")
        else:
            return True
       
    def get_file_url(self):
        """ returns the absolute downloadable filepath
        """
        if not self.want_reconnect:
            file_url_pattern = 'loadfilelink\.decode\("([^"]+)'
            return b64decode(re.search(file_url_pattern, self.html[1]).group(1))
        else:
            return False

    def check_file(self, local_file):
        if self.api_data and self.api_data["checksum"]:
            h = hashlib.md5()
            f = open(local_file, "rb")
            h.update(f.read())
            f.close()<commit_msg>python 2.5 compatibility fix for ShareOnline.Biz<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import os.path
import re
import tempfile
from time import time
from base64 import b64decode
import hashlib
import random
from time import sleep

from module.plugins.Hoster import Hoster

class ShareonlineBiz(Hoster):
    __name__ = "ShareonlineBiz"
    __type__ = "hoster"
    __pattern__ = r"(?:http://)?(?:www.)?share-online.biz/download.php\?id="
    __version__ = "0.1"
    __description__ = """Shareonline.biz Download Hoster"""
    __author_name__ = ("spoob")
    __author_mail__ = ("<EMAIL>")

    def __init__(self, parent):
        Hoster.__init__(self, parent)
        self.parent = parent
        self.html = [None, None]
        self.want_reconnect = False
        self.init_ocr()
        self.url = self.parent.url
        self.read_config()
        if self.config['premium']:
            self.multi_dl = True
        else:
            self.multi_dl = False

    def prepare(self, thread):
        pyfile = self.parent

        self.download_api_data()
        if self.api_data["status"]:
            for i in range(5):
                if self.download_html():
                    break
                else:
                    pyfile.status.waituntil = self.time_plus_wait
                    thread.wait(self.pyfile)
            pyfile.status.filename = self.api_data["filename"]
            pyfile.status.waituntil = self.time_plus_wait
            pyfile.status.url = self.get_file_url()
            pyfile.status.want_reconnect = self.want_reconnect
            return True
        else:
            return False

    def download_api_data(self):
        """
        http://images.rapidshare.com/apidoc.txt
        """
        api_url_base = "http://www.share-online.biz/linkcheck/linkcheck.php?md5=1"
        api_param_file = {"links": self.url}
        src = self.load(api_url_base, cookies=False, post=api_param_file)

        fields = src.split(";")
        self.api_data = {}
        self.api_data["fileid"] = fields[0]
        self.api_data["status"] = fields[1]
        if self.api_data["status"] == "NOTFOUND":
            return
        self.api_data["filename"] = fields[2]
        self.api_data["size"] = fields[3] # in bytes
        self.api_data["checksum"] = fields[4].strip().lower().replace("\n\n", "") # md5        

    def download_html(self):
        if self.config['premium']:
            post_vars = {"act": "login",
                         "location": "service.php",
                         "dieseid": "",
                         "user": self.config['username'],
                         "pass": self.config['password'],
                         "login":"Log+me+in",
                         "folder_autologin":"1"}
            self.load("http://www.share-online.biz/login.php", cookies=True, post=post_vars)
        url = self.parent.url
        self.html[0] = self.load(url, cookies=True)
        
        if not self.config['premium']:
            #captcha_image = tempfile.NamedTemporaryFile(suffix=".jpg").name

            for i in range(10):
                try:
                    captcha_image = tempfile.NamedTemporaryFile(suffix=".jpg", delete=False)
                    
                # Fallback for python version <2.6
                except TypeError:
                    captcha_image_name = os.path.join(tempfile.gettempdir(), "pyload_tmp_%d"%time())
                    captcha_image = open(captcha_image_name, "w+b")
                    
                imgStr = self.req.load("http://www.share-online.biz/captcha.php?rand="+ "0." + str(random.randint(10**15,10**16)), cookies=True)
                captcha_image.write(imgStr)
                captcha_image.close()
                
                # again fallback
                try:
                    captcha = self.ocr.get_captcha(captcha_image.name)
                    os.remove(captcha_image.name)
                    
                except AttributeError:
                    captcha = self.ocr.get_captcha(captcha_image_name)
                    os.remove(captcha_image_name)
                    
                self.logger.debug("%s Captcha %s: %s" % (self.__name__, i, captcha))
                sleep(3)
                self.html[1] = self.load(url, post={"captchacode": captcha}, cookies=True)
                if re.search(r"no slots available", self.html[1]):
                    self.time_plus_wait = time() + 120
                    self.logger.debug("%s: no free slots, waiting 120 seconds" % (self.__name__))
                    return False
                if re.search(r"Der Download ist Ihnen zu langsam", self.html[1]):
                    self.time_plus_wait = time() + 15
                    return True

            raise Exception("Captcha not decrypted")
        else:
            return True
       
    def get_file_url(self):
        """ returns the absolute downloadable filepath
        """
        if not self.want_reconnect:
            file_url_pattern = 'loadfilelink\.decode\("([^"]+)'
            return b64decode(re.search(file_url_pattern, self.html[1]).group(1))
        else:
            return False

    def check_file(self, local_file):
        if self.api_data and self.api_data["checksum"]:
            h = hashlib.md5()
            f = open(local_file, "rb")
            h.update(f.read())
            f.close()
{'fraction_non_alphanumeric': 0.12491373360938578, 'fraction_numerical': 0.009464655427388346, 'mean_word_length': 1.789111905416552, 'pattern_counts': {'":': 20, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2751445', 'n_tokens_mistral': 3098, 'n_tokens_neox': 2930, 'n_words': 556}