<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re

from module.plugins.Crypter import Crypter

class HoerbuchIn(Crypter):
    __name__ = "HoerbuchIn"
    __type__ = "container"
    __pattern__ = r"http://(www\.)?hoerbuch\.in/(blog\.php\?id=|download_(.*)\.html)"
    __version__ = "0.4"
    __description__ = """Hoerbuch.in Container Plugin"""
    __author_name__ = ("spoob")
    __author_mail__ = ("<EMAIL>")

    def __init__(self, parent):
        Crypter.__init__(self, parent)
        self.parent = parent
        self.html = None

    def download_html(self):
        url = self.parent.url
        self.html = self.req.load(url)

    def file_exists(self):
        """ returns True or False
        """
        self.download_html()
        if re.search(r"Download", self.html) is not None:
            return True
        return False

    def proceed(self, url, location):
        temp_links = []
        download_container = ("Download", "Mirror #1", "Mirror #2", "Mirror #3")
        for container in download_container:
            download_content = re.search("<BR><B>" + container + ":</B>(.*?)<BR><B>", self.html).group(1)
            tmp = re.findall('<A HREF="http://www.hoerbuch.in/cj/out.php\?pct=\d+&url=(http://rs\.hoerbuch\.in/.+?)" TARGET="_blank">Part \d+</A>', download_content)
            if tmp == []: continue
            for link in tmp:
                link_html = self.req.load(link, cookies=True)
                temp_links.append(re.search('<FORM ACTION="(http://.*?)" METHOD="post"', link_html).group(1))
            break

        self.links = temp_links<commit_msg>fixed #206 (HoerbuchIn)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re

from module.plugins.Crypter import Crypter
from module.lib.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup

class HoerbuchIn(Crypter):
    __name__ = "HoerbuchIn"
    __type__ = "container"
    __pattern__ = r"http://(www\.)?hoerbuch\.in/(blog\.php\?id=|download_(.*)\.html)"
    __version__ = "0.5"
    __description__ = """Hoerbuch.in Container Plugin"""
    __author_name__ = ("spoob", "mkaay")
    __author_mail__ = ("<EMAIL>", "<EMAIL>")

    def decrypt(self, pyfile):
        self.pyfile = pyfile
        
        self.html = self.req.load(self.pyfile.url)
        if re.search(r"Download", self.html) is None:
            self.offline()
        
        soup = BeautifulSoup(self.html, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
        package_base = soup.find("a", attrs={"href": re.compile(self.__pattern__)}).text
        
        links = {}
        out = re.compile("http://www.hoerbuch.in/cj/out.php\?pct=\d+&url=(http://rs\.hoerbuch\.in/.*)")
        for a in soup.findAll("a", attrs={"href": out}):
            part = int(a.text.replace("Part ", ""))
            if not part in links.keys():
                links[part] = []
            links[part].append(out.search(a["href"]).group(1))
        
        sortedLinks = {}
        for mirrors in links.values():
            decrypted_mirrors = []
            for u in mirrors:
                src = self.load(u)
                decrypted_mirrors.append(re.search('<FORM ACTION="(http://.*?)" METHOD="post"', src).group(1))
            
            results = self.core.pluginManager.parseUrls(decrypted_mirrors)
        
            for url, hoster in results:
                if not sortedLinks.has_key(hoster):
                    sortedLinks[hoster] = []
                sortedLinks[hoster].append(url)
        
        for hoster, urls in sortedLinks.iteritems():
            self.packages.append(("%s (%s)" % (package_base, hoster), urls, self.pyfile.package().folder))

{'fraction_non_alphanumeric': 0.16158038147138964, 'fraction_numerical': 0.004359673024523161, 'mean_word_length': 2.3372727272727274, 'pattern_counts': {'":': 3, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3729894', 'n_tokens_mistral': 1224, 'n_tokens_neox': 1195, 'n_words': 218}