<commit_before># -*- coding: utf-8 -*-

import re
import urlparse

from module.plugins.internal.SimpleCrypter import SimpleCrypter


class MultiUpOrg(SimpleCrypter):
    __name__    = "MultiUpOrg"
    __type__    = "crypter"
    __version__ = "0.09"
    __status__  = "testing"

    __pattern__ = r'http://(?:www\.)?multiup\.org/(en/|fr/)?(?P<TYPE>project|download|mirror)/\w+(/\w+)?'
    __config__  = [("activated"         , "bool"          , "Activated"                                        , True     ),
                   ("use_premium"       , "bool"          , "Use premium account if available"                 , True     ),
                   ("folder_per_package", "Default;Yes;No", "Create folder for each package"                   , "Default"),
                   ("max_wait"          , "int"           , "Reconnect if waiting time is greater than minutes", 10       ),
                   ("hosts_priority"    , "str"           , "Prefered hosts priority (bar-separated)"          , ""       ),
                   ("ignored_hosts"     , "str"           , "Ignored hosts list (bar-separated)"               , ""       ),
                   ("grab_all"          , "bool"          , "Grab all URLs (default only first match)"         , False    )]

    __description__ = """MultiUp.org crypter plugin"""
    __license__     = "GPLv3"
    __authors__     = [("<NAME>", "<EMAIL>")]


    NAME_PATTERN = r'<title>.*(?:Project|Projet|ownload|élécharger) (?P<N>.+?) (\(|- )'

    URL_REPLACEMENTS = [(r'http://(?:www\.)?multiup\.org/(en/|fr/)?', "http://www.multiup.org/en/")]


    def get_links(self):
        m_type = self.info['pattern']['TYPE']
        hosts_priority = [_h for _h in self.config.get('hosts_priority').split('|') if _h]
        ignored_hosts  = [_h for _h in self.config.get('ignored_hosts').split('|') if _h]
        grab_all       = self.config.get('grab_all')

        if m_type == "project":
            return re.findall(r'\n(http://www\.multiup\.org/(?:en|fr)/download/.*)', self.data)

        elif m_type == "download":
            dl_pattern = r'href="(.*)">.*\n.*<h5>DOWNLOAD</h5>'
            mirror_page = urlparse.urljoin("http://www.multiup.org/", re.search(dl_pattern, self.data).group(1))
            self.data = self.load(mirror_page)

        hosts_data = {}
        for _a in re.findall(r'<a\s*class="btn btn-small disabled link host"(.+?)/a>', self.data, re.S):
            validity = re.search(r'validity=(.+)', _a).group(1)
            if validity in ("valid", "unknown"):
                host = re.search(r'nameHost="(.+?)"', _a).group(1)
                url = re.search(r'href="(.+?)"', _a).group(1)
                hosts_data[host] = url

        choosen_hosts = []
        # priority hosts goes first
        for _h in hosts_priority:
            if _h in hosts_data and _h not in ignored_hosts:
                self.log_debug(_("Adding '%s' link") % _h)
                choosen_hosts.append(_h)
                if not grab_all:
                    break

        # Now the hosts of the hosts
        if grab_all or (not grab_all and not choosen_hosts):
            for _h in hosts_data:
                if _h not in ignored_hosts and _h not in choosen_hosts:
                    self.log_debug(_("Adding '%s' link") % _h)
                    choosen_hosts.append(_h)
                    if not grab_all:
                        break

        return [hosts_data[_h] for _h in choosen_hosts]


<commit_msg>[MultiUpOrg] Typo<commit_after># -*- coding: utf-8 -*-

import re
import urlparse

from module.plugins.internal.SimpleCrypter import SimpleCrypter


class MultiUpOrg(SimpleCrypter):
    __name__    = "MultiUpOrg"
    __type__    = "crypter"
    __version__ = "0.10"
    __status__  = "testing"

    __pattern__ = r'http://(?:www\.)?multiup\.org/(en/|fr/)?(?P<TYPE>project|download|mirror)/\w+(/\w+)?'
    __config__  = [("activated"         , "bool"          , "Activated"                                        , True     ),
                   ("use_premium"       , "bool"          , "Use premium account if available"                 , True     ),
                   ("folder_per_package", "Default;Yes;No", "Create folder for each package"                   , "Default"),
                   ("max_wait"          , "int"           , "Reconnect if waiting time is greater than minutes", 10       ),
                   ("hosts_priority"    , "str"           , "Prefered hosts priority (bar-separated)"          , ""       ),
                   ("ignored_hosts"     , "str"           , "Ignored hosts list (bar-separated)"               , ""       ),
                   ("grab_all"          , "bool"          , "Grab all URLs (default only first match)"         , False    )]

    __description__ = """MultiUp.org crypter plugin"""
    __license__     = "GPLv3"
    __authors__     = [("<NAME>", "<EMAIL>")]


    NAME_PATTERN = r'<title>.*(?:Project|Projet|ownload|élécharger) (?P<N>.+?) (\(|- )'

    URL_REPLACEMENTS = [(r'http://(?:www\.)?multiup\.org/(en/|fr/)?', "http://www.multiup.org/en/")]


    def get_links(self):
        m_type = self.info['pattern']['TYPE']
        hosts_priority = [_h for _h in self.config.get('hosts_priority').split('|') if _h]
        ignored_hosts  = [_h for _h in self.config.get('ignored_hosts').split('|') if _h]
        grab_all       = self.config.get('grab_all')

        if m_type == "project":
            return re.findall(r'\n(http://www\.multiup\.org/(?:en|fr)/download/.*)', self.data)

        elif m_type == "download":
            dl_pattern = r'href="(.*)">.*\n.*<h5>DOWNLOAD</h5>'
            mirror_page = urlparse.urljoin("http://www.multiup.org/", re.search(dl_pattern, self.data).group(1))
            self.data = self.load(mirror_page)

        hosts_data = {}
        for _a in re.findall(r'<a\s*class="btn btn-small disabled link host"(.+?)/a>', self.data, re.S):
            validity = re.search(r'validity=(.+)', _a).group(1)
            if validity in ("valid", "unknown"):
                host = re.search(r'nameHost="(.+?)"', _a).group(1)
                url = re.search(r'href="(.+?)"', _a).group(1)
                hosts_data[host] = url

        chosen_hosts = []
        # priority hosts goes first
        for _h in hosts_priority:
            if _h in hosts_data and _h not in ignored_hosts:
                self.log_debug(_("Adding '%s' link") % _h)
                chosen_hosts.append(_h)
                if not grab_all:
                    break

        # Now the rest of the hosts
        if grab_all or (not grab_all and not chosen_hosts):
            for _h in hosts_data:
                if _h not in ignored_hosts and _h not in chosen_hosts:
                    self.log_debug(_("Adding '%s' link") % _h)
                    chosen_hosts.append(_h)
                    if not grab_all:
                        break

        return [hosts_data[_h] for _h in chosen_hosts]



{'fraction_non_alphanumeric': 0.15324562671678474, 'fraction_numerical': 0.003758854994940003, 'mean_word_length': 1.7672, 'pattern_counts': {'":': 4, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3501872', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2127, 'n_words': 457}