<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from time import sleep

from pyload.Api import LinkStatus, DownloadStatus as DS, ProgressInfo, ProgressType
from pyload.utils import uniqify, accumulate
from pyload.plugins.Base import Abort, Retry
from pyload.plugins.Crypter import Package

from BaseThread import BaseThread


class DecrypterThread(BaseThread):
    """thread for decrypting"""

    def __init__(self, manager, data, fid, pid, owner):
        BaseThread.__init__(self, manager, owner)
        # [... (plugin, url) ...]
        self.data = data
        self.fid = fid
        self.pid = pid
        # holds the progress, while running
        self.progress = None
        # holds if an error happened
        self.error = False

        self.start()

    def getProgress(self):
        return self.progress

    def run(self):
        pack = self.m.core.files.getPackage(self.pid)
        links, packages = self.decrypt(accumulate(self.data), pack.password)

        if links:
            self.log.info(
                _("Decrypted %(count)d links into package %(name)s") % {"count": len(links), "name": pack.name})
            self.m.core.api.addLinks(self.pid, [l.url for l in links])

        # TODO: add single package into this one and rename it?
        # TODO: nested packages
        for p in packages:
            self.m.core.api.addPackage(p.name, p.getURLs(), pack.password)

        self.m.core.api.files.setDownloadStatus(self.fid, "finished" if not self.error else "failed")
        self.m.done(self)

    def decrypt(self, plugin_map, password=None, err=False):
        result = []

        self.progress = ProgressInfo("BasePlugin", "",  _("decrypting"),
                                         0, 0, len(self.data), self.owner, ProgressType.Decrypting)
        # TODO QUEUE_DECRYPT
        for name, urls in plugin_map.iteritems():
            klass = self.m.core.pluginManager.loadClass("crypter", name)
            plugin = None
            plugin_result = []

            # updating progress
            self.progress.plugin = name
            self.progress.name = _("Decrypting %s links") % len(urls) if len(urls) > 1 else urls[0]

            #TODO: dependency check, there is a new error code for this
            # TODO: decrypting with result yielding
            if not klass:
                if err:
                    plugin_result.extend(LinkStatus(url, url, -1, DS.NotPossible, name) for url in urls)
                self.log.debug("Plugin for decrypting was not loaded")
            else:
                try:
                    plugin = klass(self.m.core, password)
<commit_msg>additional fix #4<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from time import sleep

from pyload.Api import LinkStatus, DownloadStatus as DS, ProgressInfo, ProgressType
from pyload.utils import uniqify, accumulate
from pyload.plugins.Base import Abort, Retry
from pyload.plugins.Crypter import Package

from BaseThread import BaseThread


class DecrypterThread(BaseThread):
    """thread for decrypting"""

    def __init__(self, manager, data, fid, pid, owner):
        BaseThread.__init__(self, manager, owner)
        # [... (plugin, url) ...]
        self.data = data
        self.fid = fid
        self.pid = pid
        # holds the progress, while running
        self.progress = None
        # holds if an error happened
        self.error = False

        self.start()

    def getProgress(self):
        return self.progress

    def run(self):
        pack = self.m.core.files.getPackage(self.pid)
        links, packages = self.decrypt(accumulate(self.data), pack.password)

        if links:
            self.log.info(
                _("Decrypted %(count)d links into package %(name)s") % {"count": len(links), "name": pack.name})
            self.m.core.api.addLinks(self.pid, [l.url for l in links])

        # TODO: add single package into this one and rename it?
        # TODO: nested packages
        for p in packages:
            self.m.core.api.addPackage(p.name, p.getURLs(), pack.password)

        self.m.core.files.setDownloadStatus(self.fid, "finished" if not self.error else "failed")
        self.m.done(self)

    def decrypt(self, plugin_map, password=None, err=False):
        result = []

        self.progress = ProgressInfo("BasePlugin", "",  _("decrypting"),
                                         0, 0, len(self.data), self.owner, ProgressType.Decrypting)
        # TODO QUEUE_DECRYPT
        for name, urls in plugin_map.iteritems():
            klass = self.m.core.pluginManager.loadClass("crypter", name)
            plugin = None
            plugin_result = []

            # updating progress
            self.progress.plugin = name
            self.progress.name = _("Decrypting %s links") % len(urls) if len(urls) > 1 else urls[0]

            #TODO: dependency check, there is a new error code for this
            # TODO: decrypting with result yielding
            if not klass:
                self.error = True
                if err:
                    plugin_result.extend(LinkStatus(url, url, -1, DS.NotPossible, name) for url in urls)
                self.log.debug("Plugin '%s' for decrypting was not loaded" % plugin)
            else:
                try:
                    plugin = klass(self.m.core, password)

{'fraction_non_alphanumeric': 0.10840210052513129, 'fraction_numerical': 0.0024381095273818456, 'mean_word_length': 2.0880138969310944, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1665997', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1512, 'n_words': 434}