<commit_before>from tools.url import UrlMgr
from tools.helper import textextract, textextractall
import re
import os

import logging
log = logging.getLogger('pyload')

import imp

class Pyload(object):

    def getFiles(self, dirs):
        latest=self.latest
        url = UrlMgr(url='https://bitbucket.org/spoob/pyload/src/'+latest+'/'+('/'.join(dirs)))
        items = textextractall(url.data, '<td class="name filename"><a href="/spoob/pyload/src/'+latest+'/'+('/'.join(dirs))+"/", '"')
        basePath = 'pyload'
        for i in dirs:
            basePath = os.path.join(basePath, i)
        if os.path.isdir(basePath) is False:
            os.makedirs(basePath)
        for i in items:
            url = UrlMgr(url='https://bitbucket.org/spoob/pyload/raw/'+latest+'/'+('/'.join(dirs))+"/"+i)
            data = url.data
            data = data.replace('  self.pyfile.name = ', '  pass # flashget remove: self.pyfile.name = ')
            data = data.replace('  pyfile.name = ', '  pass # flashget remove: pyfile.name = ')
            data = data.replace('from UploadkingCom', 'from module.plugins.hoster.UploadkingCom')
            data = data.replace('from interaction.', 'from module.interaction.')
            data = re.sub('from module\.', 'from pyload.module.', data)
            path = os.path.join(basePath, i)
            open(path, "w").write(data)
            yield (i, data,path)

    def run(self):
        url = UrlMgr(url='https://bitbucket.org/spoob/pyload/changesets')
        self.latest = textextract(url.data, '<li><a href="/spoob/pyload/src/', '"')


        path = os.path.join("pyload", '__init__.py')
        open(path, "w").write(" ")

        for folder in ['plugins', 'network', 'common', 'interaction']:
            dirs = ['module', folder]
            for i,data,path in self.getFiles(dirs):
                pass

        for folder in ['internal', 'hoster']:
            dirs = ['module', 'plugins', folder]
            for i,data,path in self.getFiles(dirs):
                pass
        dirs = ['module']
        for i,data,path in self.getFiles(dirs):
            pass

        dirs = ['module', 'plugins', 'hoster']
        for i,data,path in self.getFiles(dirs):
            if i == '__init__.py':
                continue
            mod = imp.load_source(i[0][:-3], path)
            yield mod<commit_msg>don't download __init__ files since they are empty and won't get cached<commit_after>from tools.url import UrlMgr
from tools.helper import textextract, textextractall
import re
import os

import logging
log = logging.getLogger('pyload')

import imp

class Pyload(object):

    def getFiles(self, dirs):
        latest=self.latest
        url = UrlMgr(url='https://bitbucket.org/spoob/pyload/src/'+latest+'/'+('/'.join(dirs)))
        items = textextractall(url.data, '<td class="name filename"><a href="/spoob/pyload/src/'+latest+'/'+('/'.join(dirs))+"/", '"')
        basePath = 'pyload'
        for i in dirs:
            basePath = os.path.join(basePath, i)
        if os.path.isdir(basePath) is False:
            os.makedirs(basePath)
        open(os.path.join(basePath, '__init__.py'), "w").write(" ")
        for i in items:
            if i == '__init__.py':
                continue
            url = UrlMgr(url='https://bitbucket.org/spoob/pyload/raw/'+latest+'/'+('/'.join(dirs))+"/"+i)
            data = url.data
            data = data.replace('  self.pyfile.name = ', '  pass # flashget remove: self.pyfile.name = ')
            data = data.replace('  pyfile.name = ', '  pass # flashget remove: pyfile.name = ')
            data = data.replace('from UploadkingCom', 'from module.plugins.hoster.UploadkingCom')
            data = data.replace('from interaction.', 'from module.interaction.')
            data = re.sub('from module\.', 'from pyload.module.', data)
            path = os.path.join(basePath, i)
            open(path, "w").write(data)
            yield (i, data,path)

    def run(self):
        url = UrlMgr(url='https://bitbucket.org/spoob/pyload/changesets')
        self.latest = textextract(url.data, '<li><a href="/spoob/pyload/src/', '"')


        path = os.path.join("pyload", '__init__.py')
        open(path, "w").write(" ")

        for folder in ['plugins', 'network', 'common', 'interaction']:
            dirs = ['module', folder]
            for i,data,path in self.getFiles(dirs):
                pass

        for folder in ['internal', 'hoster']:
            dirs = ['module', 'plugins', folder]
            for i,data,path in self.getFiles(dirs):
                pass
        dirs = ['module']
        for i,data,path in self.getFiles(dirs):
            pass

        dirs = ['module', 'plugins', 'hoster']
        for i,data,path in self.getFiles(dirs):
            mod = imp.load_source(i[0][:-3], path)
            yield mod
{'fraction_non_alphanumeric': 0.1632822213012847, 'fraction_numerical': 0.0008288437629506838, 'mean_word_length': 2.2526954177897576, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5726363', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1548, 'n_words': 331}