<commit_before>from tools.page import Page
from tools.extension import Extension
from tools.url import UrlMgr
from tools.helper import textextract, textextractall

class GenxAnime(Page):
    def __init__(self):
        self.name = 'genx-anime'
        self.url = 'http://www.genx-anime.org'
        self.cookies = {}
        Page.__init__(self)

    def getAllPages(self, link):
        return []

    def extract(self, link):
        if not self.beforeExtract():
            return None
        url = UrlMgr(url=link, cookies=self.cookies)

        name = textextract(url.data, '<h2>&raquo; ', ' &laquo;</h2>')
        media = Page.getMedia(self, name, link)
        if not media:
            return None
        media.origUrl = textextract(url.data, 'on <a style="color:#BBB;" href="', '"')
        '''
<td><b>Folge 2: Lektion 2

-> 'id="folge_2"', '</div>' -> 'go&link=', '"'
-> 'id="download_2"', '</div>' -> 'go&link=', '"'

                <td colspan="2">

                    <div id="folge_2" style="display: none;">
                       <a target="_blank" href="index.php?do=go&link=aHR0cDovL3d3dy5kaXZ4c3RhZ2UubmV0L3ZpZGVvL3JvNDlnZGVobmlpMG0="><img
                       name="play" src="images/hoster/StandartP.png"></a><a target="_blank"
                       href="index.php?do=go&link=aHR0cDovL3N0cmVhbWNsb3VkLmV1LzFqNXRtNDExOHlzcy9nZW5YLUFuaW1lLm9yZ19HLlQuT19fTGVrdGlvbl8wMi54dmlkLmdlci1zdWIuQlMtQUUuYXZpLmh0bWw="><img
                       name="play" src="images/hoster/StreamCloud.eu.png"></a>                    </div>
                    <div id="download_2" style="display: none;">
                    <a target="_blank" href="index.php?do=go&link=aHR0cDovL3d3dy5zaGFyZS1vbmxpbmUuYml6L2RsL0VDU0pSODVNSkRY"><img
                    src="images/hoster/Share-Online.biz.png"></a><a target="_blank"
                    href="index.php?do=go&link=aHR0cDovL3VsLnRvL2hxbm0ydjlv"><img src="images/hoster/Uploaded.to.png"></a><a target="_blank"
                    href="index.php?do=go&link=aHR0cDovL3VwbG9hZGVkLnRvL2ZpbGUvM3E1enZqNGU="><img src="images/hoster/StandartD.png"></a>
                    </div>
                </td>
                '''
        pageData = url.data
        for number in textextractall(pageData, '<td><b>Folge ', ':'):
            part = media.createSub()
            part.num = number
            part.name = textextract(pageData, '<td><b>Folge '+number+": ", "</td>")
            part.name = part.name.rstrip()
            if part.name:
                part.name = part.name
            else:
                print part.num
            streamArea = textextract(pageData, 'id="folge_'+number+'"', '</div>')
            for link in textextractall(streamArea, 'go&link=', '"'):
                alternative = part.createSub()
                alternativePart = alternative.createSub()
                alternativePart.url = link.decode('base64')
                self.setPinfo(alternativePart)
            dlArea = textextract(pageData, 'id="download_'+number+'"', '</div>')
            for link in textextractall(dlArea, 'go&link=', '"'):
                alternative = part.createSub()
                alternativePart = alternative.createSub()
                alternativePart.url = link.decode('base64')
                self.setPinfo(alternativePart)


        return media




baseRegex = '^(http://)?(www\.)?genx-anime\.org'
class SingleGenxAnimeExtension(GenxAnime, Extension):
    eregex = baseRegex+'/index\.php\?do=show_download.*'
    ename = 'genxanime_s'
    def get(self, link):
        return GenxAnime.extract(self, link)<commit_msg>fixed genx for new layout<commit_after>from tools.page import Page
from tools.extension import Extension
from tools.url import UrlMgr
from tools.helper import textextract, textextractall

class GenxAnime(Page):
    def __init__(self):
        self.name = 'genx-anime'
        self.url = 'http://www.genx-anime.org'
        self.cookies = {}
        Page.__init__(self)

    def getAllPages(self, link):
        return []

    def extract(self, link):
        if not self.beforeExtract():
            return None
        url = UrlMgr(url=link, cookies=self.cookies)

        name = textextract(url.data, '<h2>', ')</h2>')+')'
        name = textextract(name, '| ', '')
        media = Page.getMedia(self, name, link)
        if not media:
            return None
        '''
<td><b>Folge 2: Lektion 2

-> 'id="folge_2"', '</div>' -> 'go&link=', '"'
-> 'id="download_2"', '</div>' -> 'go&link=', '"'

                <td colspan="2">

                    <div id="folge_2" style="display: none;">
                       <a target="_blank" href="index.php?do=go&link=aHR0cDovL3d3dy5kaXZ4c3RhZ2UubmV0L3ZpZGVvL3JvNDlnZGVobmlpMG0="><img
                       name="play" src="images/hoster/StandartP.png"></a><a target="_blank"
                       href="index.php?do=go&link=aHR0cDovL3N0cmVhbWNsb3VkLmV1LzFqNXRtNDExOHlzcy9nZW5YLUFuaW1lLm9yZ19HLlQuT19fTGVrdGlvbl8wMi54dmlkLmdlci1zdWIuQlMtQUUuYXZpLmh0bWw="><img
                       name="play" src="images/hoster/StreamCloud.eu.png"></a>                    </div>
                    <div id="download_2" style="display: none;">
                    <a target="_blank" href="index.php?do=go&link=aHR0cDovL3d3dy5zaGFyZS1vbmxpbmUuYml6L2RsL0VDU0pSODVNSkRY"><img
                    src="images/hoster/Share-Online.biz.png"></a><a target="_blank"
                    href="index.php?do=go&link=aHR0cDovL3VsLnRvL2hxbm0ydjlv"><img src="images/hoster/Uploaded.to.png"></a><a target="_blank"
                    href="index.php?do=go&link=aHR0cDovL3VwbG9hZGVkLnRvL2ZpbGUvM3E1enZqNGU="><img src="images/hoster/StandartD.png"></a>
                    </div>
                </td>
                '''
        pageData = url.data
        for partName in textextractall(pageData, '<dt class="open" style="width:100%;">', '</dt>'):
            partName = partName.strip()
            number = textextract(partName, 'Folge ', ' ')
            part = media.createSub()
            part.num = number
            part.name = textextract(partName, ' | ', '')
            if part.name:
                part.name = part.name
            else:
                print part.num
            streamArea = textextract(pageData, partName, '<dt style="width:100%;font-weight:bold;border-bottom:1px solid #D1D1D1;">Download</dt>')
            for link in textextractall(streamArea, 'do=fwd&link=', '"'):
                alternative = part.createSub()
                alternativePart = alternative.createSub()
                alternativePart.url = link.decode('base64')
                self.setPinfo(alternativePart)
        return media




baseRegex = '^(http://)?(www\.)?genx-anime\.org'
class SingleGenxAnimeExtension(GenxAnime, Extension):
    eregex = baseRegex+'/index\.php\?do=display.*'
    ename = 'genxanime_s'
    def get(self, link):
        return GenxAnime.extract(self, link)
{'fraction_non_alphanumeric': 0.1463167053364269, 'fraction_numerical': 0.018851508120649653, 'mean_word_length': 2.333494441759304, 'pattern_counts': {'":': 1, '<': 69, '<?xml version=': 0, '>': 76, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2599639', 'n_tokens_mistral': 2494, 'n_tokens_neox': 2378, 'n_words': 343}