<commit_before>from bs4 import BeautifulSoup
from cum import output
from cum.config import config
from cum.scrapers.base import BaseChapter, BaseSeries
from urllib.parse import urljoin
import re
import requests

name_re = r'-? c([0-9-]+).*?(?: \[(.*)\])?\.'
fallback_re = r'\- (.*) (?:\[(.*)\])?'


class MadokamiSeries(BaseSeries):
    def __init__(self, url):
        r = requests.get(url)
        self.url = url
        self.soup = BeautifulSoup(r.text, config.html_parser)
        self.chapters = self.get_chapters()

    @property
    def name(self):
        return self.soup.find('span', class_='title').string

    def get_chapters(self):
        rows = (self.soup
                .find('table', class_='mobile-files-table')
                .find_all('tr'))
        chapters = []
        for row in rows[1:]:
            # If the Read link cannot be found in the current row, the row is
            # assumed to be a non-manga file uploaded to the directory and will
            # thus be skipped.
            if not row.find('a', text='Read'):
                continue

            link = row.find('a')

            url = urljoin(self.url, link.get('href'))

            name = link.string
            name_parts = re.search(name_re, name)
            if not name_parts:
                name_parts = re.search(fallback_re, name)
            chapter = name_parts.group(1)
            if name_parts.group(2):
                groups = name_parts.group(2).split('][')
            else:
                groups = []

            c = MadokamiChapter(name=self.name, alias=self.alias,
                                chapter=chapter, url=url, groups=groups)
            chapters.append(c)
        return chapters


class MadokamiChapter(BaseChapter):
    uses_pages = False

    def __init__(self, name=None, alias=None, chapter=None,
                 url=None, groups=[], title=None):
        self.name = name
        self.alias = alias
        self.chapter = chapter
        self.title = title
        self.url = url
        self.groups = groups

    def download(self):
        auth = requests.auth.HTTPBasicAuth(*config.madokami.login)
        r = requests.get(self.url, auth=auth, stream=True)<commit_msg>Skip Madokami files with retarded names.<commit_after>from bs4 import BeautifulSoup
from cum import output
from cum.config import config
from cum.scrapers.base import BaseChapter, BaseSeries
from urllib.parse import urljoin
import re
import requests

name_re = r'-? c([0-9-]+).*?(?: \[(.*)\])?\.'
fallback_re = r'\- (.*) (?:\[(.*)\])?'


class MadokamiSeries(BaseSeries):
    def __init__(self, url):
        r = requests.get(url)
        self.url = url
        self.soup = BeautifulSoup(r.text, config.html_parser)
        self.chapters = self.get_chapters()

    @property
    def name(self):
        return self.soup.find('span', class_='title').string

    def get_chapters(self):
        rows = (self.soup
                .find('table', class_='mobile-files-table')
                .find_all('tr'))
        chapters = []
        for row in rows[1:]:
            # If the Read link cannot be found in the current row, the row is
            # assumed to be a non-manga file uploaded to the directory and will
            # thus be skipped.
            if not row.find('a', text='Read'):
                continue

            link = row.find('a')

            url = urljoin(self.url, link.get('href'))

            name = link.string
            name_parts = re.search(name_re, name)
            if not name_parts:
                name_parts = re.search(fallback_re, name)
            try:
                chapter = name_parts.group(1)
            except:
                continue
            if name_parts.group(2):
                groups = name_parts.group(2).split('][')
            else:
                groups = []

            c = MadokamiChapter(name=self.name, alias=self.alias,
                                chapter=chapter, url=url, groups=groups)
            chapters.append(c)
        return chapters


class MadokamiChapter(BaseChapter):
    uses_pages = False

    def __init__(self, name=None, alias=None, chapter=None,
                 url=None, groups=[], title=None):
        self.name = name
        self.alias = alias
        self.chapter = chapter
        self.title = title
        self.url = url
        self.groups = groups

    def download(self):
        auth = requests.auth.HTTPBasicAuth(*config.madokami.login)
        r = requests.get(self.url, auth=auth, stream=True)
{'fraction_non_alphanumeric': 0.11943274983381343, 'fraction_numerical': 0.0031021493463328164, 'mean_word_length': 1.9197930142302717, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6166579', 'n_tokens_mistral': 1356, 'n_tokens_neox': 1291, 'n_words': 331}