<commit_before># coding=utf-8
from __future__ import print_function, unicode_literals

import glob
import os
import re
import time
from collections import defaultdict
from itertools import chain

import requests
import tqdm

from bgmi.config import MAX_PAGE, BGMI_SAVE_PATH, IS_PYTHON3
from bgmi.models import Bangumi, Filter, Subtitle, STATUS_FOLLOWED, STATUS_UPDATED, Followed
from bgmi.script import ScriptRunner
from bgmi.utils import (parse_episode, print_warning, print_info,
                        test_connection, normalize_path)

if IS_PYTHON3:
    _unicode = str
else:
    _unicode = unicode


class BaseWebsite(object):
    cover_url = ''
    parse_episode = staticmethod(parse_episode)

    def search(self, keyword='', count=1, filter_=None):
        if not filter_:
            filter_ = '(.*)'
        match_title = re.compile(filter_)
        result = self.raw_search(keyword, count)
        # filter
        filtered_result = []
        for episode in result:
            if match_title.match(episode['title']):
                filtered_result.append(episode)
        result = filtered_result[:]

        # remove duplicated episode in result
        ret = []
        episodes = list({i['episode'] for i in result})
        for i in result:
            if i['episode'] in episodes:
                ret.append(i)
                del episodes[episodes.index(i['episode'])]

        if os.environ.get('DEBUG', None):
            for i in ret:
                print(i['title'], i['download'])
        return ret

    def raw_search(self, keyword='', count=1, ):
        return self.search_by_keyword(keyword, count)

    @staticmethod
    def save_data(data):
        b = Bangumi(**data)
        b.save()

    def fetch(self, save=False, group_by_weekday=True):
        bangumi_result, subtitle_group_result = self.fetch_bangumi_calendar_and_subtitle_group()
        if subtitle_group_result:
            for subtitle_group in subtitle_group_result:
                s = Subtitle(id=_unicode(subtitle_group['id']), name=_unicode(subtitle_group['name']))
                if not s.select():
                    s.save()
        if not bangumi_result:
            print('no result return None')
            return []

        if save:
            for bangumi in bangumi_result:
                self.save_data(bangumi)
        if group_by_weekday:
            result_group_by_weekday = defaultdict(list)
            for bangumi in bangumi_result:
                result_group_by_weekday[bangumi['update_time'].lower()].append(bangumi)
            bangumi_result = result_group_by_weekday
        return bangumi_result

    @staticmethod
    def followed_bangumi():
        weekly_list_followed = Bangumi.get_all_bangumi(status=STATUS_FOLLOWED)
        weekly_list_updated = Bangumi.get_all_bangumi(status=STATUS_UPDATED)
        weekly_list = defaultdict(list)
        for k, v in chain(weekly_list_followed.items(), weekly_list_updated.items()):
            weekly_list[k].extend(v)
        for bangumi_list in weekly_list.values():
            for bangumi in bangumi_list:
                bangumi['subtitle_group'] = [{'name': x['name'], 'id': x['id']}
                                             for x in Subtitle.get_subtitle(bangumi['subtitle_group'].split(', '))]
        return weekly_list

    def bangumi_calendar(self, force_update=False, save=True, cover=False):
        if force_update and not test_connection():
            force_update = False
            print_warning('network is unreachable')

        if force_update:
            print_info('fetching bangumi info ...')
            Bangumi.delete_all()
            weekly_list = self.fetch(save=save)
        else:
            weekly_list = Bangumi.get_all_bangumi()
        if not weekly_list:
            print_warning('warning: no bangumi schedule, fetching ...')
            weekly_list = self.fetch(save=save)

        runner = ScriptRunner()
        patch_list = runner.get_models_dict()
        for i in patch_list:
            weekly_list[i['update_time'].lower()].append(i)

        result = deepcopy(weekly_list)
        spacial_append_chars = ['Ⅱ', 'Ⅲ', '♪', 'Δ', '×', '☆', 'é', '·', '♭']
        spacial_remove_chars = []

        for index, weekday in enumerate(weekday_order):
            if weekly_list[weekday.lower()]:
                print('%s%s. %s' % (GREEN,
                                    weekday if not today else 'Bangumi Schedule for Today (%s)' % weekday, COLOR_END),
                      end='')
                if not followed:
                    print()
                    print_line()

                for i, bangumi in enumerate(weekly_list[weekday.lower()]):
                    if bangumi['status'] in (STATUS_UPDATED, STATUS_FOLLOWED) and 'episode' in bangumi:
                        bangumi['name'] = '%s(%d)' % (bangumi['name'], bangumi['episode'])

                    half = len(re.findall('[%s]' % string.printable, bangumi['name']))
                    full = (len(bangumi['name']) - half)
                    space_count = col - 2 - (full * 2 + half)

                    for s in spacial_append_chars:
                        if s in bangumi['name']:
                            space_count += 1

                    for s in spacial_remove_chars:
                        if s in bangumi['name']:
                            space_count -= 1

                    if bangumi['status'] == STATUS_FOLLOWED:
                        bangumi['name'] = '%s%s%s' % (YELLOW, bangumi['name'], COLOR_END)

                    if bangumi['status'] == STATUS_UPDATED:
                        bangumi['name'] = '%s%s%s' % (GREEN, bangumi['name'], COLOR_END)

                    if followed:
                        if i > 0:
                            print(' ' * 5, end='')
                        f = map(lambda x: x['name'], Subtitle.get_subtitle(bangumi['subtitle_group'].split(', ')))
                        print(bangumi['name'], ', '.join(f))
                    else:
                        print(' ' + bangumi['name'], ' ' * space_count, end='')
                        if (i + 1) % row == 0 or i + 1 == len(weekly_list[weekday.lower()]):
                            print()

                if not followed:
                    print()

        # for web api
        r = result.copy()
        for day, value in result.items():
            for index, bangumi in enumerate(value):
                if isinstance(bangumi['subtitle_group'], list):
                    subtitle_group = list(map(lambda x: {'name': x['name'], 'id': x['id']},
                                              Subtitle.get_subtitle_by_id(
                                                  bangumi['subtitle_group'])))
                else:
                    subtitle_group = list(map(lambda x: {'name': x['name'], 'id': x['id']},
                                              Subtitle.get_subtitle_by_id(
                                                  bangumi['subtitle_group'].split(', ' ''))))

                r[day][index]['subtitle_group'] = subtitle_group

        # download cover to local
        cover_to_be_download = []
        for daily_bangumi in result.values():
            for bangumi in daily_bangumi:
                followed_obj = Followed(bangumi_name=bangumi['name'])
                if followed_obj:
                    bangumi['status'] = followed_obj.status
                _, file_path, _ = self.convert_cover_to_path(bangumi['cover'])

                if not glob.glob(file_path):
                    cover_to_be_download.append(bangumi['cover'])


            for daily_bangumi in weekly_list.values():
                for bangumi in daily_bangumi:
                    followed_obj = Followed(bangumi_name=bangumi['name'])
                    if followed_obj:
                        bangumi['status'] = followed_obj.status
                    _, file_path, _ = self.convert_cover_to_path(bangumi['cover'])
                    bangumi['cover'] = normalize_path(bangumi['cover'])
                    if not glob.glob(file_path):
                        cover_to_be_download.append(bangumi['cover'])

            if os.environ.get('TRAVIS_CI'):
                cover_to_be_download = []

            if cover_to_be_download:
                print_info('updating cover')
                for cover in tqdm.tqdm(cover_to_be_download):
                    self.download_cover(cover)
        return weekly_list

    def convert_cover_to_path(self, cover_url):
        """
        convert bangumi cover to file path

        :param cover_url: bangumi cover path
        :type cover_url:str
        :rtype: str,str,str
        :return:file_path, dir_path, url
        """
        if cover_url.startswith('https://') or cover_url.startswith('http://'):
            url = cover_url
        else:
            url = '{}/{}'.format(self.cover_url, cover_url)

        cover_url = normalize_path(cover_url)
        file_path = os.path.join(BGMI_SAVE_PATH, 'cover')
        file_path = os.path.join(file_path, cover_url)
        dir_path = os.path.dirname(file_path)

        return dir_path, file_path, url

    def download_cover(self, cover_url):
        """
        :type cover_url:str
        :param cover_url:
        :return: None
        """
        dir_path, file_path, url = self.convert_cover_to_path(cover_url)

        if not glob.glob(dir_path):
            os.makedirs(dir_path)

        if os.environ.get('DEV', False):
            url = url.replace('https://', 'http://localhost:8092/https/')
            url = url.replace('http://', 'http://localhost:8092/http/')

        r = requests.get(url)
        with open(file_path, 'wb+') as f:
            f.write(r.content)

    def get_maximum_episode(self, bangumi, subtitle=True, ignore_old_row=True, max_page=MAX_PAGE):
        followed_filter_obj = Filter(bangumi_name=bangumi.name)
        followed_filter_obj.select_obj()

        subtitle_group = followed_filter_obj.subtitle if followed_filter_obj and subtitle else None
        include = followed_filter_obj.include if followed_filter_obj and subtitle else None
        exclude = followed_filter_obj.exclude if followed_filter_obj and subtitle else None
        regex = followed_filter_obj.regex if followed_filter_obj and subtitle else None

        data = [i for i in self.fetch_episode(_id=bangumi.keyword, name=bangumi.name,
                                              subtitle_group=subtitle_group,
                                              include=include, exclude=exclude, regex=regex, max=max_page)
                if i['episode'] is not None]
<commit_msg>fix bug #10<commit_after># coding=utf-8
from __future__ import print_function, unicode_literals

import glob
import os
import re
import time
from collections import defaultdict
from itertools import chain

import requests
import tqdm

from bgmi.config import MAX_PAGE, BGMI_SAVE_PATH, IS_PYTHON3
from bgmi.models import Bangumi, Filter, Subtitle, STATUS_FOLLOWED, STATUS_UPDATED, Followed
from bgmi.script import ScriptRunner
from bgmi.utils import (parse_episode, print_warning, print_info,
                        test_connection, normalize_path)

if IS_PYTHON3:
    _unicode = str
else:
    _unicode = unicode


class BaseWebsite(object):
    cover_url = ''
    parse_episode = staticmethod(parse_episode)

    def search(self, keyword='', count=1, filter_=None):
        if not filter_:
            filter_ = '(.*)'
        match_title = re.compile(filter_)
        result = self.raw_search(keyword, count)
        # filter
        filtered_result = []
        for episode in result:
            if match_title.match(episode['title']):
                filtered_result.append(episode)
        result = filtered_result[:]

        # remove duplicated episode in result
        ret = []
        episodes = list({i['episode'] for i in result})
        for i in result:
            if i['episode'] in episodes:
                ret.append(i)
                del episodes[episodes.index(i['episode'])]

        if os.environ.get('DEBUG', None):
            for i in ret:
                print(i['title'], i['download'])
        return ret

    def raw_search(self, keyword='', count=1, ):
        return self.search_by_keyword(keyword, count)

    @staticmethod
    def save_data(data):
        b = Bangumi(**data)
        b.save()

    def fetch(self, save=False, group_by_weekday=True):
        bangumi_result, subtitle_group_result = self.fetch_bangumi_calendar_and_subtitle_group()
        if subtitle_group_result:
            for subtitle_group in subtitle_group_result:
                s = Subtitle(id=_unicode(subtitle_group['id']), name=_unicode(subtitle_group['name']))
                if not s.select():
                    s.save()
        if not bangumi_result:
            print('no result return None')
            return []

        if save:
            for bangumi in bangumi_result:
                self.save_data(bangumi)
        if group_by_weekday:
            result_group_by_weekday = defaultdict(list)
            for bangumi in bangumi_result:
                result_group_by_weekday[bangumi['update_time'].lower()].append(bangumi)
            bangumi_result = result_group_by_weekday
        return bangumi_result

    @staticmethod
    def followed_bangumi():
        weekly_list_followed = Bangumi.get_all_bangumi(status=STATUS_FOLLOWED)
        weekly_list_updated = Bangumi.get_all_bangumi(status=STATUS_UPDATED)
        weekly_list = defaultdict(list)
        for k, v in chain(weekly_list_followed.items(), weekly_list_updated.items()):
            weekly_list[k].extend(v)
        for bangumi_list in weekly_list.values():
            for bangumi in bangumi_list:
                bangumi['subtitle_group'] = [{'name': x['name'], 'id': x['id']}
                                             for x in Subtitle.get_subtitle(bangumi['subtitle_group'].split(', '))]
        return weekly_list

    def bangumi_calendar(self, force_update=False, save=True, cover=False):
        if force_update and not test_connection():
            force_update = False
            print_warning('network is unreachable')

        if force_update:
            print_info('fetching bangumi info ...')
            Bangumi.delete_all()
            weekly_list = self.fetch(save=save)
        else:
            weekly_list = Bangumi.get_all_bangumi()
        if not weekly_list:
            print_warning('warning: no bangumi schedule, fetching ...')
            weekly_list = self.fetch(save=save)

        runner = ScriptRunner()
        patch_list = runner.get_models_dict()
        for i in patch_list:
            weekly_list[i['update_time'].lower()].append(i)
        if cover:
            # download cover to local
            cover_to_be_download = []
            for daily_bangumi in weekly_list.values():
                for bangumi in daily_bangumi:
                    followed_obj = Followed(bangumi_name=bangumi['name'])
                    if followed_obj:
                        bangumi['status'] = followed_obj.status
                    _, file_path, _ = self.convert_cover_to_path(bangumi['cover'])

                    if not glob.glob(file_path):
                        cover_to_be_download.append(bangumi['cover'])

            if os.environ.get('TRAVIS_CI'):
                cover_to_be_download = []

            if cover_to_be_download:
                print_info('updating cover')
                for cover in tqdm.tqdm(cover_to_be_download):
                    self.download_cover(cover)
        return weekly_list

    def convert_cover_to_path(self, cover_url):
        """
        convert bangumi cover to file path

        :param cover_url: bangumi cover path
        :type cover_url:str
        :rtype: str,str,str
        :return:file_path, dir_path, url
        """
        if cover_url.startswith('https://') or cover_url.startswith('http://'):
            url = cover_url
        else:
            url = '{}/{}'.format(self.cover_url, cover_url)

        cover_url = normalize_path(cover_url)
        file_path = os.path.join(BGMI_SAVE_PATH, 'cover')
        file_path = os.path.join(file_path, cover_url)
        dir_path = os.path.dirname(file_path)

        return dir_path, file_path, url

    def download_cover(self, cover_url):
        """
        :type cover_url:str
        :param cover_url:
        :return: None
        """
        dir_path, file_path, url = self.convert_cover_to_path(cover_url)

        if not glob.glob(dir_path):
            os.makedirs(dir_path)
        if os.environ.get('DEV', False):
            url = url.replace('https://', 'http://localhost:8092/https/')
            url = url.replace('http://', 'http://localhost:8092/http/')

        r = requests.get(url)
        with open(file_path, 'wb+') as f:
            f.write(r.content)

    def get_maximum_episode(self, bangumi, subtitle=True, ignore_old_row=True, max_page=MAX_PAGE):
        followed_filter_obj = Filter(bangumi_name=bangumi.name)
        followed_filter_obj.select_obj()

        subtitle_group = followed_filter_obj.subtitle if followed_filter_obj and subtitle else None
        include = followed_filter_obj.include if followed_filter_obj and subtitle else None
        exclude = followed_filter_obj.exclude if followed_filter_obj and subtitle else None
        regex = followed_filter_obj.regex if followed_filter_obj and subtitle else None

        data = [i for i in self.fetch_episode(_id=bangumi.keyword, name=bangumi.name,
                                              subtitle_group=subtitle_group,
                                              include=include, exclude=exclude, regex=regex, max=max_page)
                if i['episode'] is not None]

{'fraction_non_alphanumeric': 0.10115150146759991, 'fraction_numerical': 0.00208850756378415, 'mean_word_length': 2.0013552430967305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4697939', 'n_tokens_mistral': 5297, 'n_tokens_neox': 5045, 'n_words': 1091}