<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# moehime / saimoe / data source - 2channel
#
# Copyright (C) 2012 <NAME> <<EMAIL>>
#
# This file is part of moehime.
#
# moehime is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# moehime is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with moehime.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals, division

import abc
from itertools import izip
from urllib import urlencode

from . import src_manager
from .base import DatasourceBase, URL_READ, URL_SEARCH
from ..net.fetch import ResourceRequester
from .s_2ch_thread import PostThreadInfo_2ch
from .s_2ch_post import Post_2ch

FIND_STRING = 'アニメ最萌トーナメント2012 board:投票所'


class Base2chDatasource(DatasourceBase, ResourceRequester):
    __metaclass__ = abc.ABCMeta

    THREAD_INFO_CLASS = PostThreadInfo_2ch
    POST_CLASS = Post_2ch

    def __init__(self):
        super(Base2chDatasource, self).__init__()

    @abc.abstractproperty
    def board_name(self):
        return b'www'

    @abc.abstractproperty
    def readcgi_url(self):
        return b''

    @abc.abstractproperty
    def post_list_selector(self):
        return b''

    @property
    def post_url_prefix(self):
        return b'http://%s.2ch.net/' % (self.board_name, )

    @property
    def search_url_prefix(self):
        return b'http://find.2ch.net/'

    def format_url(self, kind, *args):
        if kind == URL_READ:
            tid, start, end = args[:3]
            end_str = b'' if end is None else str(end)
            url = self.post_url_prefix + self.readcgi_url % {
                    'tid': tid,
                    'start': start,
                    'end': end_str,
                    }
        elif kind == URL_SEARCH:
            qs, max_count = args[:2]
            url = b''.join([
                    self.search_url_prefix,
                    b'?',
                    urlencode((
                        ('STR', self.to_str(kind, qs)),
                        ('COUNT', 10 if max_count is None else max_count),
                        ('TYPE', 'TITLE'),
                        ('BBS', 'ALL'),
                        )),
                        ])
        else:
            raise NotImplementedError

        return self.to_str(kind, url)

    def _do_get_thread_list(self, max_count):
        result_page = self.fetch_pq(URL_SEARCH, FIND_STRING, max_count)
        dl = result_page(b'.wrapper dl')
        dt = dl(b'dt')[:-1]
        dd = dl(b'dd')[:]
        if len(dt) != len(dd):
            raise RuntimeError('thread list structure imbalanced')
<commit_msg>make the cases of URL generation more elegant 使 URL 构造的几种情况更优雅<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# moehime / saimoe / data source - 2channel
#
# Copyright (C) 2012 <NAME> <<EMAIL>>
#
# This file is part of moehime.
#
# moehime is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# moehime is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with moehime.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals, division

import abc
from itertools import izip
from urllib import urlencode

from . import src_manager
from .base import DatasourceBase, URL_READ, URL_SEARCH
from ..net.fetch import ResourceRequester
from .s_2ch_thread import PostThreadInfo_2ch
from .s_2ch_post import Post_2ch

FIND_STRING = 'アニメ最萌トーナメント2012 board:投票所'


class Base2chDatasource(DatasourceBase, ResourceRequester):
    __metaclass__ = abc.ABCMeta

    THREAD_INFO_CLASS = PostThreadInfo_2ch
    POST_CLASS = Post_2ch

    def __init__(self):
        super(Base2chDatasource, self).__init__()

    @abc.abstractproperty
    def board_name(self):
        return b'www'

    @abc.abstractproperty
    def readcgi_url(self):
        return b''

    @abc.abstractproperty
    def post_list_selector(self):
        return b''

    @property
    def post_url_prefix(self):
        return b'http://%s.2ch.net/' % (self.board_name, )

    @property
    def search_url_prefix(self):
        return b'http://find.2ch.net/'

    def _format_url_read(self, tid, start, end):
        end_str = b'' if end is None else str(end)
        return self.post_url_prefix + self.readcgi_url % {
                'tid': tid,
                'start': start,
                'end': end_str,
                }

    def _format_url_search(self, qs, max_count=None):
        count = 10 if max_count is None else max_count
        return b''.join([
                self.search_url_prefix,
                b'?',
                urlencode((
                    ('STR', self.to_str(kind, qs)),
                    ('COUNT', count),
                    ('TYPE', 'TITLE'),
                    ('BBS', 'ALL'),
                    )),
                ])

    def format_url(self, kind, *args, **kwargs):
        if kind == URL_READ:
            url = self._format_url_read(*args, **kwargs)
        elif kind == URL_SEARCH:
            url = self._format_url_search(*args, **kwargs)
        else:
            raise NotImplementedError

        return self.to_str(kind, url)

    def _do_get_thread_list(self, max_count):
        result_page = self.fetch_pq(URL_SEARCH, FIND_STRING, max_count)
        dl = result_page(b'.wrapper dl')
        dt = dl(b'dt')[:-1]
        dd = dl(b'dd')[:]
        if len(dt) != len(dd):
            raise RuntimeError('thread list structure imbalanced')

{'fraction_non_alphanumeric': 0.09775031685678073, 'fraction_numerical': 0.007921419518377694, 'mean_word_length': 2.359765832889835, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2995755', 'n_tokens_mistral': 1993, 'n_tokens_neox': 1924, 'n_words': 577}