<commit_before># coding: utf-8

from collections import OrderedDict
from urlparse import urljoin
from urllib import urlencode
from scrapy.spider import Spider
from scrapy.selector import Selector
from ..items import Art, Result


BASE_URL = 'http://www.toranoana.jp/'
QUERY_URL = 'http://www.toranoana.jp/cgi-bin/R2/allsearch.cgi'


class Tora(Spider):

    name = 'tora'

    def __init__(self, query, *args, **kargs):
        Spider.__init__(self, *args, **kargs)
        self.start_urls = [makeuri(encode(query))]

    def parse(self, response):
        sel = Selector(response)
        trs = list(sel.xpath('//table[@class="FixFrame"]//tr'))

        def gen():
            for tr in trs[2:-1:2]:
                yield Art(
                    title=tr.xpath('td[@class="c1"]/a/text()').extract()[0],
                    author=tr.xpath('td[@class="c2"]/a/text()').extract()[0],
                    company=tr.xpath('td[@class="c3"]/a/text()').extract()[0],
                    uri=urljoin(BASE_URL, tr.xpath('td[@class="c1"]/a/@href').extract()[0]),
                    status='reserve' if u'予' in tr.xpath('td[@class="c7"]/text()').extract() else 'other',
                )

        return Result(arts=list(gen()))


def encode(query):
    return query.decode('utf-8') if isinstance(query, str) else query


def makeuri(query, start=0):
    return QUERY_URL + '?' + urlencode(OrderedDict([
        ('item_kind', '0401'),
        ('bl_fg', '0'),
        ('search', query.encode('Shift_JIS')),
        ('ps', start + 1),
    ]))
<commit_msg>add query uri in result<commit_after># coding: utf-8

from collections import OrderedDict
from urlparse import urljoin
from urllib import urlencode
from scrapy.spider import Spider
from scrapy.selector import Selector
from ..items import Art, Result


BASE_URL = 'http://www.toranoana.jp/'
QUERY_URL = 'http://www.toranoana.jp/cgi-bin/R2/allsearch.cgi'


class Tora(Spider):

    name = 'tora'

    def __init__(self, query, *args, **kargs):
        Spider.__init__(self, *args, **kargs)
        self.start_urls = [makeuri(encode(query))]

    def parse(self, response):
        sel = Selector(response)
        trs = list(sel.xpath('//table[@class="FixFrame"]//tr'))

        def gen():
            for tr in trs[2:-1:2]:
                yield Art(
                    title=tr.xpath('td[@class="c1"]/a/text()').extract()[0],
                    author=tr.xpath('td[@class="c2"]/a/text()').extract()[0],
                    company=tr.xpath('td[@class="c3"]/a/text()').extract()[0],
                    uri=urljoin(BASE_URL, tr.xpath('td[@class="c1"]/a/@href').extract()[0]),
                    status='reserve' if u'予' in tr.xpath('td[@class="c7"]/text()').extract() else 'other',
                )

        return Result(
            uri=self.start_urls[0],
            arts=list(gen())
        )


def encode(query):
    return query.decode('utf-8') if isinstance(query, str) else query


def makeuri(query, start=0):
    return QUERY_URL + '?' + urlencode(OrderedDict([
        ('item_kind', '0401'),
        ('bl_fg', '0'),
        ('search', query.encode('Shift_JIS')),
        ('ps', start + 1),
    ]))

{'fraction_non_alphanumeric': 0.18144395186827106, 'fraction_numerical': 0.014249525015832806, 'mean_word_length': 2.5694915254237287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5257758', 'n_tokens_mistral': 1088, 'n_tokens_neox': 1038, 'n_words': 204}