<commit_before># -*- coding: utf-8 -*-

import urlparse

from sbscraper.extract import base


class RedMartExtractor(base.Extractor):

    API_VERSION = 'v1.5.6'

    def __init__(self, url, categories=None, start_page=0, page_size=50):
        super(RedMartExtractor, self).__init__(url=url, categories=categories,
                                               start_page=start_page,
                                               page_size=page_size)

    def get_data(self, response):
        return response.data.get('products', [])

    def has_more(self, response):
        data = response.data
        if ('page' in data and
                'page_size' in data and
                'total' in data):
            page = data['page']
            page_size = data['page_size']
            total = data['total']
            return (page + 1) * page_size < total
        return False

    def request(self, category, page, page_size):
        url = urlparse.urljoin(self.url, "%s/catalog/search" % self.API_VERSION)
        parameters = {
            'category': category,
            'page': page,
            'pageSize': page_size
        }
        response = self.session.get(url, params=parameters)
        response.data = response.json()
        return response<commit_msg>fix how has_more is checked<commit_after># -*- coding: utf-8 -*-

import urlparse

from sbscraper.extract import base


class RedMartExtractor(base.Extractor):

    API_VERSION = 'v1.5.6'

    def __init__(self, url, categories=None, start_page=0, page_size=50):
        super(RedMartExtractor, self).__init__(url=url, categories=categories,
                                               start_page=start_page,
                                               page_size=page_size)

    def get_data(self, response):
        return response.data.get('products', [])

    def has_more(self, response):
        data = response.data
        if 'page' in data and 'total' in data:
            page = data['page']
            total = data['total']
            return (page + 1) * self.page_size < total
        return False

    def request(self, category, page, page_size):
        url = urlparse.urljoin(self.url, "%s/catalog/search" % self.API_VERSION)
        parameters = {
            'category': category,
            'page': page,
            'pageSize': page_size
        }
        response = self.session.get(url, params=parameters)
        response.data = response.json()
        return response
{'fraction_non_alphanumeric': 0.10810810810810811, 'fraction_numerical': 0.00645421540943929, 'mean_word_length': 1.9142185663924793, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1978630', 'n_tokens_mistral': 691, 'n_tokens_neox': 672, 'n_words': 161}