<commit_before># -*- coding: utf-8 -*-
from scrapy.contrib.linkextractors import LinkExtractor
from scrapy.contrib.spiders import CrawlSpider, Rule

from gscholar.items import GscholarItem
import re

class GscholarSpider(CrawlSpider):
    name = "gscholar"
    allowed_domains = ["scholar.google.com"]
    start_urls = [
        "https://scholar.google.com/citations?user=Zmwh_JAAAAAJ&hl=en",
    ]

    rules = [
        Rule(LinkExtractor(allow=r'view_op=list_colleagues', process_value='process_value'),
            callback='parse_colleagues', follow=True),
        #Rule(LinkExtractor(allow=r'citations\?user='),
        #    follow=True)
    ]

    def process_value(self, value):
        self.log('[SASHA]: Hi, this is an item page! %s' % value)
        if "citations?user=" in value:
            self.log('[SASHA]: Hi, this is an item page! %s' % value)
            return "value" + "&view_op=list_colleagues"
        else:
            return None
    #
    def process_links(self, links):
        for link in links:
            link.url = "%s&view_op=list_colleagues" % link.url
        #
        #print links
        return links
    #

    def parse_x(self, response):<commit_msg>last commit before spider rewrite<commit_after># -*- coding: utf-8 -*-
from scrapy.contrib.linkextractors import LinkExtractor
from scrapy.contrib.spiders import CrawlSpider, Rule

from gscholar.items import GscholarItem
import re

class GscholarSpider(CrawlSpider):
    name = "gscholar"
    allowed_domains = ["scholar.google.com"]
    start_urls = [
        "https://scholar.google.com/citations?user=Zmwh_JAAAAAJ&hl=en",
    ]

    rules = [
        Rule(LinkExtractor(allow=r'view_op=list_colleagues', process_value='process_value'),
            callback='parse_colleagues', follow=True),
        #Rule(LinkExtractor(allow=r'citations\?user='),
        #    follow=True)
    ]
    #
    def process_value(self, value):
        self.log('[SASHA]: Hi, this is an item page! %s' % value)
        if "citations?user=" in value:
            self.log('[SASHA]: Hi, this is an item page! %s' % value)
            return "value" + "&view_op=list_colleagues"
        else:
            return None
    #
    def process_links(self, links):
        for link in links:
            link.url = "%s&view_op=list_colleagues" % link.url
        #
        #print links
        return links
    #

    def parse_x(self, response):
{'fraction_non_alphanumeric': 0.12172829248026588, 'fraction_numerical': 0.0008309098462816784, 'mean_word_length': 2.6989247311827955, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7539278', 'n_tokens_mistral': 801, 'n_tokens_neox': 783, 'n_words': 165}