<commit_before>from crawler.crawler_items import CrawlerItem
from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
import sys

class CrawlerSpider(BaseSpider):
    name = "crawler"

    def __init__(self, domain=None, start_urls=None):
        self.allowed_domains = [domain]
	self.start_urls = [start_urls] 

    def parse(self, response):
        print "Entered"
        hxs = HtmlXPathSelector(response)
        titles = hxs.select("//input")
        item = CrawlerItem()
        item["form"] = []
        item["url"] = response.url
        for title in titles:
            item["form"].append(title.extract())
        return item<commit_msg>added way to send cookies<commit_after>from crawler.crawler_items import CrawlerItem
from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
import sys
import scrapy
import ast

class CrawlerSpider(BaseSpider):
    name = "crawler"

    def __init__(self, domain=None, start_urls=None, cookies=None):
        self.allowed_domains = [domain]
        self.start_urls = [start_urls]        
        self.cookies = ast.literal_eval(cookies)	      	      
	      
    def start_requests(self):
        return [scrapy.Request(url=self.start_urls[0], cookies=self.cookies, callback=self.parse_2)]

    def parse_2(self, response):        
        print "Entered"
        item = CrawlerItem()
        hxs = HtmlXPathSelector(response)
        titles = hxs.select("//input")        
        item["form"] = []
        item["url"] = response.url
        for title in titles:
            item["form"].append(title.extract())            
        return item
{'fraction_non_alphanumeric': 0.10151975683890578, 'fraction_numerical': 0.00182370820668693, 'mean_word_length': 2.715575620767494, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2545423', 'n_tokens_mistral': 496, 'n_tokens_neox': 495, 'n_words': 110}