<commit_before># -*- coding: utf-8 -*-
from scrapy import Spider 
from scrapy.selector import Selector
from stackjobs.items import StackjobItem

import datetime 

class StackjobSpider(Spider):
    name = "Stackjob"
    allowed_domains = ["stackoverflow.com"]
    start_urls = (
        'https://stackoverflow.com/jobs?sort=p',
        'https://stackoverflow.com/jobs?sort=p&pg=2',
        'https://stackoverflow.com/jobs?sort=p&pg=3'
    )
    def parse(self, response):
        jobs = Selector(response).xpath('//div[contains(@class, "-item") and contains(@class, "-job")]')
        results = [] 
        
        t = datetime.date.today()
        
        for job in jobs:             
            item = StackjobItem()    
            item['date'] = t.strftime('%Y-%m-%d')                           
            item['weeknum'] = t.isocalendar()[1]                                  <commit_msg>Adding more seed links to the spider<commit_after># -*- coding: utf-8 -*-
from scrapy import Spider 
from scrapy.selector import Selector
from stackjobs.items import StackjobItem

import datetime 

class StackjobSpider(Spider):
    name = "Stackjob"
    allowed_domains = ["stackoverflow.com"]
    start_urls = (
        'https://stackoverflow.com/jobs?sort=p',
        'https://stackoverflow.com/jobs?sort=p&pg=2',
        'https://stackoverflow.com/jobs?sort=p&pg=3',
        'https://stackoverflow.com/jobs?sort=p&pg=4',
        'https://stackoverflow.com/jobs?sort=p&pg=5',
        'https://stackoverflow.com/jobs?sort=p&pg=6',
        'https://stackoverflow.com/jobs?sort=p&pg=7',
        'https://stackoverflow.com/jobs?sort=p&pg=8',
        'https://stackoverflow.com/jobs?sort=p&pg=9'
    )
    def parse(self, response):
        jobs = Selector(response).xpath('//div[contains(@class, "-item") and contains(@class, "-job")]')
        results = [] 
        
        t = datetime.date.today()
        
        for job in jobs:             
            item = StackjobItem()    
            item['date'] = t.strftime('%Y-%m-%d')                           
            item['weeknum'] = t.isocalendar()[1]                                  
{'fraction_non_alphanumeric': 0.15413358243811304, 'fraction_numerical': 0.006539000467071462, 'mean_word_length': 2.073170731707317, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3925625', 'n_tokens_mistral': 673, 'n_tokens_neox': 644, 'n_words': 110}