<commit_before># -*- coding: utf-8 -*-
import scrapy
# from scrapy.shell import inspect_response
from jobcrawl.items import JobItem

import sys
import locale
import codecs
import re
import datetime


class DrushimSpider(scrapy.Spider):
    name = "drushim"
    allowed_domains = ["drushim.co.il"]
    start_urls = (
        'https://www.drushim.co.il/jobs/search/%22%22/?ssaen=1',
    )

    def __init__(self):

        sys.stdout = codecs.getwriter(
            locale.getpreferredencoding())(sys.stdout)
        reload(sys)
        sys.setdefaultencoding('utf-8')

    def parse(self, response):

        job_container_list = response.xpath(
            "//div[@class='job-item-main pb-3 job-hdr']")
        for job_container in job_container_list:

            job_link = job_container.xpath(
                ".//div[@class='flex nowrap align-self-center pc-view open-job text-center']/a/@href").extract_first()

            try:
                job_id = "-".join(job_link.split("/")[-2:])
            except:
                job_id = ""

            try:
                job_title = job_container.xpath(
                    ".//span[@class='job-url primary--text font-weight-bold primary--text']").xpath(
                        "normalize-space(string())").extract_first()
            except:
                job_title = ""

            try:
                company = job_container.xpath(
                    ".//div[@class='layout job-details-top mt-md-2 align-baseline']"
                    "/div[@class='flex grow-none ml-3']/p").xpath("normalize-space(string())").extract_first()
            except:<commit_msg>Fix drushim job link<commit_after># -*- coding: utf-8 -*-
import scrapy
# from scrapy.shell import inspect_response
from jobcrawl.items import JobItem

import sys
import locale
import codecs
import re
import datetime


class DrushimSpider(scrapy.Spider):
    name = "drushim"
    allowed_domains = ["drushim.co.il"]
    base_url = "https://www.drushim.co.il"
    start_urls = (
        'https://www.drushim.co.il/jobs/search/%22%22/?ssaen=1',
    )

    def __init__(self):

        sys.stdout = codecs.getwriter(
            locale.getpreferredencoding())(sys.stdout)
        reload(sys)
        sys.setdefaultencoding('utf-8')

    def parse(self, response):

        job_container_list = response.xpath(
            "//div[@class='job-item-main pb-3 job-hdr']")
        for job_container in job_container_list:

            job_link = job_container.xpath(
                ".//div[@class='flex nowrap align-self-center pc-view open-job text-center']/a/@href").extract_first()
            if job_link:
                job_link = "{}{}".format(self.base_url, job_link)

            try:
                job_id = "-".join(job_link.split("/")[-2:])
            except:
                job_id = ""

            try:
                job_title = job_container.xpath(
                    ".//span[@class='job-url primary--text font-weight-bold primary--text']").xpath(
                        "normalize-space(string())").extract_first()
            except:
                job_title = ""

            try:
                company = job_container.xpath(
                    ".//div[@class='layout job-details-top mt-md-2 align-baseline']"
                    "/div[@class='flex grow-none ml-3']/p").xpath("normalize-space(string())").extract_first()
            except:
{'fraction_non_alphanumeric': 0.14180206794682423, 'fraction_numerical': 0.006499261447562777, 'mean_word_length': 1.9341421143847486, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5274856', 'n_tokens_mistral': 1061, 'n_tokens_neox': 1005, 'n_words': 177}