<commit_before>from os import environ

from scrapy_splash import SplashRequest
from scrapy.spiders.init import InitSpider
from scrapy.http import Request, FormRequest


class PeopleSearchSpider(InitSpider):
    name = 'people_search'
    allowed_domains = ['linkedin.com']
    login_page = 'https://www.linkedin.com/uas/login'

    def __init__(self, *args, **kwargs):
        try:
            self.username = (kwargs.pop('username', None) or
                             environ['SPIDER_USERNAME'])
            self.password = (kwargs.pop('password', None) or
                             environ['SPIDER_PASSWORD'])
        except KeyError:
            raise Exception('Both username and password need to be specified '
                            'by -a option or SPIDER_<PARAM> environment var')

        query = kwargs.pop('query', 'Mateusz+Moneta')
        self.start_urls = [
            'https://www.linkedin.com/vsearch/f?type=people&keywords=%s' % query
        ]

        super().__init__(*args, **kwargs)

    def init_request(self):
        return Request(url=self.login_page, callback=self.login)

    def login(self, response):
        return FormRequest.from_response(
            response, callback=self.check_login_response,
            formdata={'session_key': self.username,
                      'session_password': self.password})

    def parse(self, response):
        for search_result in response.css('li.mod.result.people'):
            *first_name, last_name = search_result.css('b::text').extract()
            yield {
                'first_name': ' '.join(first_name),
                'last_name': last_name,
            }

    def check_login_response(self, response):
        if b'Sign Out' in response.body:
            self.logger.debug("Successfully logged in. Let's start crawling!")
            return self.initialized()

        self.logger.error('Login failed!')

    def make_requests_from_url(self, url):
        # Do SplashRequest instead of regular one to be able to evaluate
        # JavaScript responsible for dynamic page generation.
        return SplashRequest(url)<commit_msg>Handle cases when searched word is only part of the people name.<commit_after>from os import environ

from scrapy_splash import SplashRequest
from scrapy.spiders.init import InitSpider
from scrapy.http import Request, FormRequest


class PeopleSearchSpider(InitSpider):
    name = 'people_search'
    allowed_domains = ['linkedin.com']
    login_page = 'https://www.linkedin.com/uas/login'

    def __init__(self, *args, **kwargs):
        try:
            self.username = (kwargs.pop('username', None) or
                             environ['SPIDER_USERNAME'])
            self.password = (kwargs.pop('password', None) or
                             environ['SPIDER_PASSWORD'])
        except KeyError:
            raise Exception('Both username and password need to be specified '
                            'by -a option or SPIDER_<PARAM> environment var')

        query = kwargs.pop('query', 'Mateusz+Moneta')
        self.start_urls = [
            'https://www.linkedin.com/vsearch/f?type=people&keywords=%s' % query
        ]

        super().__init__(*args, **kwargs)

    def init_request(self):
        return Request(url=self.login_page, callback=self.login)

    def login(self, response):
        return FormRequest.from_response(
            response, callback=self.check_login_response,
            formdata={'session_key': self.username,
                      'session_password': self.password})

    def parse(self, response):
        for search_result in response.css('li.mod.result.people'):
            names = search_result.css('a.title.main-headline').xpath(
                'string(.)').extract_first()
            *first_name, last_name = names.split()

            yield {
                'first_name': ' '.join(first_name),
                'last_name': last_name,
            }

    def check_login_response(self, response):
        if b'Sign Out' in response.body:
            self.logger.debug("Successfully logged in. Let's start crawling!")
            return self.initialized()

        self.logger.error('Login failed!')

    def make_requests_from_url(self, url):
        # Do SplashRequest instead of regular one to be able to evaluate
        # JavaScript responsible for dynamic page generation.
        return SplashRequest(url)
{'fraction_non_alphanumeric': 0.10509554140127389, 'fraction_numerical': 0.0, 'mean_word_length': 2.3513719512195124, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1071466', 'n_tokens_mistral': 1217, 'n_tokens_neox': 1188, 'n_words': 287}