<commit_before># -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html


from scrapy import Item, Field


class JobItem(Item):
    website_url = Field()
    website_language = Field()

    publication_date = Field()
    posting_id = Field()
    url = Field()
    number_of_views = Field()

    contact_email = Field()
    contact_name = Field()

    employment_type = Field()
    workload = Field()
    duration = Field()
    remote = Field()

    title = Field(primary_key=True)
    keywords = Field()
    abstract = Field()
    description = Field()

    salary = Field()
    level = Field()

    responsabilities = Field()
    required_skills = Field()
    required_languages = Field()

    company = Field(primary_key=True)
    city = Field()
    country = Field()
    postal_code = Field()

    company_website = Field()
    company_category = Field()
    company_description = Field()

    start_date = Field()
    end_date = Field()


class DataScienceJobsJobItem(Job):
    pass


<commit_msg>Add  JobItemLoader and DataScienceJobsItemLoader class.<commit_after># -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html


from scrapy import Item, Field
from scrapy.loader import Identity, ItemLoader
from scrapy.loader.processors import TakeFirst


class JobItem(Item):
    website_url = Field()
    website_language = Field()

    publication_date = Field()
    posting_id = Field()
    url = Field()
    number_of_views = Field()

    contact_email = Field()
    contact_name = Field()

    employment_type = Field()
    workload = Field()
    duration = Field()
    remote = Field()

    title = Field(primary_key=True)
    keywords = Field()
    abstract = Field()
    description = Field()

    salary = Field()
    level = Field()

    responsabilities = Field()
    required_skills = Field()
    required_languages = Field()

    company = Field(primary_key=True)
    city = Field()
    country = Field()
    postal_code = Field()

    company_website = Field()
    company_category = Field()
    company_description = Field()

    start_date = Field()
    end_date = Field()


class DataScienceJobsJobItem(JobItem):
    pass


class JobItemLoader(ItemLoader):
    default_input_processor = Identity()
    default_output_processor = TakeFirst()


class DataScienceJobsItemLoader(JobItemLoader):
    pass
{'fraction_non_alphanumeric': 0.10738522954091817, 'fraction_numerical': 0.0007984031936127744, 'mean_word_length': 3.1976549413735342, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5471218', 'n_tokens_mistral': 766, 'n_tokens_neox': 732, 'n_words': 192}