<commit_before># Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

class DatasetPipeline(object):
    def process_item(self, item, spider):
        return item<commit_msg>Convert item processing to pipeline module<commit_after>import re
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

class DatasetPipeline(object):

    title_regex = re.compile('(((((\\(?[A-Za-z]{1}[-A-Za-z]+,?\\)?)|[-0-9]+)|-)|\\(?[A-Za-z0-9]+\\)?) *)+')
    frequency_regex = re.compile('([A-Z]{1}[a-z]+ *)+')

    def process_item(self, item, spider):

        if item['name']:
            item['name'] = self.title_regex.search(item['name'][0].encode('ascii', 'ignore')).group()
        else:
            item['name'] = 'Dataset Title Regex Matching Unsuccessful'

        if item['frequency']:
             item['frequency'] = self.frequency_regex.search(item['frequency'][0].encode('ascii','ignore')).group()
        else:
            item['frequency'] = 'Dataset Frequency Attribute Regex Matching Unsuccessful'

        return item
{'fraction_non_alphanumeric': 0.17363344051446947, 'fraction_numerical': 0.006430868167202572, 'mean_word_length': 3.5940959409594098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1754540', 'n_tokens_mistral': 408, 'n_tokens_neox': 397, 'n_words': 85}