<commit_before>import cluster_labeling.words as words
import cluster_labeling.word_senses as ws

import datetime

import scrapy.log as log

class DjangoWriterPipeline(object):
    def process_item(self, spider, item):
        word, word_dne = words.Word.create_if_dne_and_return(item['word'])
        if word_dne: word.save()

        name, name_dne = words.Word.create_if_dne_and_return(item['name'])
        if name_dne: name.save()<commit_msg>A word's synonym crawl time is updated, even if the synonym page is not found<commit_after>import cluster_labeling.words as words
import cluster_labeling.word_senses as ws

import datetime

import scrapy.log as log

class DjangoWriterPipeline(object):
    def process_item(self, spider, item):
        word, word_dne = words.Word.create_if_dne_and_return(item['word'])

        word_found = item['found']
        if not word_found:
            word.synonyms_last_crawled_date = datetime.date.today()
            word.save()
            return item

        name, name_dne = words.Word.create_if_dne_and_return(item['name'])
        if name_dne: name.save()
{'fraction_non_alphanumeric': 0.08802177858439202, 'fraction_numerical': 0.0, 'mean_word_length': 3.6737288135593222, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4038910', 'n_tokens_mistral': 359, 'n_tokens_neox': 346, 'n_words': 84}