<commit_before>import json
import string
from stories.models import Entity
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt

def get_relevant_stories(entity):
    return [{
        'id': story.id,
        'headline': story.headline,
        'byline': story.byline,
        'pubdate': story.pubdate.strftime('%Y-%m-%dT%H:%M:%S'),
        'description': story.description,
        'large_image_url': story.large_image_url,
        'small_image_url': story.small_image_url,
        'video_url': story.video_url,
    } for story in entity.story_set.all()]


def loop_through_words(words, words_offset=0, entity_matches=[]):
    for index, word in enumerate(words):
        e = Entity.objects.filter(entity_name__startswith=word
            ).select_related('entity_type__entity_type'
            ).prefetch_related('story_set')
        for entity in e:
            entity_words = entity.entity_name.split(' ')
            if words[index:index+len(entity_words)] == entity_words and not entity.id in [i['id'] for i in entity_matches]:<commit_msg>Make keyword-related stories smarter<commit_after>import json
import string
from stories.models import Entity
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt

def get_relevant_stories(entity):
    query_by_date = entity.story_set.order_by('pubdate')

    stories = {}
    stories['latest'] = query_by_date.last()
    stories['earliest'] = query_by_date.first()
    excluded_query = entity.story_set.exclude(id__in=[stories['latest'].id, stories['earliest'].id])
    stories['longest'] = excluded_query.order_by('-word_count').first()
    stories['video'] = excluded_query.exclude(id=stories['longest'].id).filter(video_url__isnull=False).first()
    stories = dict(filter(lambda i: i[1] is not None, stories.items()))

    return [{
        'type': story_type,
        'id': story.id,
        'headline': story.headline,
        'byline': story.byline,
        'pubdate': story.pubdate.strftime('%Y-%m-%dT%H:%M:%S'),
        'description': story.description,
        'large_image_url': story.large_image_url,
        'small_image_url': story.small_image_url,
        'video_url': story.video_url,
    } for story_type, story in stories.items()]


def loop_through_words(words, words_offset=0, entity_matches=[]):
    for index, word in enumerate(words):
        e = Entity.objects.filter(entity_name__startswith=word
            ).select_related('entity_type__entity_type'
            ).prefetch_related('story_set')
        for entity in e:
            entity_words = entity.entity_name.split(' ')
            if words[index:index+len(entity_words)] == entity_words and not entity.id in [i['id'] for i in entity_matches]:
{'fraction_non_alphanumeric': 0.12782228696285505, 'fraction_numerical': 0.0010924981791697013, 'mean_word_length': 3.7199312714776633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '573369', 'n_tokens_mistral': 857, 'n_tokens_neox': 858, 'n_words': 167}