<commit_before>"""
Functions for the items index.
"""

from collections import OrderedDict
from itertools import chain
from urlparse import urlparse

from elasticsearch_dsl import F

from django.core.urlresolvers import resolve

from . import index
from ..utils import clean_query_string


def data_for_object(obj):
    doc_type = index.document_types.get(obj.__class__, None)
    if doc_type is None:
        return None

    # will raise an exception if it's not there! should change
    return index.es.get(
        index=index.name, doc_type=doc_type.type_label, id=obj.id)


def get_referencing_items(item_url):
    """
    Get all items which have referenced the given item URL
    """
    query_filter = F('term', **{'serialized._embedded': item_url})

    if 'topic' in item_url:
        query_filter = query_filter | (
            F('term', **{'serialized.related_topic.url': item_url}))

    query = index.make_search()\
        .filter(query_filter)\
        .fields(['display_url'])

    return [(result.display_url[0]) for result in query.execute().hits]


def get_data_for_urls(item_urls):
    docs = []
    ret = OrderedDict()

    item_urls = list(item_urls)
    item_urls.sort()
<commit_msg>Correctly return empty embedded value when appropriate<commit_after>"""
Functions for the items index.
"""

from collections import OrderedDict
from itertools import chain
from urlparse import urlparse

from elasticsearch_dsl import F

from django.core.urlresolvers import resolve

from . import index
from ..utils import clean_query_string


def data_for_object(obj):
    doc_type = index.document_types.get(obj.__class__, None)
    if doc_type is None:
        return None

    # will raise an exception if it's not there! should change
    return index.es.get(
        index=index.name, doc_type=doc_type.type_label, id=obj.id)


def get_referencing_items(item_url):
    """
    Get all items which have referenced the given item URL
    """
    query_filter = F('term', **{'serialized._embedded': item_url})

    if 'topic' in item_url:
        query_filter = query_filter | (
            F('term', **{'serialized.related_topic.url': item_url}))

    query = index.make_search()\
        .filter(query_filter)\
        .fields(['display_url'])

    return [(result.display_url[0]) for result in query.execute().hits]


def get_data_for_urls(item_urls):
    docs = []
    ret = OrderedDict()

    if not item_urls:
        return ret

    item_urls = list(item_urls)
    item_urls.sort()

{'fraction_non_alphanumeric': 0.10204081632653061, 'fraction_numerical': 0.0008003201280512205, 'mean_word_length': 3.424778761061947, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3294399', 'n_tokens_mistral': 784, 'n_tokens_neox': 768, 'n_words': 215}