<commit_before>import json
import requests
import datetime
from kafka import SimpleProducer, KafkaClient


def add_entry(collection, text, text_feats, title, url, date, website, lang):
    """
    Function that creates the dictionary of content to add to a MongoDB
    instance and inserts the information into an external data store.

    Parameters
    ----------

    collection : pymongo Collection.
                    Collection within MongoDB that in which results are stored.

    text : String.
            Text from a given webpage.

    text_feats : Dict.
                    Features returned by the hermes API.

    title : String.
            Title of the news story.

    url : String.
            URL of the webpage from which the content was pulled.

    date : String.
            Date pulled from the RSS feed.

    website : String.
                Nickname of the site from which the content was pulled.

    Returns
    -------

    object_id : String
    """

    to_insert = make_entry(collection, text, text_feats, title, url, date,
                           website, lang)
    object_id = collection.insert(to_insert)

    json_friendly = to_insert
    json_friendly['_id'] = str(json_friendly['_id'])
    json_friendly['date'] = str(json_friendly['date'])
    json_friendly['date_added'] = str(json_friendly['date_added'])
    json_friendly = json.dumps(json_friendly)

    # Send to Elasticsearch
    base_url = 'http://172.16.58.3:9200/api/stories/'
    url = base_url + '{}/_create'.format(json_friendly['_id'])
    out = requests.put(url, data=json_friendly)

    if out != 201:
        print('\tError sending to ES.')
    # Send "ISIL-related" stories to XDATA
    # Keywords defined by Uncharted
    keywords = ['terror', 'attack', 'weapon', 'bomb', 'militant', 'islam',
                'isil', 'eiil', 'isis', 'islamic', 'taliban', 'qaeda',
                'jihad', 'iraq', 'syria', 'suicide', 'infidel', 'pakistan',
                'taliban', 'afghanistan', 'yemen', 'kurdish', 'caliphate']
    if any([x in text for x in keywords]):
        print('\tSending to Kafka...')

        kafka = KafkaClient('k01.istresearch.com:9092')
        producer = SimpleProducer(kafka)
        producer.send_messages("caerus-news", json_friendly)

    return object_id
<commit_msg>Update ISIL keywords.<commit_after>import json
import requests
import datetime
from kafka import SimpleProducer, KafkaClient


def add_entry(collection, text, text_feats, title, url, date, website, lang):
    """
    Function that creates the dictionary of content to add to a MongoDB
    instance and inserts the information into an external data store.

    Parameters
    ----------

    collection : pymongo Collection.
                    Collection within MongoDB that in which results are stored.

    text : String.
            Text from a given webpage.

    text_feats : Dict.
                    Features returned by the hermes API.

    title : String.
            Title of the news story.

    url : String.
            URL of the webpage from which the content was pulled.

    date : String.
            Date pulled from the RSS feed.

    website : String.
                Nickname of the site from which the content was pulled.

    Returns
    -------

    object_id : String
    """

    to_insert = make_entry(collection, text, text_feats, title, url, date,
                           website, lang)
    object_id = collection.insert(to_insert)

    json_friendly = to_insert
    json_friendly['_id'] = str(json_friendly['_id'])
    json_friendly['date'] = str(json_friendly['date'])
    json_friendly['date_added'] = str(json_friendly['date_added'])
    json_friendly = json.dumps(json_friendly)

    # Send to Elasticsearch
    base_url = 'http://172.16.58.3:9200/api/stories/'
    url = base_url + '{}/_create'.format(json_friendly['_id'])
    print('\tSending to ES.')
    out = requests.put(url, data=json_friendly)

    if out != 201:
        print('\tError sending to ES.')
    # Send "ISIL-related" stories to XDATA
    # Keywords defined by Uncharted
    keywords = ['isil', 'eiil', 'isis', 'islamic', 'taliban', 'qaeda',
                'caliphate', 'daesh']
    if any([x in text for x in keywords]):
        print('\tSending to Kafka...')

        kafka = KafkaClient('k01.istresearch.com:9092')
        producer = SimpleProducer(kafka)
        producer.send_messages("caerus-news", json_friendly)

    return object_id

{'fraction_non_alphanumeric': 0.10911540188594522, 'fraction_numerical': 0.009429726088908847, 'mean_word_length': 2.4750390015600625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2087149', 'n_tokens_mistral': 1352, 'n_tokens_neox': 1273, 'n_words': 396}