<commit_before>import os
import redis
import requests
import feedparser
import datetime
import dateutil.parser
from json import loads, dumps

REDIS_URL = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')
r_server = redis.StrictRedis.from_url(REDIS_URL)

def update_topstories():
    """ fetches topstories from rss feeds and updates redis """

    try:
        topstories = loads(r_server.get('topstories'))
        topstories_archive = loads(r_server.get('topstories_archive'))
    except TypeError:
        topstories = {}
        topstories_archive = {}

    # collect search terms from google doc
    search_terms_csv_url = 'https://docs.google.com/spreadsheet/pub?key=0AtHxCskz4p33dEs5elE0eUxfd3Z4VnlXTVliVWJxRHc&single=true&gid=4&output=csv'
    response = requests.get(search_terms_csv_url)
    assert response.status_code == 200, 'Wrong status code'

    search_terms = {}
    for line in response.content.split("\n"):
        probe_name = line.split(',')[0].replace(',',' ').strip()
        search_terms[probe_name.lower()] = [l.strip('"') for l in line.split(',')[1:] if l]  # removes empty strngs

    # read each feed
    with open('news_feeds.txt') as f:
        news_feeds = f.readlines()[1:]

    # keep track of when we add a new top story
    new_top_stories = {}

    for url in news_feeds:

        feed = feedparser.parse(url)
        for post in feed.entries:

            # nowsearch each term against each post.title
            for probe_name, terms in search_terms.items():
                for term in terms:

                    if post.title.find(term) > -1:<commit_msg>-mstrips whitespace from search terms<commit_after>import os
import redis
import requests
import feedparser
import datetime
import dateutil.parser
from json import loads, dumps

REDIS_URL = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')
r_server = redis.StrictRedis.from_url(REDIS_URL)

def update_topstories():
    """ fetches topstories from rss feeds and updates redis """

    try:
        topstories = loads(r_server.get('topstories'))
        topstories_archive = loads(r_server.get('topstories_archive'))
    except TypeError:
        topstories = {}
        topstories_archive = {}

    # collect search terms from google doc
    search_terms_csv_url = 'https://docs.google.com/spreadsheet/pub?key=0AtHxCskz4p33dEs5elE0eUxfd3Z4VnlXTVliVWJxRHc&single=true&gid=4&output=csv'
    response = requests.get(search_terms_csv_url)
    assert response.status_code == 200, 'Wrong status code'

    search_terms = {}
    for line in response.content.split("\n"):
        probe_name = line.split(',')[0].replace(',',' ').strip()
        search_terms[probe_name.lower()] = [l.strip('"').strip() for l in line.split(',')[1:] if l]  # removes empty strngs

    # read each feed
    with open('news_feeds.txt') as f:
        news_feeds = f.readlines()[1:]

    # keep track of when we add a new top story
    new_top_stories = {}

    for url in news_feeds:

        feed = feedparser.parse(url)
        for post in feed.entries:

            # nowsearch each term against each post.title
            for probe_name, terms in search_terms.items():
                for term in terms:

                    if post.title.find(term) > -1:
{'fraction_non_alphanumeric': 0.11324257425742575, 'fraction_numerical': 0.012376237623762377, 'mean_word_length': 3.1716129032258062, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '174935', 'n_tokens_mistral': 1101, 'n_tokens_neox': 1045, 'n_words': 251}