<commit_before>import nltk.data
import datetime
import logging
import codecs
import time
from pymongo import MongoClient
from apscheduler.scheduler import Scheduler


def make_conn():
    client = MongoClient()
    database = client.event_scrape
    return database['stories']


def query_all(collection, less_than_date, greater_than_date):
    output = []
    posts = collection.find({"$and": [{"date_added": {"$lt": less_than_date}},
                                      {"date_added": {"$gt": greater_than_date}}
                                      ]})

    sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')

    for num, post in enumerate(posts):
        try:
            #print 'Processing entry {}...'.format(num)
            content = post['content'].encode('utf-8')
            if post['source'] == 'aljazeera':
                content = content.replace("""Caution iconAttention The browser or device you are using is out of date.  It has known security flaws and a limited feature set.  You will not see all the features of some websites.  Please update your browser.""", '')
            header = '  '.join(sent_detector.tokenize(content.strip())[:4])
            string = '{}\t{}\t{}\n{}\n'.format(num, post['date'], post['url'],
                                               header)
            output.append(string)
        except Exception:
            print 'Error on entry {}...'.format(num)
    final_out = '\n'.join(output)
    return final_out


def main(file_stem):
    conn = make_conn()

    curr = datetime.datetime.utcnow()
    less_than = datetime.datetime(curr.year, curr.month, curr.day)
    greater_than = less_than - datetime.timedelta(days=2)
    desired_date = less_than - datetime.timedelta(days=1)<commit_msg>More cleanup on the logging.<commit_after>import nltk.data
import datetime
import logging
import codecs
import time
from pymongo import MongoClient
from apscheduler.scheduler import Scheduler


def make_conn():
    client = MongoClient()
    database = client.event_scrape
    return database['stories']


def query_all(collection, less_than_date, greater_than_date):
    output = []
    posts = collection.find({"$and": [{"date_added": {"$lt": less_than_date}},
                                      {"date_added": {"$gt": greater_than_date}}
                                      ]})

    sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')

    for num, post in enumerate(posts):
        try:
            #print 'Processing entry {}...'.format(num)
            content = post['content'].encode('utf-8')
            if post['source'] == 'aljazeera':
                content = content.replace("""Caution iconAttention The browser or device you are using is out of date.  It has known security flaws and a limited feature set.  You will not see all the features of some websites.  Please update your browser.""", '')
            header = '  '.join(sent_detector.tokenize(content.strip())[:4])
            string = '{}\t{}\t{}\n{}\n'.format(num, post['date'], post['url'],
                                               header)
            output.append(string)
        except Exception as e:
            print 'Error on entry {}: {}.'.format(num, e)
    final_out = '\n'.join(output)
    return final_out


def main(file_stem):
    conn = make_conn()

    curr = datetime.datetime.utcnow()
    less_than = datetime.datetime(curr.year, curr.month, curr.day)
    greater_than = less_than - datetime.timedelta(days=2)
    desired_date = less_than - datetime.timedelta(days=1)
{'fraction_non_alphanumeric': 0.12489427685367917, 'fraction_numerical': 0.002255427121511136, 'mean_word_length': 2.4990138067061145, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6305462', 'n_tokens_mistral': 1025, 'n_tokens_neox': 991, 'n_words': 264}