<commit_before># !/usr/bin/python
# -*- coding: utf-8 -*-
#
# Created on Oct 16, 2015
# @author:       <NAME>
# @email:        <EMAIL>
# @website:      http://yenching.org
# @organization: Harvard Kennedy School

# libraries
import socket
import smtplib

from pymongo import MongoClient

from wbcrawler.settings import EMAIL_PASSWORD
from wbcrawler.log import *


# receiver string
# example rief_report('<EMAIL>;<EMAIL>', "weibo")
# funcs


def brief_report(pis, project, address, port):
    sender = '<EMAIL>'
    username = '<EMAIL>'
    t = datetime.datetime.now(TZCHINA).strftime('%Y-%m-%d')

    pi_str = ''
    for pi in pis:
        pi_str += (pi + ',')

    now = datetime.datetime.now()
    utc_now_1 = now - datetime.timedelta(days=1)
    utc_now_2 = now - datetime.timedelta(days=2)
    utc_now_5 = now - datetime.timedelta(days=5)

    client = MongoClient(address, port)
    inused = client.local.accounts.find({'inused': True}).count()
    total = client.local.accounts.find().count()

    db = client[project]

    total_posts = db.posts.find().count()
    total_users = db.users.find().count()

    count_1 = db.posts.find({"timestamp": {"$gt": utc_now_1}}).count()
    count_2 = db.posts.find({"timestamp": {"$gt": utc_now_2}}).count()
    count_5 = db.posts.find({"timestamp": {"$gt": utc_now_5}}).count()
    count_6 = db.users.find({"latlng": {"$ne": [0, 0]}}).count()
    count_7 = db.users.find({"path": {"$ne": []}}).count()

    line_1 = "Occupied / total Robots: %d / %d" % (inused, total)
    line_2 = "Total posts: %d" % total_posts
    line_3 = "Within the past 24 hours: %d collected" % count_1
    line_4 = "Within the past 2 days: %d collected" % count_2
    line_5 = "Within the past 5 days: %d collected" % count_5
    line_6 = "Users with paths / not: %d / %d" % (count_7, total_users - count_7)
    line_7 = "Geocoded users / not: %d / %d" % (count_6, total_users - count_6)

    msg = '''From: Crawler Server <<EMAIL>>
To: ''' + pi_str[:-1] + '''
Subject: [''' + t + '''] Daily Briefing for ''' + project.capitalize() + ''' Project
MIME-Version: 1.0

Dear PI(s),

Here is a brief report about the progress of data harvest:

     ''' + line_1 + '''
     ''' + line_2 + '''
     ''' + line_3 + '''
     ''' + line_4 + '''
     ''' + line_5 + '''
     ''' + line_6 + '''
     ''' + line_7 + '''
--
Sent from Weibo Cralwer Server'''
    # The actual mail send
    try:
        server = smtplib.SMTP()
        server.connect('smtp.gmail.com', '587')
        server.ehlo()
        server.starttls()
        server.login(username, EMAIL_PASSWORD)
        server.sendmail(sender, pi_str, msg)
        server.quit()
    except socket.gaierror, e:
        print str(e) + "/n error raises when sending E-mails."<commit_msg>2.8.2 correct a bug on sending to multiple PIs.<commit_after># !/usr/bin/python
# -*- coding: utf-8 -*-
#
# Created on Oct 16, 2015
# @author:       <NAME>
# @email:        <EMAIL>
# @website:      http://yenching.org
# @organization: Harvard Kennedy School

# libraries
import socket
import smtplib

from pymongo import MongoClient

from wbcrawler.settings import EMAIL_PASSWORD
from wbcrawler.log import *


# receiver string
# example rief_report('<EMAIL>;<EMAIL>', "weibo")
# funcs


def brief_report(pis, project, address, port):
    sender = '<EMAIL>'
    username = '<EMAIL>'
    t = datetime.datetime.now(TZCHINA).strftime('%Y-%m-%d')

    pi_str = ''
    for pi in pis:
        pi_str += (pi + ';')

    now = datetime.datetime.now()
    utc_now_1 = now - datetime.timedelta(days=1)
    utc_now_2 = now - datetime.timedelta(days=2)
    utc_now_5 = now - datetime.timedelta(days=5)

    client = MongoClient(address, port)
    inused = client.local.accounts.find({'inused': True}).count()
    total = client.local.accounts.find().count()

    db = client[project]

    total_posts = db.posts.find().count()
    total_users = db.users.find().count()

    count_1 = db.posts.find({"timestamp": {"$gt": utc_now_1}}).count()
    count_2 = db.posts.find({"timestamp": {"$gt": utc_now_2}}).count()
    count_5 = db.posts.find({"timestamp": {"$gt": utc_now_5}}).count()
    count_6 = db.users.find({"latlng": {"$ne": [0, 0]}}).count()
    count_7 = db.users.find({"path": {"$ne": []}}).count()

    line_1 = "Occupied / total Robots: %d / %d" % (inused, total)
    line_2 = "Total posts: %d" % total_posts
    line_3 = "Within the past 24 hours: %d collected" % count_1
    line_4 = "Within the past 2 days: %d collected" % count_2
    line_5 = "Within the past 5 days: %d collected" % count_5
    line_6 = "Users with paths / not: %d / %d" % (count_7, total_users - count_7)
    line_7 = "Geocoded users / not: %d / %d" % (count_6, total_users - count_6)

    msg = '''From: Crawler Server <<EMAIL>>
To: ''' + pi_str[:-1] + '''
Subject: [''' + t + '''] Daily Briefing for ''' + project.capitalize() + ''' Project
MIME-Version: 1.0

Dear PI(s),

Here is a brief report about the progress of data harvest:

     ''' + line_1 + '''
     ''' + line_2 + '''
     ''' + line_3 + '''
     ''' + line_4 + '''
     ''' + line_5 + '''
     ''' + line_6 + '''
     ''' + line_7 + '''
--
Sent from Weibo Cralwer Server'''
    # The actual mail send
    try:
        server = smtplib.SMTP()
        server.connect('smtp.gmail.com', '587')
        server.ehlo()
        server.starttls()
        server.login(username, EMAIL_PASSWORD)
        server.sendmail(sender, pis, msg)
        server.quit()
    except socket.gaierror, e:
        print str(e) + "/n error raises when sending E-mails."
{'fraction_non_alphanumeric': 0.17778983541327545, 'fraction_numerical': 0.020075963103635377, 'mean_word_length': 2.984149855907781, 'pattern_counts': {'":': 20, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6395878', 'n_tokens_mistral': 2052, 'n_tokens_neox': 1975, 'n_words': 473}