<jupyter_start><jupyter_text>Sample Recent r/feminism newcomer commenters  Data integrity checks and load background data<jupyter_code>## LOOK AT HOW RECENT OUR COMMENTS DATASET IS (HOPE THAT WE HAVE ALL THE WAY TO THE BEGINNING OF MARCH)
## LOOK AT HOW RECENT OUR SUBMISSIONS DATASET IS (HOPE THAT WE HAVE ALL THE WAY TO THE BEGINNING OF MARCH)

## CONSTRUCT A MEASURE OF THE SIX MONTH WINDOW (FROM Feminism Newcomer Analysis)
begin_date = '2018-03-01'
query_text = """
select * from comments 
    WHERE subreddit_id = '2qr7i'
    AND created_at >= {begin_date}
    ORDER BY created_utc ASC;
""".format(begin_date = begin_date)

## ADDED THE SECOND CREATED_AT TO PRESERVE EXPERIMENT INTEGRITY

database_comments = []

for row in db_engine.execute(text(query_text)):
    comment = {}
    comment_data = json.loads(row['comment_data'])
    for key in comment_data.keys():
        comment[key] = comment_data[key]
    for key in row.keys():
        comment[key]=row[key]
    comment['created'] = comment['created_utc']
    database_comments.append(comment)
    
print("Loaded {0} Comments".format(len(database_comments)))
file_comments = []
comment_ids = set()
comment_count = 0

comments_filename="full_feminism_comments_through_02_2018.json"

with open(os.path.join(FILE_BASE_DIR, "selected_output", comments_filename), "r") as f:
    for line in f:
        item = json.loads(line)
        if(item['id'] not in comment_ids):
            item['created'] = datetime.utcfromtimestamp(float(item['created_utc']))
            item['body.charlength'] = len(item['body'])
            #item['body'] = None
            file_comments.append(item)
            comment_ids.add(item['id'])
        comment_count += 1
#file_comments = sorted(file_comments, key = lambda x: x['created'])        

print("Loaded {0} Comments".format(len(file_comments)))
print("Loaded {0} Comment lines".format(comment_count))<jupyter_output>Loaded 85412 Comments
Loaded 90555 Comment lines<jupyter_text>Add database comments to file-loaded comments<jupyter_code>keys_to_include = []
for key in file_comments[0].keys():
    if key in database_comments[0].keys():
        keys_to_include.append(key)
#print("Keys in file and in database: [{0}]".format((",").join(keys_to_skip)))


all_comments = file_comments

added_comment_ids = 0
duplicate_comment_ids = 0

for comment in database_comments:
    if comment['id'] not in comment_ids:
        trimmed_comment = {}
        for key in keys_to_include:
            trimmed_comment[key] = comment[key]
        all_comments.append(trimmed_comment)
        comment_ids.add(comment['id'])
        added_comment_ids += 1
    else:
        duplicate_comment_ids += 1
print("Merged comment datasets. Added {0} comments and skipped {1} duplicates".format(
    added_comment_ids, duplicate_comment_ids))

## NOW SORT COMMENTS
all_comments = sorted(all_comments, key = lambda x: x['created'])<jupyter_output>Merged comment datasets. Added 7088 comments and skipped 148 duplicates<jupyter_text>Record the number of previous comments that a participant made in the prior six months<jupyter_code>def previous_actions():
    return {"comments":[], "posts":[]}

author_records = defaultdict(previous_actions)

for item in all_comments:
    author_id = item['author']
    author_records[author_id]['comments'].append(item)

#one_eighty_days in seconds
def count_if_eligible(current, comparator):
    one_eighty_days = 60*60*24*180
    if(current['created'] > comparator['created'] and 
       (current['created'] - comparator['created']).total_seconds()<one_eighty_days):
        return 1
    return 0

earliest_date = all_comments[0]['created']

sys.stdout.write("\ncomments")
sys.stdout.flush()
items_processed = 0
for item in all_comments:
    previous_comments = 0
    for comment in author_records[item['author']]['comments']:
        if(item['created'] > comment['created']):
            previous_comments += count_if_eligible(item, comment) 
    items_processed += 1
    item['previous.comments'] = previous_comments
    item['eligible'] = item['created'] > earliest_date
    
    if(items_processed % 1000 == 0):
        sys.stdout.write(".")
        sys.stdout.flush()<jupyter_output>comments............................................................................................<jupyter_text>Create a Sample of Recent Commenters First time commenters within the last 45 days<jupyter_code>recent_comment_in_seconds = 60*60*24*45
current_time = datetime.utcnow()
eligible_comments = [x for x in all_comments if x['eligible'] and 
                      (current_time - x['created']).total_seconds() < recent_comment_in_seconds and
                      x['previous.comments']==0]

print("Total comments: {0}. Eligible comments: {1}".format(
    len(all_comments),
    len(eligible_comments)))<jupyter_output>Total comments: 92500. Eligible comments: 1632<jupyter_text>Label comments by how many weeks previously they commented for the first time (for a stratified random sample)<jupyter_code>one_week_in_seconds = 60*60*24*7

for comment in eligible_comments:
    comment['first.comment.week.diff'] = int((current_time - comment['created']).total_seconds() / 
                                             one_week_in_seconds)
    
    
    
plt.hist([comment['first.comment.week.diff'] for comment in eligible_comments])
plt.title("How many weeks previously was this comment posted")
plt.show()<jupyter_output><empty_output><jupyter_text>Create a stratified random sample of accounts to message, based on the elapsed number of weeks since they originally posted their first commentLater, we may want to do a post-hoc analysis of how many comments they had previously made in the subreddit. Note that we will want to include them int he sample even if their account doesn't exist, because we're going to be including those kinds of accounts in the study, and we will want to think about that attrition as an expected part of the sample.<jupyter_code>eligible_by_week = {}
for key in set([x['first.comment.week.diff'] for x in eligible_comments]):
    eligible_by_week[key] = [x['author'] for x in eligible_comments if 
                             x['first.comment.week.diff'] == key]
for key in sorted(eligible_by_week.keys()):
    print("Week age {0}: {1} authors ".format(key, len(eligible_by_week[key])))
##  Decision: survey 100 authors per group
##  Only survey the last 4 weeks
random.seed(880442) # current milliseconds at code time
sample_size = 100
selected_by_week = {}
for week in sorted(eligible_by_week.keys()):
    if week <=4:
        selected_by_week[week] = random.sample(eligible_by_week[week], sample_size)
        
for week in sorted(selected_by_week.keys()):
    print("Week age {0}: {1} authors ".format(week, len(selected_by_week[week])))<jupyter_output>Week age 0: 100 authors 
Week age 1: 100 authors 
Week age 2: 100 authors 
Week age 3: 100 authors 
Week age 4: 100 authors<jupyter_text>Construct and personalize the message  Construct the message template<jupyter_code>MESSAGE_SUBJECT = "Welcome! Help r/feminism understand first-time commenters with 3 questions"
MESSAGE_TEMPLATE = """Dear {username},

Hello and welcome from r/feminism! 

Might you answer a 2 minute, 3-question survey to help us learn more about your experience? We're' currently working with researchers at Princeton University to improve newcomer experiences.

[Learn more and answer the survey here]({url}). Thanks! 

-- [CivilServant](http://civilservant.io/) and the [r/feminism moderators](https://www.reddit.com/r/feminism/about/moderators)
"""
print(MESSAGE_TEMPLATE.format(username="natematias", 
                              url="https://docs.google.com/forms/d/e/1FAIpQLSfmgOcsF0ALqWGPHGpJq2cwUr-xIG-uizbQqXPzPuTkrPM-Lg/viewform?usp=pp_url&entry.360321409={username}&entry.1933578897".format(
                              username="natematias")))
r.send_message("natematias", MESSAGE_SUBJECT, MESSAGE_TEMPLATE.format(username="natematias"))<jupyter_output><empty_output><jupyter_text>Add the ability to validate users<jupyter_code>def get_reddit_user(username):
    try:
        user_dict = r.get_redditor(username, fetch=True).json_dict
        logging.info("User %s found: %s" %(username, str(user_dict)))
        return user_dict
    except praw.errors.NotFound as e:
        logging.error("User not found: %s" % username)
    except Exception as e:
        logging.exception("Failed to retrieve user")
import time
send_log_dir = "/home/nathan/CivilServant-Analysis/SOC412/feminism"
send_log_filename = "message_sender_feminism_04_17_2018.csv"
def sent_message_previously(username):
    with open(os.path.join(send_log_dir, send_log_filename), "r") as f:
        send_log = []
        for row in csv.DictReader(f):
            send_log.append(row)
            if(username == row['username']):
                return True
    return False<jupyter_output><empty_output><jupyter_text>Add the ability to send messages<jupyter_code>## RETURN TRUE IF MESSAGE WAS SENT SUCCESSFULLY
def send_reddit_message(**message_data):
    user = message_data["username"]    
    message = MESSAGE_TEMPLATE.format(**message_data)    
    try:
        logging.info("Sending a message to user %s with data %s" % (user, str(message_data)))
        response = r.send_message(user, MESSAGE_SUBJECT, message, raise_captcha_exception=True)
        if response["errors"]:
            logging.error("Error in response when sending a message to user %s: %s" % (user, str(response)))
            return False
        else:
            logging.info("Message successfully sent to user %s" % user)
            return True
    except praw.errors.InvalidCaptcha as e:
        logging.exception("Message sending requires a captcha")
        logging.error(e.response)
        return False
    except Exception as e:
        logging.exception("Failed to send message to %s" % user)
        return False<jupyter_output><empty_output><jupyter_text>Wrap the messenger function with anything survey-specific<jupyter_code>def send_survey_message(**message_data):
    message = MESSAGE_TEMPLATE.format(**message_data)        
    sent_status = send_reddit_message(**message_data)
    with open(os.path.join(send_log_dir,send_log_filename), "a") as f:
        print("APPENDING TO FILE: {0}".format(message_data))
        f.write(",".join(map(str, [
            message_data['username'],
            True, #account exists
            message_data['get_info_date'],
            message_data['deleted'],
            message_data['url'],
            message_data['first_comment_week'],
            sent_status
        ])) + "\n")
        f.close()
    
    logging.info("first_comment_week: %s" % message_data["first_comment_week"])
    logging.info("url: %s" % message_data["url"])
    logging.info("get_info_data: %s" % message_data["get_info_date"])
    logging.info("deleted: %s" % message_data["deleted"])
    
def decide_and_send_to_user(username, first_comment_week):
    if(sent_message_previously(username)):
        logging.info("User %s previously received a message" % username)
        return None
    else:
        time.sleep(2)
        user = get_reddit_user(username)

    if user:
        send_survey_message(
            username = username,
            first_comment_week = first_comment_week,
            get_info_date = str(datetime.utcnow()),
            deleted = False,
            url = "https://docs.google.com/forms/d/e/1FAIpQLSfmgOcsF0ALqWGPHGpJq2cwUr-xIG-uizbQqXPzPuTkrPM-Lg/viewform?usp=pp_url&entry.360321409={username}&entry.1933578897".format(
                username=username))
    else:
        with open(os.path.join(send_log_dir, send_log_filename), "a") as f:
            f.write(",".join(map(str, [
                username,
                False,
                str(datetime.utcnow()),
                None,
                None,
                first_comment_week,
                False
            ])) + "\n")
            f.close()
## TEST SENDING
#decide_and_send_to_user("natematias", 1)
sum([len(x) for x in selected_by_week.values()])<jupyter_output><empty_output><jupyter_text>Send Surveys to Participants<jupyter_code># import time
# for week in sorted(selected_by_week.keys()):
#     print("Week age {0}: {1} authors ".format(week, len(selected_by_week[week])))
#     for username in selected_by_week[week]:
#         decide_and_send_to_user(username, week)
#x = r.send_message("natematias", "test message", "test body")
#x = r.send_message("natematias1234", "test message subject", "test body")<jupyter_output>substituting https://oauth.reddit.com for https://api.reddit.com in url
POST: https://oauth.reddit.com/api/compose/.json<jupyter_text>Merge Historical Data with Survey Results (later)<jupyter_code>survey_responses = {}
with open("feminism/merged_survey_responses-04.24.2018-07.52-ET.csv", "r") as f:
    for row in csv.DictReader(f):
        survey_responses[row['account']] = row<jupyter_output><empty_output><jupyter_text>Load mod_actions<jupyter_code>import datetime

recent_mod_actions = []
for row in db_engine.execute(text("""
SELECT action_data FROM mod_actions 
    WHERE subreddit_id="2qr7i" 
    AND created_utc >= "2018-03-01" 
    ORDER BY created_utc;
""")):
    mod_action = json.loads(row['action_data'])
    mod_action['created'] = utc.localize(datetime.datetime.utcfromtimestamp(mod_action['created_utc']))
    recent_mod_actions.append(mod_action)
print("{0} moderator actions loaded".format(len(recent_mod_actions)))
comment_dict = {}

for comment in all_comments:
    comment['visible'] = True
    comment_dict[comment['id']]  = comment

for action in recent_mod_actions:
    if action['action'] == "removecomment":
        key = action['target_fullname'].replace("t1_","")
        if key in comment_dict.keys():
            comment_dict[key]['visible'] = False
    elif action['action'] == 'approvecomment':
        key = action['target_fullname'].replace("t1_","")
        if key in comment_dict.keys():
            comment_dict[key]['visible'] = True
author_comments = defaultdict(list)
for comment_id, comment in comment_dict.items():
    author_comments[comment['author']].append(comment)
    
for key, comments in author_comments.items():
    author_comments[key] = sorted(comments, key=lambda x: x['created'])<jupyter_output><empty_output><jupyter_text>Merge comment dataset with survey responses<jupyter_code>author_comment_dataset = []
for author, survey in survey_responses.items():
    if(author in author_comments.keys()):
        # select the last comment they made before the survey
        comments = copy.copy(author_comments[author])
        for comment in comments:
            row = copy.copy(comment)
            row.update(survey)
            author_comment_dataset.append(row)
pd.DataFrame(author_comment_dataset).to_csv("feminism/author_survey_comments-04.25.2018.csv")
survey<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11905401773716742, 'fraction_numerical': 0.01572158022037087, 'mean_word_length': 3.7877130910260535, 'pattern_counts': {'":': 3, '<': 54, '<?xml version=': 0, '>': 56, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '65188', 'license': 'MIT', 'n_tokens_mistral': 4775, 'n_tokens_neox': 4539, 'n_words': 1083, 'path': 'research-designs/SOC412/Feminism Messaging Pilot.ipynb', 'repo_name': 'c4fcm/CivilServant-Analysis', 'size': '14884'}