<commit_before>#!/usr/bin/python

# input: csv-formatted stream, with each line corresponding to the payload for a message
# output: csv-formatted stream, with each line corresponding to higher-level features for a specific user
# assumes the input messages from a specific user are contiguous    

import fileinput
import json
import csv
from collections import defaultdict

inds = {}

def main():
    # print inds
    parseMessages()



# returns an array of parsed messages
def parseMessages():
    fieldsnames = [u'experiment',
        u'addon_version', 
        u'test_mode', 
        u'locale', 
        u'experiment_version', 
        u'systemversion', 
        u'os',
        u'systemname',
        u'userid',  
        u'arm',
        u'ts',
        u'type', u'value',   u'triggerid']

    
    currUserId = None
    userMessages = []


    for line in fileinput.input():   
        if fileinput.isfirstline():
            fields = line.strip().split('\t')
            
            for i in range(len(fields)):
                inds[fields[i]] = i
                
            print inds

        else: 
            jsonrow = [json.loads(val) for val in line.split('\t')]
            
            userId = jsonrow[inds['userid']]
            currUserId = currUserId or userId

            if userId != currUserId:
                processUser(userMessages, currUserId)
                currUserId = userId
                userMessages = []

            userMessages.append(jsonrow)

    processUser(userMessages, currUserId)   # last line 



def processUser(userMessages, userId):
    print hasUserSeenRecommendation(userMessages, 'gmail')






# returns a dictionary with the userid as its key and the array of messages of
# a user as its value
def getMessagesByUserIds(messagesArr):
    messagesDict = defaultdict(list)

    for message in messagesArr:
        messagesDict[message['userid']].append(message)

    return messagesDict


# filters by: experiment version, test_mode
def basicFilter(messagesArr):

    messagesArr = [message for message in messagesArr 
        if message['experiment_version'] == "2.0.0"  # Experiment Version
        and message['test_mode'] == False   # Test Mode
        ]
    

    return messagesArr




def hasUserSeenRecommendation(messagesArr, featurename):
    for message in messagesArr:
        if message[inds['triggerid']] == featurename and message[inds['type']] == 'PANELSHOW': return True

    return False




if __name__ == "__main__":
    main()<commit_msg>testing the basic structure<commit_after>#!/usr/bin/python

# input: csv-formatted stream, with each line corresponding to the payload for a message
# output: csv-formatted stream, with each line corresponding to higher-level features for a specific user
# assumes the input messages from a specific user are contiguous    

import fileinput
import json
import csv
from collections import defaultdict

inds = {}

counter = 0

def main():
    # print inds
    parseMessages()

    print counter



# returns an array of parsed messages
def parseMessages():
    fieldsnames = [u'experiment',
        u'addon_version', 
        u'test_mode', 
        u'locale', 
        u'experiment_version', 
        u'systemversion', 
        u'os',
        u'systemname',
        u'userid',  
        u'arm',
        u'ts',
        u'type', u'value',   u'triggerid']

    
    currUserId = None
    userMessages = []


    for line in fileinput.input():   
        if fileinput.isfirstline():
            fields = line.strip().split('\t')
            
            for i in range(len(fields)):
                inds[fields[i]] = i
                

        else: 
            jsonrow = [json.loads(val) for val in line.split('\t')]
            
            userId = jsonrow[inds['userid']]
            currUserId = currUserId or userId

            if userId != currUserId:
                processUser(userMessages, currUserId)
                currUserId = userId
                userMessages = []

            userMessages.append(jsonrow)

    processUser(userMessages, currUserId)   # last line 



def processUser(userMessages, userId):
    
    hasUserSeenRecommendation(userMessages, 'newbookmark')






# returns a dictionary with the userid as its key and the array of messages of
# a user as its value
def getMessagesByUserIds(messagesArr):
    messagesDict = defaultdict(list)

    for message in messagesArr:
        messagesDict[message['userid']].append(message)

    return messagesDict


# filters by: experiment version, test_mode
def basicFilter(messagesArr):

    messagesArr = [message for message in messagesArr 
        if message['experiment_version'] == "2.0.0"  # Experiment Version
        and message['test_mode'] == False   # Test Mode
        ]
    

    return messagesArr

def count():
    global counter
    counter += 1


def hasUserSeenRecommendation(messagesArr, featurename):

    for message in messagesArr:
        if message[inds['triggerid']] == featurename and message[inds['type']] == 'PANELSHOW': return True
    count()
    return False




if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.09140625, 'fraction_numerical': 0.0015625, 'mean_word_length': 2.4094540612516644, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2110915', 'n_tokens_mistral': 1466, 'n_tokens_neox': 1416, 'n_words': 420}