<commit_before>import sys
import re
import json

def lines(fp):
    print str(len(fp.readlines()))


def tweetdict(fr):
    tweets={}
    i=0
    from collections import defaultdict
    frequency=defaultdict(int)
    for line in fr:
        if "text" in json.loads(line).keys():
            # edit rawtext and remove any non alphanumeric character 
            #re.sub("[^a-zA-Z]","", old_string)
            #tweets[i]=json.loads(line)["text"]
            tweets[i]= [re.sub("[^a-zA-Z']","", k) for k in json.loads(line)["text"].lower().split(" ") if re.sub("[^a-zA-Z']","", k)!=""]
            
        else:
            tweets[i]= []
        i+=1
        
    return tweets

def scoringfunc(tweet):
    
    temp=[]
    
    newtermssent={}
    for t in tweet.keys():
        #print tweet[t]
        
        temp= tweet[t]
        newterms=[]
        
        for wrd in temp:
            newterms.append(wrd)
        
        for term in newterms:
            
            if term in newtermssent.keys():
                
                newtermssent[term]+=1
            else:
                newtermssent[term]=1
    
            # also add new for loop with all new terms her.
    return newtermssent
        
        

def main():
    sent_file = open(sys.argv[1])
    tweet_file = open(sys.argv[2])
    #hw()
    #lines(sent_file)
    #lines(tweet_file)

    tweets= tweetdict(tweet_file)
    scores =scoringfunc(tweets)
    
    newterms=scores
    #print len(newterms)
    allterms=sum([i[1] for i in newterms.items()])
    for i in newterms.items():
        print i[0],i[1]/float(allterms)
<commit_msg>problem 4 forgot only one input<commit_after>import sys
import re
import json

def lines(fp):
    print str(len(fp.readlines()))


def tweetdict(fr):
    tweets={}
    i=0
    from collections import defaultdict
    frequency=defaultdict(int)
    for line in fr:
        if "text" in json.loads(line).keys():
            # edit rawtext and remove any non alphanumeric character 
            #re.sub("[^a-zA-Z]","", old_string)
            #tweets[i]=json.loads(line)["text"]
            tweets[i]= [re.sub("[^a-zA-Z']","", k) for k in json.loads(line)["text"].lower().split(" ") if re.sub("[^a-zA-Z']","", k)!=""]
            
        else:
            tweets[i]= []
        i+=1
        
    return tweets

def scoringfunc(tweet):
    
    temp=[]
    
    newtermssent={}
    for t in tweet.keys():
        #print tweet[t]
        
        temp= tweet[t]
        newterms=[]
        
        for wrd in temp:
            newterms.append(wrd)
        
        for term in newterms:
            
            if term in newtermssent.keys():
                
                newtermssent[term]+=1
            else:
                newtermssent[term]=1
    
            # also add new for loop with all new terms her.
    return newtermssent
        
        

def main():
    tweet_file = open(sys.argv[1])
    #hw()
    #lines(sent_file)
    #lines(tweet_file)

    tweets= tweetdict(tweet_file)
    scores =scoringfunc(tweets)
    
    newterms=scores
    #print len(newterms)
    allterms=sum([i[1] for i in newterms.items()])
    for i in newterms.items():
        print i[0],i[1]/float(allterms)

{'fraction_non_alphanumeric': 0.134411947728687, 'fraction_numerical': 0.005600497822028625, 'mean_word_length': 1.7956521739130435, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5369113', 'n_tokens_mistral': 1106, 'n_tokens_neox': 1069, 'n_words': 245}