<commit_before>from __future__ import division, unicode_literals
from textblob import TextBlob as tb
import json
import sys
import os
import math
reload(sys)
sys.setdefaultencoding('utf-8')


# class gss:
def findValue(state,cinema,sports):
    stateNews = json.load(open(state,'r'))
    cinemaNews = json.load(open(cinema, 'r'))
    sportsNews = json.load(open(sports, 'r'))
    results = "output.txt"
    if not os.path.exists(results):
        os.makedirs(results)
    cinemaWordDict = createDict(cinemaNews)
    stateWordDict = createDict(stateNews)
    sportsWordsDict = createDict(sportsNews)

    setAllWords = buildCorpus(cinemaWordDict,stateWordDict,sportsWordsDict)
    gssDict = gss(setAllWords , cinemaWordDict ,len(cinemaNews), stateWordDict ,len(stateNews),sportsWordsDict,len(sportsNews))

    json.dump(gssDict , open(results,'w+'), indent=4)
    <commit_msg>Added AllDicts.txt as output file for all dictionaries and length of each type<commit_after>from __future__ import division, unicode_literals
from textblob import TextBlob as tb
import json
import sys
import os
import math
reload(sys)
sys.setdefaultencoding('utf-8')
import pickle


# class gss:
def findValue(state,cinema,sports):
    stateNews = json.load(open(state,'r'))
    cinemaNews = json.load(open(cinema, 'r'))
    sportsNews = json.load(open(sports, 'r'))
    results = "output.txt"
    allDicts = "allDicts.txt"
    if not os.path.exists(results):
        os.makedirs(results)
    cinemaWordDict = createDict(cinemaNews)
    stateWordDict = createDict(stateNews)
    sportsWordsDict = createDict(sportsNews)
    lenDict = {}
    lenDict["lengthCinema"] = len(cinemaNews)
    lenDict["lengthState"] = len(stateNews)
    lenDict["lengthSports"] = len(sportsNews)
    setAllWords = buildCorpus(cinemaWordDict,stateWordDict,sportsWordsDict)
    gssDict = gss(setAllWords , cinemaWordDict ,len(cinemaNews), stateWordDict ,len(stateNews),sportsWordsDict,len(sportsNews))

    pickle.dump(gssDict , open(results,'w+'))

    finalDict = []
    finalDict.append(cinemaWordDict)
    finalDict.append(stateWordDict)
    finalDict.append(sportsWordsDict)
    finalDict.append(lenDict)

    pickle.dump(finalDict, open(allDicts, 'w+'))
    
{'fraction_non_alphanumeric': 0.11291779584462511, 'fraction_numerical': 0.0013550135501355014, 'mean_word_length': 5.03542234332425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '147598', 'n_tokens_mistral': 745, 'n_tokens_neox': 730, 'n_words': 142}