<jupyter_start><jupyter_text>To run this file download To run download https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit?usp=sharing File name : "GoogleNews-vectors-negative300.bin" Can also be found here: https://code.google.com/archive/p/word2vec/  Reading Files Line by Line<jupyter_code>def sentenceExtractionForTraining(dirName, fileName, classes):
    sentencesClass = []
    for i in range(0,len(classes)):
        sentences = readFile(dirName+fileName[i])
        sentencesClass.append(sentences)
    return sentencesClass
def readFile(fileName):
    f = open(fileName,"r+")
    sentences = []
    for line in f.readlines():
        line = line.strip()
        if len(line)==0 :
            continue
        if "TIMEX3" in line:
            continue 
        sentences.append(line.lower().rstrip())
    f.close()
    return sentences
def createVocab(sentencesClass):
    vocab = set()
    for i in range(0, len(sentencesClass)):
        for j in range(0,len(sentencesClass[i])):
            words = sentencesClass[i][j].split()
            for w in words:
                vocab.add(w)
    return vocab<jupyter_output><empty_output><jupyter_text>Removing fancy characters<jupyter_code>def removeFancyChars(sentences):
    lengthPhrase = len(sentences)
    for i in range(lengthPhrase):
        sentences[i] = re.sub(r'([^\s\w]|_)+', '', sentences[i])
    return sentences
def removeFC(sentencesClass):
    for i in range(0, len(sentencesClass)):
        sentencesClass[i] = removeFancyChars(sentencesClass[i])
    return sentencesClass<jupyter_output><empty_output><jupyter_text>word2vec.<jupyter_code>def load_bin_vec(fname, vocab):
    """
    Loads 300x1 word vecs from Google (Mikolov) word2vec
    """
    word_vecs = {}
    with open(fname, "rb") as f:
        header = f.readline()
        vocab_size, layer1_size = map(int, header.split())
        binary_len = np.dtype('float32').itemsize * layer1_size
        for line in xrange(vocab_size):
            word = []
            while True:
                ch = f.read(1)
                if ch == ' ':
                    word = ''.join(word)
                    break
                if ch != '\n':
                    word.append(ch)   
            if word in vocab:
               word_vecs[word] = np.fromstring(f.read(binary_len), dtype='float32')  
            else:
                f.read(binary_len)
    return word_vecs
def add_unknown_words(word_vecs, vocab, min_df=1, k=300):
    for word in vocab:
        if word not in word_vecs:
            word_vecs[word] = np.random.uniform(-0.25,0.25,k)
def initializeWordVecs(sentencesClass):
    vocab = createVocab(sentencesClass)
    w2vFile = "GoogleNews-vectors-negative300.bin"
    
    w2v = load_bin_vec(w2vFile, vocab)
    add_unknown_words(w2v, vocab)
        
    return w2v<jupyter_output><empty_output><jupyter_text>For Parts Of Speech<jupyter_code>def POSForSentence(sentence):
    text = word_tokenize(sentence)
    posSentence = nltk.pos_tag(text)
    posSentence = [y for x, y in posSentence]
    return posSentence
def getUniquePOS():
    tagdict = load('help/tagsets/upenn_tagset.pickle')
    return len(tagdict), tagdict.keys()<jupyter_output><empty_output><jupyter_text>For w2v<jupyter_code>def totalSentences(sentencesClass):
    size = 0
    for i in range(0, len(sentencesClass)):
        size += len(sentencesClass[i])
    return size;
def defineW2V(sentencesClass, w2v, dim = 300):
    n = totalSentences(sentencesClass)
    mat = np.zeros((n, dim))
    labels = np.zeros(n)
    k = 0
    for i in range(0, len(sentencesClass)):
        for j in range(0, len(sentencesClass[i])):
            words = sentencesClass[i][j].split()
            d = np.zeros(300)
            ind = 0.0
            for w in words:
                if w not in w2v:
                    w2v[w] = np.random.uniform(-0.25,0.25,dim) 
                d += w2v[w]
                ind += 1.0
            d /= ind
            mat[k] = d
            labels[k] = i+1
            k+=1
    return mat, labels
def defineW2VPOS(originalSentencesClass, sentencesClass, w2v, dim = 300):
    n = totalSentences(sentencesClass)
    labels = np.zeros(n)
    sizePOS, POSList = getUniquePOS()
    mat = np.zeros((n, dim + sizePOS))
    matFromW2V, labels = defineW2V(sentencesClass, w2v)
    
    for i in range(0, n):
        mat[i,:dim] = matFromW2V[i]
        
    k = 0
    for i in range(0, len(originalSentencesClass)):
        for j in range(0,  len(originalSentencesClass[i])):
            pos = POSForSentence(originalSentencesClass[i][j])
            for p in pos:
                mat[k, dim + POSList.index(p)] = 1.0
            k+=1
    
    return mat, labels<jupyter_output><empty_output><jupyter_text>Saving to file<jupyter_code>def savew2vToFile(w2v):
    fileName = "word2VecDict.npy"
    np.save(fileName, w2v)
def finalFeaturesLabel(X,y):
    n, d = X.shape
    finalMat = np.zeros((n,d+1))
    for i in range(0, n):
        finalMat[i, 0] = y[i]
        finalMat[i, 1:] = X[i]
    return finalMat
def saveW2V(fileName, finalMat):
    np.save(fileName, finalMat)<jupyter_output><empty_output><jupyter_text>Loading from file<jupyter_code>def loadW2V():
    w2v = np.load('word2VecDict.npy').item()
    
    i = 0
    for key, value in w2v.iteritems():
        if i>10:
            break
        print key, value
        i = i + 1
def main():
    dirName = "Email-classification_dataset/"
    classes = [1,2,3,4,5]
    fileName = ["RD-positive-800.txt", "meetings-positive-800.txt", "negative-800.txt", "fyi-positive-800.txt", "tp-positive-500.txt",]
    originalsentencesClass = sentenceExtractionForTraining(dirName, fileName, classes)
    sentencesClass = removeFC(originalsentencesClass)
    w2v = initializeWordVecs(sentencesClass)
    
    savew2vToFile(w2v)
    
    XW2V, yW2V = defineW2V(sentencesClass, w2v)
    
    XW2VPOS, yW2VPOS = defineW2VPOS(originalsentencesClass, sentencesClass, w2v)
    
    finalMatW2V = finalFeaturesLabel(XW2V, yW2V)
    finalMatW2VPOS = finalFeaturesLabel(XW2VPOS, yW2VPOS)
    
    saveW2V("w2v.npy",finalMatW2V)
    saveW2V("w2vpos.npy",finalMatW2VPOS)
if __name__=="__main__":
    main()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1196319018404908, 'fraction_numerical': 0.02437843073942525, 'mean_word_length': 2.7184873949579833, 'pattern_counts': {'":': 1, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '83931', 'license': 'MIT', 'n_tokens_mistral': 2263, 'n_tokens_neox': 2194, 'n_words': 429, 'path': 'PRNN/Assignment/Assignment3/word2Vec.ipynb', 'repo_name': 'ashutoshml/Assignments', 'size': '6194'}