<jupyter_start><jupyter_text>将词向量简单的做平均来构造一个文档的向量DocVector.<jupyter_code>import numpy as np  # Make sure that numpy is imported

def makeFeatureVec(words, model, num_features):
    # Function to average all of the word vectors in a given
    # paragraph
    #
    # Pre-initialize an empty numpy array (for speed)
    featureVec = np.zeros((num_features,),dtype="float64")
    #
    nwords = 0
    # 
    # Index2word is a list that contains the names of the words in 
    # the model's vocabulary. Convert it to a set, for speed 
    index2word_set = set(model.wv.index2word)
    #
    # Loop over each word in the review and, if it is in the model's
    # vocaublary, add its feature vector to the total
    for word in words:
        if word in index2word_set: 
            nwords = nwords + 1
            featureVec = np.add(featureVec,model[word])
    # 
    # Divide the result by the number of words to get the average
    featureVec = np.divide(featureVec,nwords*1.0)
    return featureVec


def getAvgFeatureVecs(reviews, model, num_features):
    # Given a set of reviews (each one a list of words), calculate 
    # the average feature vector for each one and return a 2D numpy array 
    # 
    # Initialize a counter
    counter = 0
    # 
    # Preallocate a 2D numpy array, for speed
    reviewFeatureVecs = np.zeros((len(reviews),num_features),dtype="float64")
    # 
    # Loop through the reviews
    for review in reviews:
       # Print a status message every 1000th review
        if counter%1000 == 0:
            print "Review %d of %d" % (counter, len(reviews))
        # Call the function (defined above) that makes average feature vectors
        reviewFeatureVecs[counter] = makeFeatureVec(review, model, \
           num_features)
        # Increment the counter
        counter = counter + 1
    return reviewFeatureVecs
clean_train_reviews = []
for review in train["review"]:
    clean_train_reviews.append(review_to_wordlist(review, remove_stopwords=True))
    
trainDataVecs = getAvgFeatureVecs(clean_train_reviews, model, num_features)
print "Creating average feature vecs for test reviews"
clean_test_reviews = []
for review in test["review"]:
    clean_test_reviews.append( review_to_wordlist( review, \
        remove_stopwords=True ))

testDataVecs = getAvgFeatureVecs( clean_test_reviews, model, num_features )
np.isnan(testDataVecs)
testDataVecs = np.nan_to_num(testDataVecs)
from sklearn.ensemble import RandomForestClassifier
forest = RandomForestClassifier(n_estimators = 100)
print "Fitting a random forest to labeled training data..."
forest = forest.fit(trainDataVecs, train["sentiment"])
forest.predict(testDataVecs)
output = pd.DataFrame(data={"id":test["id"], "sentiment":result})
output.to_csv( "Word2Vec_AverageVectors.csv", index=False, quoting=3)
from sklearn.cluster import KMeans
import time

start = time.time() # Start time

# Set "k" (num_clusters) to be 1/5th of the vocabulary size, or an
# average of 5 words per cluster
word_vectors = model.wv.syn0
num_clusters = word_vectors.shape[0] / 5

# Initalize a k-means object and use it to extract centroids
kmeans_clustering = KMeans( n_clusters = num_clusters )
idx = kmeans_clustering.fit_predict( word_vectors )

# Get the end time and print how long the process took
end = time.time()
elapsed = end - start
print "Time taken for K Means clustering: ", elapsed, "seconds."
# Create a Word / Index dictionary, mapping each vocabulary word to
# a cluster number                                                                                            
word_centroid_map = dict(zip( model.wv.index2word, idx ))
# For the first 10 clusters
for cluster in xrange(0,10):
    #
    # Print the cluster number  
    print "\nCluster %d" % cluster
    #
    # Find all of the words for that cluster number, and print them out
    words = []
    for i in xrange(0,len(word_centroid_map.values())):
        if( word_centroid_map.values()[i] == cluster ):
            words.append(word_centroid_map.keys()[i])
    print words<jupyter_output>Cluster 0
[u'awaiting', u'preparing']

Cluster 1
[u'airwolf']

Cluster 2
[u'ably']

Cluster 3
[u'bombs', u'boats', u'missiles', u'planes', u'helicopters', u'plates', u'heat', u'rifles', u'tanks', u'plants', u'drilling', u'nuke']

Cluster 4
[u'hordes', u'nazis,', u"american's", u'riders', u'towers', u'pows', u'armies', u'kings', u'disasters', u'knights', u'continent', u'afghanistan', u'vikings', u'rainbow', u'ashes']

Cluster 5
[u'plans.', u'dominating', u'exemplifies', u'dignity.', u'desire.', u'strength.', u'ridicule', u'goal.', u'surely,', u'defense,']

Cluster 6
[u'suspenseful,', u'entertaining,', u'watch.the', u'exciting,', u'well-acted', u'gripping,', u'admittedly', u'thrilling,']

Cluster 7
[u'revolver', u'gun', u'pistol', u'bow', u'weapon,', u'gun.', u'gun,', u'rifle', u'double', u'sniper']

Cluster 8
[u'maclaine', u'barrymore', u'turner', u"davis's", u'tierney', u'lombard', u'co-star', u'lana', u'bette', u'davis', u'katharine', u'garbo', u'judi', u'davis.', u'desmond', u'crawf[...]<jupyter_text>先使用Kmeans得出每一个单词属于的类别有多少种，这可以看成无监督学习出来隐含空间的文本特征属性，每一个类代表一个维度的特征，然后统计一个文本中的单词都属于哪些类的词频作为对应维度的特征值，构成一个DocVector；前面的是直接将词向量简单的做平均来构造一个文档的向量DocVector, 当然也可以乘以tf-idf权重。但是以上的方法都没有考虑单词的位置以及顺序问题。这也是传统方法的缺陷之一。<jupyter_code>def create_bag_of_centroids(wordlist, word_centroid_map):
    num_centroids = max(word_centroid_map.values()) + 1
    bag_of_centroids = np.zeros(num_centroids, dtype="float32")
    for word in wordlist:
        if word in word_centroid_map:
            index = word_centroid_map[word]
            bag_of_centroids[index] += 1
    return bag_of_centroids


# Pre-allocate an array for the training set bags of centroids (for speed)
train_centroids = np.zeros( (train["review"].size, num_clusters), \
    dtype="float32" )

# Transform the training set reviews into bags of centroids
counter = 0
for review in clean_train_reviews:
    train_centroids[counter] = create_bag_of_centroids( review, \
        word_centroid_map )
    counter += 1

# Repeat for test reviews 
test_centroids = np.zeros(( test["review"].size, num_clusters), \
    dtype="float32" )

counter = 0
for review in clean_test_reviews:
    test_centroids[counter] = create_bag_of_centroids( review, \
        word_centroid_map )
    counter += 1
forest = RandomForestClassifier(n_estimators = 100)
print "Fitting the forest may take a few minutes"
forest = forest.fit(train_centroids, train["sentiment"])
result = forest.predict(test_centroids)
output = pd.DataFrame(data = {"id":test["id"], "sentiment":result})
output.to_csv("BagOfCentroids.csv", index=False, quoting=3)<jupyter_output>Fitting the forest may take a few minutes
{'fraction_non_alphanumeric': 0.11231069125805968, 'fraction_numerical': 0.010196431249062828, 'mean_word_length': 4.095492742551566, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '405184', 'license': 'Apache-2.0', 'n_tokens_mistral': 2431, 'n_tokens_neox': 2361, 'n_words': 603, 'path': 'cs224n/kaggleNLP/kaggleNLP.ipynb', 'repo_name': 'kitianFresh/datascience', 'size': '6669'}