<jupyter_start><jupyter_text>Length of dataset<jupyter_code>df_list.head()
df_list.dropna(axis=0,how='any',inplace=True)
df_concat = df_list['title'] + ' ' + df_list['content']
df_list['doc'] = df_concat.map(lambda d: d.replace('-','_'))
print "Total length = %s"%df_list.shape[0]
print "Length of null data = %s"%df_list[df_list['doc'].isnull()].shape[0]
print "Length of null data = %s"%df_list[df_list['tags'].isnull()].shape[0]<jupyter_output>Length of null data = 0<jupyter_text>Number of unique tags in training datasets<jupyter_code>df_list['tags'][0:10]
# Y = [item.replace("-","_").split(" ") for item in df_list['tags']]
all_tags = df_list['tags'].map(lambda d: d.replace('-','_'))
length_of_tags = pd.DataFrame(zip(*np.unique([len(tag) for tag in Y],return_counts=True)),columns=['length','count'])
%matplotlib inline
# ax = plt.subplot()
# ax.plot(length_of_tags['length'],label='')
ax = length_of_tags.set_index("length").plot(kind='bar',rot=0,title="Length of tags")<jupyter_output><empty_output><jupyter_text>Analyze type of data<jupyter_code>type_of_tags = []

def get_post_tag(tags):
    return np.array(nltk.pos_tag(tags.split(' ')))[:,1]

for tag in df_list.tags:
    type_of_tags.extend(get_post_tag(tag))

count_tag = pd.DataFrame(zip(*np.unique(type_of_tags,return_counts=True)),columns=['tag','count'])
%matplotlib inline
count_tag.sort_values(['count'],ascending=False).iloc[0:10].set_index(['tag']).plot(kind='bar',title="Type of tag",rot=0)<jupyter_output><empty_output><jupyter_text>Number of tags in title<jupyter_code>def remove_number(demo):
    a = []
    for f in demo.split(' '):
        if not(f.isdigit()):
            a.append(f)
    return a
# token_pattern='^[0-9]*'
word_title = [remove_number(item) for item in df_list['title']]
title_dict = list(set(sum(word_title,[])))
title_dict = set(title_dict)
number_of_tags_in_title = title_dict.intersection(tags)
print "Ratio of tags in title = %s %%"%(len(number_of_tags_in_title)*100/len(all_tags))<jupyter_output>Ratio of tags in title = 1 %<jupyter_text>TF-IDF + LDA<jupyter_code>tfidfVec = TfidfVectorizer(max_df=0.95)
countVec = CountVectorizer(max_df=0.95,min_df=2,ngram_range=(1,2),token_pattern='[a-zA-Z]{2,}')
df_tf = countVec.fit_transform(df_list['doc'])
n_topics = 6
lda = LatentDirichletAllocation(n_topics=n_topics, max_iter=5,
                                learning_method='online',
                                learning_offset=50.,
                                random_state=4111,n_jobs=2)
lda.fit(df_tf)
def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        print("Topic #%d:" % topic_idx)
        print(" ".join([feature_names[i]
                        for i in topic.argsort()[:-n_top_words - 1:-1]]))
    print()
print("\nTopics in LDA model:")
n_top_words = 30
tf_feature_names = countVec.get_feature_names()
print_top_words(lda, tf_feature_names, n_top_words)
idx = lda.transform(df_tf[0]).argmax()

# words_topic = zip(*(lda.components_[3],countVec.get_feature_names()))
words_topic = lda.components_[idx]
feature_names = countVec.get_feature_names()
n_words = 100

print [feature_names[i] for i in words_topic.argsort()[:-n_words - 1:-1]]
tokenizer = countVec.build_analyzer()
a = tokenizer(df_list['title'].iloc[0])
b = [(c,words_topic[countVec.vocabulary_[c]]) for c in a if countVec.vocabulary_.has_key(c)]
df_list['tags'].iloc[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16966357308584687, 'fraction_numerical': 0.015081206496519721, 'mean_word_length': 5.736328125, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '460721', 'license': 'Apache-2.0', 'n_tokens_mistral': 1334, 'n_tokens_neox': 1309, 'n_words': 200, 'path': 'QuickLookAtData.ipynb', 'repo_name': 'NhuanTDBK/Kaggle_StackedOverflow', 'size': '3448'}