<jupyter_start><jupyter_text>week6_hw PCA and K-means<jupyter_code>import jieba
import os
import pandas as pd
import func
from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfVectorizer
from datetime import timedelta, date
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.decomposition import PCA as sklearnPCA
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
import plotly
plotly.tools.set_credentials_file(username='mnbvv', api_key='P8WyPDldauQXVLtxYecA')
import plotly.plotly as py
import plotly.graph_objs as go
def is_chinese(uchar):         
    if u'\u4e00' <= uchar<=u'\u9fff':
        return True
    else:
        return False
# 專有名詞
jieba.load_userdict('dict.txt')
folders = ["金融業/"]
def get_txts(folder):
    all_files = os.listdir("KAM/" + folder)   # imagine you're one directory above test dir
    return all_files
# 讀取新聞檔案，並使用jieba切詞
def textMining(folder, file):
    with open("./KAM/" + folder + file, 'rt',encoding="utf-8") as txt:
        data = txt.read()
    #text segmentation
    seg_list = jieba.cut(data, cut_all=False)
    seg_list = list(seg_list)
    filter_list = []
    wordDict = {}
    jump = False
    for word in seg_list:
        for s in word:
            if not is_chinese(s):
                jump = True
                break
        if not jump:
            filter_list.append(word)
        jump = False
    return filter_list
def analyze(folder, corpus, txts_in_folder):
    # tfidf
    vectorizer = TfidfVectorizer(max_df = 0.9, min_df = 0.2)
    tfidf = vectorizer.fit_transform(corpus)
    words = vectorizer.get_feature_names()
    print("tfidf.shape: ", tfidf.shape)
    for i in range(len(corpus)):
        print('----{0} KAM----'.format(txts_in_folder[i]))
        for j in range(len(words)):
            if tfidf[i,j] > 0.2:
                print(words[j], tfidf[i,j], i ,j)
    
    X = tfidf.toarray()
    len(X)
    X_std = StandardScaler().fit_transform(X)
    sklearn_pca = sklearnPCA(n_components = 2)
    Y_sklearn = sklearn_pca.fit_transform(X_std)
    Y_sklearn[:3]
    
    return Y_sklearn
def plot_cpa_scatter(Y_sklearn, txts_in_folder):
    pca_data = [
        go.Scatter(
            x = Y_sklearn[:,0],
            y = Y_sklearn[:,1],
            mode = "markers",
            hoverinfo = 'text',
            text = txts_in_folder
        )
    ]

    fig = go.Figure(data = pca_data)
    return py.iplot(fig, filename = 'PCA Scatter Chart')
def plot_kmeans_scatter(Y_sklearn, txts_in_folder):
    kmeans = KMeans(n_clusters = 3)
    X_clustered = kmeans.fit_predict(Y_sklearn)
    
    #Define our own color map
    LABEL_COLOR_MAP = {0:'red', 1: 'green', 2: 'blue'}
    label_color = [LABEL_COLOR_MAP[l] for l in X_clustered]
    
    pca_data = [
        go.Scatter(
            x = Y_sklearn[:,0],
            y = Y_sklearn[:,1],
            mode = "markers",
            hoverinfo = 'text',
            text = txts_in_folder,
            marker = dict(color = label_color)
        )
    ]

    fig = go.Figure(data = pca_data)
    return py.iplot(fig, filename = 'K-means Scatter Chart')<jupyter_output><empty_output><jupyter_text>分析大同20180102資料夾<jupyter_code># get corpus from folder 大同20180102
txts_in_folder1 = get_txts(folders[0])
corpus1 = []

for file_name in txts_in_folder1:
    filter_list = textMining(folders[0], file_name)
    join_list = " ".join(filter_list)
    corpus1.append(join_list)

Y_sklearn1 = analyze(folders[0], corpus1, txts_in_folder1)
plot_cpa_scatter(Y_sklearn1, txts_in_folder1)
plot_kmeans_scatter(Y_sklearn1, txts_in_folder1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10712343665035345, 'fraction_numerical': 0.01468189233278956, 'mean_word_length': 2.998913043478261, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '353595', 'license': 'MIT', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1365, 'n_words': 279, 'path': 'FinaceKmeans.ipynb', 'repo_name': 'mnbvv/newpython', 'size': '3678'}