<jupyter_start><jupyter_text>livedoor ニュースコーパスをダウンロードおよび展開する。ここでは、`./data/text/{ category }/*.txt` にあるとする。<jupyter_code>categories = [
    "dokujo-tsushin",
    "it-life-hack",
    "kaden-channel",
    "livedoor-homme",
    "movie-enter",
    "peachy",
    "smax",
    "sports-watch",
]

corpus = []

for category in categories:
    for textfile_path in glob.glob(f"./data/text/{ category }/*.txt"):
        try:
            with open(textfile_path, "rt") as fin:
                document = {
                    "category": category,
                    "text": fin.read().strip().split("\n")[2],
                }
                corpus.append(document)
        except:
            continue
df = pd.DataFrame(corpus)
df.head(3)
df["tfidf"] = (
    df['text']
    .pipe(ja_hero.tokenize)
    .pipe(hero.tfidf)
)
df['kmeans_labels'] = (
    df['tfidf']
    .pipe(hero.kmeans, n_clusters=8)
    .astype(str)
)
df['pca'] = df['tfidf'].pipe(hero.pca)
hero.scatterplot(df, 'pca', color='category', title="K-means Livedoor News Corpus")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.176248821866164, 'fraction_numerical': 0.002827521206409048, 'mean_word_length': 2.504950495049505, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '607457', 'license': 'MIT', 'n_tokens_mistral': 428, 'n_tokens_neox': 413, 'n_words': 59, 'path': 'examples/example.ipynb', 'repo_name': 'kanjirz50/ja_texthero', 'size': '1061'}