<jupyter_start><jupyter_text>Using the twitter 'hot-list' papers as a training-set, build a classifier to detect which papers will be the next hot thing.1. divide every paper into paragraphs.2. run LDA/LSA on each paragraph, to get a topic list and compute topic similarity with new documents, or:3. user LDA2VEC on the documents and calculate the vector distance to new documents.4. Train SVM on the topics/vectors as features & binary/score output<jupyter_code>import pandas as pd
import numpy as np
import glob
import nltk
import re
import urllib.request
import ujson

from pathlib import Path

from nltk.stem.wordnet import WordNetLemmatizer
from gensim.corpora import Dictionary, HashDictionary, MmCorpus
from gensim import models, utils, similarities
nltk.download('punkt')
nltk.download('stopwords')
SPECIAL_CHARS = '[^A-Za-z0-9 ]+'
stopwords = nltk.corpus.stopwords.words('english')
wordnet_lemmatizer = WordNetLemmatizer()

def preprocess(text):
    tokens = [re.sub(SPECIAL_CHARS, '', word.lower()) for word in nltk.word_tokenize(text)]
    tokens = [wordnet_lemmatizer.lemmatize(word) for word in tokens if word not in stopwords]
    tokens = list(filter(None, tokens))
    return tokens
class Corpus(object):
    def __iter__(self):
        for file in glob.glob("data/*.txt"):
            print(file)
            paper = Path(file).read_text(encoding='utf8')
            yield paper<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08771929824561403, 'fraction_numerical': 0.005614035087719298, 'mean_word_length': 4.5271317829457365, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '597621', 'license': 'MIT', 'n_tokens_mistral': 448, 'n_tokens_neox': 437, 'n_words': 149, 'path': 'notebooks/Document similarity/Document similarity using LDA + SVM.ipynb', 'repo_name': 'Keep-Current/Research', 'size': '1425'}