<jupyter_start><jupyter_text>https://www.cs.cornell.edu/people/tj/publications/joachims_98a.pdfhttp://www.daviddlewis.com/resources/testcollections/reuters21578/readme.txt  Explore Reuters-21578  Load dataset<jupyter_code>from ds_tutorial.datasets import ReutersCorpus
data_root = Path.home() / "data" / "tmp"
reuters_documents_path = data_root / "reuters21578" / "documents.pkl"
reuters_corpus_path = data_root / "reuters21578" / "corpus.pkl"
documents = pickle.load(open(reuters_documents_path, "rb"))
reuters = pickle.load(open(reuters_corpus_path, "rb"))
df, top_ten_ids, train_labels, test_labels = reuters.build_dataframe()
train, test = reuters.split_modapte()<jupyter_output><empty_output><jupyter_text>Get some simple stats<jupyter_code>number_of_samples = reuters.number_of_samples
number_of_classes = reuters.number_of_classes
number_of_samples_per_class = int(np.average([tc for tc in reuters.topic_counts.values() if tc > 1]))
number_of_words_per_sample = int(np.median([len(d["text"].split()) for d in reuters.docs]))
samples_to_words_per_sample_ratio = int(number_of_samples / number_of_words_per_sample)
nchars = 52
print("Number of samples:".ljust(nchars), reuters.number_of_samples)
print("Number of classes:".ljust(nchars), reuters.number_of_classes)
print("Number of samples per class:".ljust(nchars), number_of_samples_per_class)
print("Number of words per sample:".ljust(nchars), number_of_words_per_sample)
print("Number of samples/number of words per sample ratio:".ljust(nchars), samples_to_words_per_sample_ratio)<jupyter_output>Number of samples:                                   10789
Number of classes:                                   119
Number of samples per class:                         148
Number of words per sample:                          89
Number of samples/number of words per sample ratio:  121<jupyter_text>Distribution of sample length for reuters21578<jupyter_code>fig, ax = plt.subplots(figsize=(15, 10))
sns.distplot([len(d["text"]) for d in reuters.docs], kde=True, ax=ax)
ax.set_title('Sample length distribution')
ax.set_xlabel('Length of a sample')
ax.set_xlim(0, 8000)
_ = ax.set_ylabel('Number of samples')<jupyter_output><empty_output><jupyter_text>Word frequency distribution<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
kwargs = {
        'ngram_range': (1, 1),
        'dtype': 'int32',
        'strip_accents': 'unicode',
        'decode_error': 'replace',
        'analyzer': 'word',  # Split text into word tokens.
}
vectorizer = CountVectorizer(**kwargs)
vectorized_texts = vectorizer.fit_transform(reuters.texts)
all_ngrams = list(vectorizer.get_feature_names())
all_counts = vectorized_texts.sum(axis=0).tolist()[0]
all_counts, all_ngrams = zip(*[(c, n) for c, n in sorted(
    zip(all_counts, all_ngrams), reverse=True)])

num_ngrams=50
ngrams = list(all_ngrams)[:num_ngrams]
counts = list(all_counts)[:num_ngrams]
idx = np.arange(num_ngrams)
fig, ax = plt.subplots(figsize=(15, 10))
sns.barplot(x=ngrams, y=counts, ax=ax, color="#95BCD9")
ax.set_title('Sample length distribution')
plt.xlabel('N-grams')
_ = plt.ylabel('Frequencies')
_ = plt.title('Frequency distribution of n-grams')
_ = plt.xticks(idx, ngrams, rotation=45)<jupyter_output><empty_output><jupyter_text>Choose a model flowchart<jupyter_code>from IPython.display import Image
from IPython.core.display import HTML
print("Number of samples/number of words per sample ratio: ", samples_to_words_per_sample_ratio)
Image(url=(
    "https://developers.google.com/machine-learning/guides/"
    "text-classification/images/TextClassificationFlowchart.png"
))<jupyter_output>Number of samples/number of words per sample ratio:  121
{'fraction_non_alphanumeric': 0.11994572591587517, 'fraction_numerical': 0.0203527815468114, 'mean_word_length': 5.617594254937163, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '446704', 'license': 'MIT', 'n_tokens_mistral': 1357, 'n_tokens_neox': 1257, 'n_words': 244, 'path': 'notebooks/text_classification/explore_data.ipynb', 'repo_name': 'ephes/data_science_tutorial', 'size': '3685'}