<jupyter_start><jupyter_text>Synthetic Data Generation<jupyter_code>import numpy as np
import pandas as pd
import tqdm

from topicnet.cooking_machine import Dataset<jupyter_output><empty_output><jupyter_text>Generating Texts<jupyter_code>NUM_HIGHLY_RELATED_WORDS = 10
NUM_WORDS_IN_TOPIC = NUM_HIGHLY_RELATED_WORDS

NUM_WORDS_IN_DOCUMENT = 1000
MAIN_TOPIC_MINIMUM_PROBABILITY = 0.8
# Fixing the total number of documents here
# Another variant would be to fix the number of documents in each topic
# (and vary the total number of documents)

NUM_DOCUMENTS = 1000

NUM_TOPICS = 100
NUM_DOCUMENTS_IN_TOPIC = NUM_DOCUMENTS // NUM_TOPICS

assert NUM_DOCUMENTS_IN_TOPIC * NUM_TOPICS == NUM_DOCUMENTS

NUM_TOPICS_IN_DOCUMENT = min(NUM_TOPICS, 3)
MAIN_TOPIC_PROBABILITY = None
SECONDARY_TOPIC_PROBABILITY = None

NUM_SECONDARY_TOPICS_IN_DOCUMENT = NUM_TOPICS_IN_DOCUMENT - 1

if NUM_SECONDARY_TOPICS_IN_DOCUMENT > 0:
    MAIN_TOPIC_PROBABILITY = MAIN_TOPIC_MINIMUM_PROBABILITY
    SECONDARY_TOPIC_PROBABILITY = (1.0 - MAIN_TOPIC_MINIMUM_PROBABILITY) / NUM_SECONDARY_TOPICS_IN_DOCUMENT
else:
    MAIN_TOPIC_PROBABILITY = 1.0
    SECONDARY_TOPIC_PROBABILITY = 0.0
topics = [f't_{i:02}' for i in range(NUM_TOPICS)]
topics[:10]<jupyter_output><empty_output><jupyter_text>Defining each topic's words<jupyter_code>topic_words = {}

for t in topics:
    words = []
    words += [f'w__{t}__{i}' for i in range(NUM_HIGHLY_RELATED_WORDS)]
    
    # TODO: add background words, NUM_BACKGROUND_WORDS
    
    topic_words[t] = words
topic_words[topics[0]]
len(topics)<jupyter_output><empty_output><jupyter_text>Constructing the documents<jupyter_code>documents = []

# TODO: memory inefficient
document_texts = {}
document_topics = {}

document_index = 0


for main_topic_index, main_topic in tqdm.tqdm(
        enumerate(topics), total=len(topics)):

    topics_except_main = topics[:main_topic_index] + topics[main_topic_index+1:]
    
    for i in range(NUM_DOCUMENTS_IN_TOPIC):
        secondary_topics = list(np.random.choice(
            topics_except_main,
            max(NUM_TOPICS_IN_DOCUMENT - 1, 0),
            replace=False
        ))

        doc = []

        doc_topics = [main_topic] + secondary_topics
        topic_probabilities = (
            [MAIN_TOPIC_PROBABILITY]
            + [SECONDARY_TOPIC_PROBABILITY] * len(secondary_topics)
        )

        word_topics = np.random.choice(
            doc_topics, NUM_WORDS_IN_DOCUMENT, p=topic_probabilities)

        for current_topic in word_topics:
            # TODO: add probabilities, for background words
            current_word = np.random.choice(topic_words[current_topic], 1)[0]
            doc.append(current_word)
        
        document_id = f'd_{document_index:04}'
        
        documents.append(document_id)
        document_texts[document_id] = doc
        document_topics[document_id] = doc_topics
        
        document_index += 1
documents[:5]
document_topics[list(document_texts.keys())[0]]
document_texts[documents[0]][:10]<jupyter_output><empty_output><jupyter_text>Phi & Theta Matrices All the unique words (of all topics)<jupyter_code>vocabulary = []

for words in topic_words.values():
    vocabulary += words
len(vocabulary)<jupyter_output><empty_output><jupyter_text>Filling a word-topic matrix Phi<jupyter_code>phi = pd.DataFrame(
    index=vocabulary,
    columns=topics,
    data=np.zeros((len(vocabulary), len(topics)), dtype=float)
)
phi.shape
phi.head()
for topic in topics:
    words = topic_words[topic]
    word_probability = 1.0 / len(words)
    phi.loc[words, topic] = word_probability

# TODO:  + background words
phi.head(15)<jupyter_output><empty_output><jupyter_text>...and a topic-document matrix Theta<jupyter_code>theta = pd.DataFrame(
    index=topics,
    columns=documents,
    data=np.zeros((len(topics), len(documents)))
)
theta.head()
document_topics['d_0000']
for doc, doc_topics in document_topics.items():
    theta.loc[doc_topics, doc] = [MAIN_TOPIC_PROBABILITY] + \
        [SECONDARY_TOPIC_PROBABILITY] * (len(doc_topics) - 1)
theta.head()
theta[theta['d_0000'] > 0]<jupyter_output><empty_output><jupyter_text>Saving the matrices<jupyter_code>phi.to_csv(f'phi_{len(topics)}.csv')
theta.to_csv(f'theta_{len(topics)}.csv')<jupyter_output><empty_output><jupyter_text>Dataset Here we are making a TopicNet's Dataset<jupyter_code>DEFAULT_MODALITY = '@text'
dataset = pd.DataFrame()

dataset['id'] = documents
dataset['raw_text'] = [
    ' '.join(document_texts[d]) for d in documents
]
dataset['vw_text'] = [
    f'{d} |{DEFAULT_MODALITY} ' + " ".join(document_texts[d]) for d in documents
]
dataset.shape
dataset.head()
dataset.iloc[0, 1][:250]
dataset_path = f'/data/datasets/_tmp_alexeev/SyntheticDataset_{len(topics)}.csv'

dataset.to_csv(dataset_path, index=False)<jupyter_output><empty_output><jupyter_text>Checking if all OK<jupyter_code>saved_dataset = Dataset(dataset_path)
saved_dataset._data.head()
del dataset
del saved_dataset<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10499899618550491, 'fraction_numerical': 0.012447299739008232, 'mean_word_length': 4.909845788849347, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '447576', 'license': 'MIT', 'n_tokens_mistral': 1894, 'n_tokens_neox': 1813, 'n_words': 300, 'path': 'experiments/Synthetic-Data-Generation.ipynb', 'repo_name': 'machine-intelligence-laboratory/OptimalNumberOfTopics', 'size': '4981'}