<jupyter_start><jupyter_text>General configuration<jupyter_code>REDDIT_MAIN_PATH = join('E:', sep, 'Datasets', 'reddit')
EXPERIMENT_PATH = 'Experiment'
RAW_DATA_PATH = join(EXPERIMENT_PATH, 'Raw_data')
RAW_NETWORKS_PATH = join(RAW_DATA_PATH, 'RawCommSizes')

from ExperimentSettings import EXPERIMENT_SETTINGS<jupyter_output><empty_output><jupyter_text>Create experiment directories<jupyter_code>EXPERIMENT_SETTINGS
group_sizes = ['min', 'quantile10', 'quartile1', 'median', 'random']
create_experiment_directories(base_dir=EXPERIMENT_PATH, network_generator_config=EXPERIMENT_SETTINGS, group_sizes = group_sizes)<jupyter_output><empty_output><jupyter_text>Generate community sizes population from Reddit network  Configuration<jupyter_code>COMM_SIZES_FILE_PATH = join(RAW_DATA_PATH, 'reddit_comm_sizes.json')<jupyter_output><empty_output><jupyter_text>Generate community sizes population from Reddit network<jupyter_code>comm_sizes_fetcher = RedditCommunitySizesFetcher(REDDIT_MAIN_PATH)
comm_sizes_fetcher.create_comm_size_dict(use_edges_csv=True);
comm_sizes_fetcher.save_comm_size_dict_json(COMM_SIZES_FILE_PATH)<jupyter_output><empty_output><jupyter_text>Generate normal communities sizes<jupyter_code>NORMAL_COMMS_GENERATOR_CONFIG = {
    'num_comms': 110,
    'min_comm_size': 30,
    'max_comm_size': 1500
}

min_seed = 1
max_seed = 5<jupyter_output><empty_output><jupyter_text>Data Creation<jupyter_code>normal_comm_sizes_generator = NormalCommunitySizesGenerator(comm_sizes_file_path=COMM_SIZES_FILE_PATH)

for seed in range(min_seed, max_seed + 1):
    print(f'Sampling normal community sizes {seed} / {max_seed}:')
    comm_sizes_list = normal_comm_sizes_generator.generate_community_sizes_from_reddit(
        **NORMAL_COMMS_GENERATOR_CONFIG, random_seed=seed)
    
    file_path = join(RAW_DATA_PATH, f'comm_sizes_{seed:02}.json')
    with open(file_path, 'w') as file:
        json.dump(comm_sizes_list, file)<jupyter_output>Sampling normal community sizes 1 / 5:
Sampling normal community sizes 2 / 5:
Sampling normal community sizes 3 / 5:
Sampling normal community sizes 4 / 5:
Sampling normal community sizes 5 / 5:<jupyter_text>Create anomaly-infused networks  Configuration<jupyter_code>TRAIN_TEST_SPLIT = 20  # number of communities in test set<jupyter_output><empty_output><jupyter_text>Netowrks creation<jupyter_code>create_experiment_networks(
    base_dir = EXPERIMENT_PATH,
    raw_comm_sizes_dir_path = RAW_NETWORKS_PATH,
    train_test_split_num=TRAIN_TEST_SPLIT,
    experiment_settings=EXPERIMENT_SETTINGS,
    num_anom_comms=10,
    verbose=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0854733614411652, 'fraction_numerical': 0.011881947106170947, 'mean_word_length': 9.1953125, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '615394', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 970, 'n_tokens_neox': 936, 'n_words': 137, 'path': 'Evaluation/GeneratdNetwork/01__Networks_creation.ipynb', 'repo_name': 'lapidshay/GenericAnomalousCommunitiesDetection', 'size': '2609'}