<jupyter_start><jupyter_text>Duplicate Dataset Analysis TemplateAuthors: <NAME>, <NAME>Write intro stuff here<jupyter_code>import dupedat.simulated_dataset_gen.create_simulated_datasets as csd
import dupedat.simulated_dataset_gen.example_simulated_datasets as esd
import dupedat.method as methods
import pandas as pd
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Table of Contents- [Import dataset](Import-dataset)- Remove extreme outliers- Check Assumptions- Get outliers  Import dataset - Instuctions for import- Specify structure of dataset- Show head of dataset so they can check- Custom error for incorrect formatting- Instructions on how to import pre build data set<jupyter_code># Use to load your own data
fp_df = input('\nInput data filepath\n\n')

dataset = esd.test_wrapper.abs_rel(True)
dataset["hi"] = dataset["true_value"]*2
dataset<jupyter_output><empty_output><jupyter_text>Remove extreme outliers<jupyter_code>duplicate_dataset = methods.duplicate_datasets.DuplicateDataset(dataset, 'replicate_1', 'replicate_2')
duplicate_dataset.create_dif_mean_plot()<jupyter_output><empty_output><jupyter_text>Code to remove points in a list<jupyter_code>vm_list = [methods.sd_models.RCSMethod] # CSMethod, RCSMethod, RZMethod, ZMethod]  # HNMethod, RZMethod, SDMethod



for method in vm_list:
    print(method)
    vm = method(duplicate_dataset, 0.95)
    print(vm.get_gaussian_sd_params())
    print('~*~*~*~*~*~**~*~*~*~*~*~*~')

    sdt_outlier = methods.outlier_detection.SDTMethod(vm)
    fig_outlier = sdt_outlier.display_outliers()

    plt.show(block=fig_outlier)
duplicate_dataset.dataframe<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10096153846153846, 'fraction_numerical': 0.003605769230769231, 'mean_word_length': 7.2835820895522385, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '66572', 'license': 'MIT', 'n_tokens_mistral': 581, 'n_tokens_neox': 547, 'n_words': 121, 'path': 'Duplicate Dataset Analysis Template.ipynb', 'repo_name': 'seperry83/dds-notebook', 'size': '1675'}