<jupyter_start><jupyter_text>Further Filtering dummy candidates by similarity between real data<jupyter_code>real_melt_temps = list(map(melt_temp, list(real['sgRNA-1'])))
real_gc_counts = list(map(calc_gc, list(real['sgRNA-1'])))
dumy['melting_temp'] = pd.Series(map(melt_temp, list(dumy['sgRNA'])))
dumy['gc_counts'] = pd.Series(map(calc_gc, list(dumy['sgRNA'])))
dumy = dumy[((dumy['melting_temp'] > min(real_gc_counts)) & 
             (dumy['melting_temp'] < max(real_gc_counts))) & 
             ((dumy['gc_counts'] > min(real_melt_temps)) & 
             (dumy['gc_counts'] < max(real_melt_temps))) ].reset_index(drop=True)
dumy.shape<jupyter_output><empty_output><jupyter_text>Check the Variables and Look for Patterns---  sgRNA_cut<jupyter_code>real['sgRNA_cut'] = real['sgRNA_cut'].map({'Yes': 1, 'No': 0})
pos_ratio = real[real['sgRNA_cut'] == 1].shape[0] / len(real['sgRNA_cut'])
neg_ratio = real[real['sgRNA_cut'] == 0].shape[0] / len(real['sgRNA_cut'])
print(pos_ratio, neg_ratio)
# Create artificial data points by keeping the real rations
pos_lst = []
neg_lst = []
pos_lst = [1] * int(np.ceil(pos_ratio*dumy.shape[0]))
neg_lst = [0] * int(np.floor(neg_ratio*dumy.shape[0]))
cut_lst = pos_lst + neg_lst
random.shuffle(cut_lst)<jupyter_output><empty_output><jupyter_text>num_worms<jupyter_code># Getting min and max of real data 
a, b = min(real['num_worms']), max(real['num_worms'])
# Getting mean and standard deviation of real data
mu, sigma = np.mean(real['num_worms']), np.std(real['num_worms'])
# Create the distribution object
dist = truncnorm((a - mu) / sigma, (b - mu) / sigma, loc=mu, scale=sigma)
# Creat data size number of values
values = dist.rvs(dumy.shape[0])
num_worms_lst = [int(i) for i in values]<jupyter_output><empty_output><jupyter_text>num_lines<jupyter_code># Getting min and max of real data 
a, b = min(real['num_lines']), max(real['num_lines'])
# Getting mean and standard deviation of real data
mu, sigma = np.mean(real['num_lines']), np.std(real['num_lines'])
# Create the distribution object
dist = truncnorm((a - mu) / sigma, (b - mu) / sigma, loc=mu, scale=sigma)
# Creat data size number of values
values = dist.rvs(dumy.shape[0])
num_lines_lst = [int(i) for i in values]
# Adding the simulated lists into dataframe
dumy['sgRNA_cut'] = pd.Series(cut_lst)
dumy['num_worms'] = pd.Series(num_worms_lst)
dumy['num_lines'] = pd.Series(num_lines_lst)

# Reorganize the columns
dumy = dumy[['sgRNA', 'target_name', 'sgRNA_cut', 'num_worms', 'num_lines',
                             'melting_temp', 'gc_counts', 'true_PAM']]

# Drop the unused columns 
dumy = dumy.drop(['melting_temp', 'gc_counts', 'true_PAM'], axis=1)
dumy.head()
dumy.to_csv('./data/new/artificial_sgRNA_c_elegans_data.csv', index=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16642651296829972, 'fraction_numerical': 0.005403458213256484, 'mean_word_length': 6.048223350253807, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '630965', 'license': 'MIT', 'n_tokens_mistral': 1135, 'n_tokens_neox': 1079, 'n_words': 209, 'path': 'artificial_data/test.ipynb', 'repo_name': 'hackseq/2017_project_8', 'size': '2776'}