<jupyter_start><jupyter_text>Load AML dataset from [ACDC paper](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5447237/pdf/btx054.pdf)...<jupyter_code># load AML data and table

### LOAD DATA ###
path = PATH_DATA + 'AML_benchmark/'
df = pd.read_csv( path + 'AML_benchmark.csv.gz', sep=',', header = 0, \
                 compression = 'gzip', engine='python')
table = pd.read_csv(path + 'AML_table.csv', sep=',', header=0, index_col=0)

### PROCESS: discard ungated events ###
df = df[df.cell_type != 'NotGated']
df = df.drop(['Time', 'Cell_length','file_number', 'event_number', 'DNA1(Ir191)Di',
              'DNA2(Ir193)Di', 'Viability(Pt195)Di', 'subject'], axis = 1)
channels = [item[:item.find('(')] for item in df.columns[:-1]]
df.columns = channels + ['cell_type']
df = df.loc[df['cell_type'] != 'NotDebrisSinglets']

table = table.fillna(0)
X = df[channels].values
table_headers = list(table)

### transform data
data = np.arcsinh((X-1.)/5.)
N, d = data.shape
emp_bounds = np.array([[data[:,d].min(), data[:,d].max()] for d in range(data.shape[1])])
ct2idx = {x:i for i,x in enumerate(table.index)}
idx2ct = [key for idx, key in enumerate(table.index)]
Y = np.array([ct2idx[_] for _ in df.cell_type])

# rename table header 'HLA-DR' to 'HLADR' to prevent error from '-'
temp_headers = list(table)
temp_headers[29] = "HLADR"
table.columns = temp_headers<jupyter_output><empty_output><jupyter_text>Learn MP trees and write accepted samples to file...<jupyter_code>%%time


###################### Parallel run #####################
# n_mcmc_chain = 50
# n_mcmc_samples = 3000
# chains = range(n_mcmc_chain)
# num_cores = multiprocessing.cpu_count()
# accepted_MP = Parallel(n_jobs=num_cores)(delayed(MP_mcmc)\
#                 (data, emp_bounds, table, i, n_mcmc_samples) for i in chains)
# write_chains_to_file(accepted_MP, PATH_SAMPLES)



n_mcmc_chain = 50
n_mcmc_samples = 3000
accepted_MP = []
for i in range(n_mcmc_chain):
    print "Sampling Chain %d..." % i
    accepted_MP.append(MP_mcmc(data, emp_bounds, table, i, n_mcmc_samples))
    burnt_samples = [sample for chain in accepted_MP for sample in chain[-20:]]   
    Y_predict = classify_cells_majority(data, burnt_samples, table, ct2idx)
    accuracy = sum(Y == Y_predict)*1.0/ N
    print "Accuracy of cell classification on all data: %.3f" % (accuracy)

write_chains_to_file(accepted_MP, PATH_SAMPLES)<jupyter_output>Sampling Chain 0...
Accuracy of cell classification on all data: 0.936
Sampling Chain 1...
Accuracy of cell classification on all data: 0.925
Sampling Chain 2...
Accuracy of cell classification on all data: 0.921
Sampling Chain 3...
Accuracy of cell classification on all data: 0.910
Sampling Chain 4...
Accuracy of cell classification on all data: 0.924
Sampling Chain 5...
Accuracy of cell classification on all data: 0.927
Sampling Chain 6...
Accuracy of cell classification on all data: 0.930
Sampling Chain 7...
Accuracy of cell classification on all data: 0.927
Sampling Chain 8...
Accuracy of cell classification on all data: 0.929
Sampling Chain 9...
Accuracy of cell classification on all data: 0.927
Sampling Chain 10...
Accuracy of cell classification on all data: 0.927
Sampling Chain 11...
Accuracy of cell classification on all data: 0.926
Sampling Chain 12...
Accuracy of cell classification on all data: 0.921
Sampling Chain 13...
Accuracy of cell classification on all data: 0.923
Sa[...]<jupyter_text>Classify cells based on accepted MP trees, and write predictions to file...<jupyter_code>burnt_samples = [sample for chain in accepted_MP for sample in chain[-10:]]
Y_predict = classify_cells_majority(data, burnt_samples, table, ct2idx)
accuracy = sum(Y == Y_predict)*1.0/ N
print "Accuracy of cell classification: %.3f" % (accuracy)

df['MP_prediction'] = pd.Series([idx2ct[i] for i in Y_predict], index=df.index)
df.to_csv(FILENAME_PREDICTIONS, compression='gzip', index = False)
print [len(i) for i in accepted_MP]
# compute accuracy of each sample on average<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13981990995497748, 'fraction_numerical': 0.0335167583791896, 'mean_word_length': 4.745689655172414, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '521018', 'license': 'MIT', 'n_tokens_mistral': 1503, 'n_tokens_neox': 1373, 'n_words': 410, 'path': 'run/flowMP-AML-run-v1.0.ipynb', 'repo_name': 'disiji/fc_mondrian', 'size': '3998'}