<jupyter_start><jupyter_text>Classification onwards from here<jupyter_code>from sklearn import svm
ds = load_dataset(location="../../datasets/radioml.dat")
# bpsk_18 = ds[("BPSK", 18)]
# qpsk_18 = ds[("QPSK", 18)]
# classes = {"bpsk": 0, "qpsk": 1}
from tqdm import tqdm

def wrapper_cyclo(data):
    """Presumes data is n datapoints."""
    n, _ = data.shape
    specs = np.zeros((n, 128), dtype = 'complex')
    for i in tqdm(range(n)):
#     for i in range(n):
        a, _ = cyclo_stationary(data[i, :])
        specs[i, :] = convert_to_1d(a)
#         print(specs[0])
    return specs
# snr_vals = list(map(lambda a: a[1], list(filter(lambda a:a[0] == 'BPSK' and a[1] != 18, list(ds.keys()), ))))
# snr_vals = [16,14, 12, 10, 8, 6]
# bpsk = ds[("BPSK", 18)]
# qpsk = ds[("QPSK", 18)]
# for i in snr_vals:
#     bpsk = np.vstack([ds[("BPSK", i)], bpsk])
#     qpsk = np.vstack([ds[("QPSK", i)], qpsk])
# bpsk = np.add(bpsk[:, 0, :], 1j*bpsk[:, 1, :])
# qpsk = np.add(qpsk[:, 0, :], 1j*qpsk[:, 1, :])
# bpsk_features = wrapper_cyclo(bpsk)
# qpsk_features = wrapper_cyclo(qpsk)
# def create_train_test(data1, data2, shuffle=False):
#     training_set = 0.7
#     n1, _ = data1.shape
#     n2, _ = data2.shape
#     num_train_1 = int(n1 * training_set)
#     num_train_2 = int(n2 * training_set)

#     if shuffle:
#         np.random.shuffle(data1)
#         np.random.shuffle(data2)

#     train_q = data1[:num_train_1, ]
#     train_b = data2[:num_train_2, ]
#     test_q = data1[num_train_1:, ]
#     test_b = data2[num_train_2:, ]

#     train_x = np.vstack((train_q, train_b))
#     train_y = np.vstack((np.zeros((num_train_1, 1)), np.ones((num_train_2, 1)))) # data1 - 0, data2 - 1

#     test_x = np.vstack((test_q, test_b))
#     test_y = np.vstack((np.zeros((n1 - num_train_1, 1)), np.ones((n2 - num_train_2, 1))))

#     # Time to mix the data
#     together = np.hstack((train_x, train_y))
#     if shuffle:
#         np.random.shuffle(together)
        
#     train_x, train_y = np.hsplit(together, [together.shape[1] - 1])

#     together = np.hstack((test_x, test_y))
    
#     if shuffle:
#         np.random.shuffle(together)
    
#     test_x, test_y = np.hsplit(together, [together.shape[1] - 1])
#     return train_x, train_y, test_x, test_y
# def percentage_success(actual, predicted):
#     return 1 - (np.linalg.norm(actual - predicted)**2 / (predicted.shape[0]))

from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier

def gamut_classification(train_x, train_y, test_x, test_y):
    train_y = train_y.reshape([-1,])
    clf = SVC(C = 1.0, kernel='poly')
    clf.fit(train_x, train_y )
    predicted = clf.predict(test_x).reshape((-1, 1))
    print("Polynomial Kernel, SVM ", percentage_success(test_y, predicted))
    clf = SVC(C = 1.0, kernel='rbf')
    clf.fit(train_x, train_y)
    predicted = clf.predict(test_x).reshape((-1, 1))
    print("RBF Kernel, SVM ", percentage_success(test_y, predicted))
    clf = SVC(C = 1.0, kernel='linear')
    clf.fit(train_x, train_y)
    predicted = clf.predict(test_x).reshape((-1, 1))
    print("Linear Kernel, SVM ", percentage_success(test_y, predicted))
    clf = RandomForestClassifier(max_depth=2, n_estimators=100)
    clf.fit(train_x, train_y)
    predicted = clf.predict(test_x).reshape((-1, 1))
    print("Random Forests ", percentage_success(test_y, predicted))
    clf = AdaBoostClassifier(n_estimators=100)
    clf.fit(train_x, train_y)
    predicted = clf.predict(test_x).reshape((-1, 1))
    print("AdaBoost ", percentage_success(test_y, predicted))
# train_x, train_y, test_x, test_y = create_train_test(bpsk_features, qpsk_features)

# train_x_real_imag = np.hstack([train_x.real, train_x.imag])
# test_x_real_imag = np.hstack([test_x.real, test_x.imag])

# train_x_mag_phase = np.hstack([np.abs(train_x), np.angle(train_x)])
# test_x_mag_phase = np.hstack([np.abs(test_x), np.angle(test_x)])


# print("Using real imaginary")
# gamut_classification(train_x_real_imag, train_y, test_x_real_imag, test_y)
# print("")
# print("Using mag phase")
# gamut_classification(train_x_mag_phase, train_y, test_x_mag_phase, test_y)

# # print("")
# print("Real only")
# gamut_classification(train_x, train_y, test_x, test_y)<jupyter_output><empty_output><jupyter_text>We have the following results for QPSK vs BPSK snrs upto 6: Using real imaginary Polynomial Kernel, SVM  0.5 RBF Kernel, SVM  0.500714285714 Linear Kernel, SVM  0.499285714286 Random Forests  0.697857142857 AdaBoost  0.855 Using mag phase Polynomial Kernel, SVM  0.723095238095 RBF Kernel, SVM  0.722142857143 Linear Kernel, SVM  0.567619047619 Random Forests  0.770476190476 AdaBoost  0.848333333333<jupyter_code># Normalization functions
def normalize(data):
    # Normalizes the norm of the entire vector to 1
    data = data.copy()
    n, _  = data.shape
    if n == 1:
        return singular_normalize(data)
    
    for i in range(n):
        data[i, :] = singular_normalize(data[i, :])
        
    return data
    
def singular_normalize(data_vector):
    # Normalizes one vector to 1
    norm = np.linalg.norm(data_vector)
    return data_vector / norm<jupyter_output><empty_output><jupyter_text>Let us generate AWGN, unit variance, normalize, and compare classification<jupyter_code># awgn_data = normalize(np.random.normal(0, 1, (bpsk_features.shape[0], 128)))
# bpsk_features_norm = normalize(bpsk_features)
# awgn_features = wrapper_cyclo(awgn_data)
# train_x, train_y, test_x, test_y = create_train_test(bpsk_features_norm, awgn_features)

# train_x_real_imag = np.hstack([train_x.real, train_x.imag])
# test_x_real_imag = np.hstack([test_x.real, test_x.imag])

# train_x_mag_phase = np.hstack([np.abs(train_x), np.angle(train_x)])
# test_x_mag_phase = np.hstack([np.abs(test_x), np.angle(test_x)])


# print("Using real imaginary")
# gamut_classification(train_x_real_imag, train_y, test_x_real_imag, test_y)
# print("")
# print("Using mag phase")
# gamut_classification(train_x_mag_phase, train_y, test_x_mag_phase, test_y)<jupyter_output><empty_output><jupyter_text>We conclude that the classifiers can baseline classify high SNR BPSK vs AWGN. Now we proceed to work with much extended datasets to classify BPSK vs QPSK. These are the ones with the most similar peaks.<jupyter_code># def load_dataset(dat_file):
#     """Input: path to .dat binary data file
#        Outputs:
#        dataset: The dataset loaded by cPickle
#        X: [num_samples, 2, vec_length=128] size 3D array. X[i][0][:] is the real part, X[i][1][:] is the imaginary part
#        uid_list: [num_samples] length list. Contains the 'uid' for each sample which can be used to find its label using lbl_map
#        lbl_map: Dictionary containing labels for a uid
#        lbl: [num_samples] length list. Contains the list of dictionary labels
#        """
#     f = open(dat_file, 'rb')
#     dataset = cPickle.load(f, encoding = 'latin-1')
    
#     Xd = dataset['samples']
#     lbl_map = dataset['uid_map']
#     total_ids = dataset['total_uids']
#     lbl = []
#     X = []
#     uid_list = []
#     for uid in Xd:
#         X.append(Xd[uid])
#         for j in range(Xd[uid].shape[0]):
#             lbl.append((lbl_map[uid]))
#             uid_list.append(uid)
#     X = np.vstack(X)
#     return [dataset, X, uid_list, lbl_map, lbl]

# def is_match(target_dict, cand_dict):
#     """Check if all the key-value pairs in target_dict are present in cand_dict"""
#     for key in target_dict:
#         if key != 'channel_model':
#             if key not in cand_dict:       
#                 return False
#             else:
#                 got_key = False
#                 for inkey in target_dict[key]:  
#                     if inkey == cand_dict[key]:
#                         got_key = True
#                         break
                
#                 if got_key is False:
#                     return False
#         else:
#             # print key
#             # print cand_dict
#             if 'channel_model' not in cand_dict:
#                 return False
#             else:
#                 cand_channel_dict = cand_dict['channel_model']
#                 if cand_channel_dict['channel_type'] != target_dict[key]:
#                     return False
#     return True


# def assemble_samples(X):
#     '''Assemble to desired format containing complex numbers'''
#     Y  = np.zeros((X.shape[0], X[0].shape[1]), dtype = 'complex')
#     for i in range(X.shape[0]):
#         Y[i,:] = X[i][0][:] + 1j*X[i][1][:]
#     return Y
    
# def get_samples(target_dict, dataset):
#     X = np.zeros((0,0))
#     for uid in dataset['samples']:
#         cur_dict = dataset['uid_map'][uid]
      
#         if is_match(target_dict, cur_dict):
#             if X.shape[0] == 0:
#                 X= assemble_samples(dataset['samples'][uid])
#             else:
#                 Y = assemble_samples(dataset['samples'][uid])
#                 X = np.vstack([X, Y])
            
#     return X
            
# dat_file = '../../datasets/basic.dat'
# [dataset, X, uid_list, lbl_map, lbl] = load_dataset(dat_file)
# target_dict_bpsk = {}
# target_dict_bpsk['mod_type'] = [ 'BPSK_G']


# target_dict_qpsk = {}
# target_dict_qpsk['mod_type'] = [ 'QPSK_G']

# bpsk_128 = get_samples(target_dict_bpsk, dataset)
# qpsk_128 = get_samples(target_dict_qpsk, dataset)


# plt.plot(bpsk_128.real[500])
# plt.plot(bpsk_128.imag[500])
# plt.title("BPSK")
# plt.show()
# plt.clf()
# plt.plot(qpsk_128.imag[500])
# plt.plot(qpsk_128.real[500])
# plt.title("QPSK")
# plt.show()<jupyter_output><empty_output><jupyter_text>We now have two arrays of split signals, each split snippet is 128 samples long. We now calculate the cyclostationary coefficients of these signals, and then run a gamut of classification techniques to see what works.<jupyter_code># N = 10000
# offset = 1000
# print(bpsk_128.shape)
# print(qpsk_128.shape)
# bpsk_features = wrapper_cyclo(bpsk_128)
# qpsk_features = wrapper_cyclo(qpsk_128)


# train_x, train_y, test_x, test_y = create_train_test(bpsk_features, qpsk_features)
# print("Done Creating Training and test set.")
# train_x_real_imag = np.hstack([train_x.real, train_x.imag])
# test_x_real_imag = np.hstack([test_x.real, test_x.imag])

# train_x_mag_phase = np.hstack([np.abs(train_x), np.angle(train_x)])
# test_x_mag_phase = np.hstack([np.abs(test_x), np.angle(test_x)])


# print("Using real imaginary")
# gamut_classification(train_x_real_imag, train_y, test_x_real_imag, test_y)
# print("")
# print("Using mag phase")
# gamut_classification(train_x_mag_phase, train_y, test_x_mag_phase, test_y)

# gamut_classification(train_x, train_y, test_x, test_y)<jupyter_output><empty_output><jupyter_text>''' Using real imaginary Polynomial Kernel, SVM  0.5RBF Kernel, SVM  0.5Linear Kernel, SVM  0.575166666667Random Forests  0.887166666667AdaBoost  0.922166666667Using mag phasePolynomial Kernel, SVM  0.876RBF Kernel, SVM  0.874Linear Kernel, SVM  0.660833333333Random Forests  0.770333333333AdaBoost  1.0 '''<jupyter_code>from sklearn.decomposition import PCA, NMF
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.neighbors import KNeighborsClassifier

def gamut_classification_pca(X, Y, n_features=[16,32,64,128],
                            c_options=[1, 10, 100, 1000], n_estimators=[10, 100, 1000],use_svm=True,use_adaboost=True,
                            use_k_neighbors=True):
    """
    Depending on passed in Arguments, does dimensionality reduction and then classifies.
    """
    pipe = Pipeline([
        ('reduce_dim', PCA()),
        ('classify', SVC())
    ])
    
    param_grid = []
    
    if use_adaboost:
        param_grid.append({
            'reduce_dim': [PCA(iterated_power=7)],
            'reduce_dim__n_components': n_features,
            'classify': [AdaBoostClassifier()],
            'classify__n_estimators': n_estimators
        })
    
    if use_svm:
        param_grid.append({
            'reduce_dim': [PCA(iterated_power=7)],
            'reduce_dim__n_components': n_features,
            'classify': [SVC()],
            'classify__C': c_options
        })
        
    if use_k_neighbors:
        param_grid.append({
            'reduce_dim': [PCA(iterated_power=7)],
            'reduce_dim__n_components': n_features,
            'classify': [SVC()],
            'classify__C': c_options
        })
        
        
    grid = GridSearchCV(pipe, n_jobs=1, param_grid=param_grid, verbose=10)
    grid.fit(X, Y)
    return grid
    
# N = 100
# offset = 1000
# bpsk_features_100 = wrapper_cyclo(bpsk_128[offset: offset + N, :])
# qpsk_features_100 = wrapper_cyclo(qpsk_128[offset: offset + N, :])
# train_x, train_y, test_x, test_y = create_train_test(bpsk_features_100, qpsk_features_100)
# train_x_real_imag = np.hstack([train_x.real, train_x.imag])
# test_x_real_imag = np.hstack([test_x.real, test_x.imag])

# train_x_mag_phase = np.hstack([np.abs(train_x), np.angle(train_x)])
# test_x_mag_phase = np.hstack([np.abs(test_x), np.angle(test_x)])

# X_real_imag = np.vstack([train_x_real_imag, test_x_real_imag])
# X_mag_phase = np.vstack([train_x_mag_phase, test_x_mag_phase])
# Y = np.vstack([train_y, test_y])

# grid_real_imag = gamut_classification_pca(X_real_imag, Y.ravel())
# grid_mag_phase = gamut_classification_pca(X_mag_phase, Y.ravel())
# print("Using real imag")
# print(grid_real_imag.best_estimator_)
# print(grid_real_imag.best_score_)
# print("Using mag phase")
# print(grid_mag_phase.best_estimator_)
# print(grid_mag_phase.best_score_)
# N = 1000
# offset = 1000
# bpsk_features_1000 = wrapper_cyclo(bpsk_128[offset: offset + N, :])
# qpsk_features_1000 = wrapper_cyclo(qpsk_128[offset: offset + N, :])
# train_x, train_y, test_x, test_y = create_train_test(bpsk_features_1000, qpsk_features_1000)

# train_x_real_imag = np.hstack([train_x.real, train_x.imag])
# test_x_real_imag = np.hstack([test_x.real, test_x.imag])

# train_x_mag_phase = np.hstack([np.abs(train_x), np.angle(train_x)])
# test_x_mag_phase = np.hstack([np.abs(test_x), np.angle(test_x)])

# X_real_imag = np.vstack([train_x_real_imag, test_x_real_imag])
# X_mag_phase = np.vstack([train_x_mag_phase, test_x_mag_phase])
# Y = np.vstack([train_y, test_y])

# grid_real_imag = gamut_classification_pca(X_real_imag, Y.ravel())
# grid_mag_phase = gamut_classification_pca(X_mag_phase, Y.ravel())
# print("Using real imag")
# print(grid_real_imag.best_estimator_)
# print(grid_real_imag.best_score_)

# print("Using mag phase")
# print(grid_mag_phase.best_estimator_)
# print(grid_mag_phase.best_score_)
# bpsk = ds[("BPSK", 18)]
# qpsk = ds[("QPSK", 18)]

# bpsk = np.add(bpsk[:, 0, :], 1j*bpsk[:, 1, :])
# qpsk = np.add(qpsk[:, 0, :], 1j*qpsk[:, 1, :])
# bpsk_features = wrapper_cyclo(bpsk)
# qpsk_features = wrapper_cyclo(qpsk)
# train_x, train_y, test_x, test_y = create_train_test(bpsk_features, qpsk_features)

# train_x_real_imag = np.hstack([train_x.real, train_x.imag])
# test_x_real_imag = np.hstack([test_x.real, test_x.imag])

# train_x_mag_phase = np.hstack([np.abs(train_x), np.angle(train_x)])
# test_x_mag_phase = np.hstack([np.abs(test_x), np.angle(test_x)])

# X_real_imag = np.vstack([train_x_real_imag, test_x_real_imag])
# X_mag_phase = np.vstack([train_x_mag_phase, test_x_mag_phase])
# Y = np.vstack([train_y, test_y])

# grid_real_imag = gamut_classification_pca(X_real_imag, Y.ravel())
# grid_mag_phase = gamut_classification_pca(X_mag_phase, Y.ravel())



# print("Using real imag")
# print(grid_real_imag.best_estimator_)
# print(grid_real_imag.best_score_)

# print("Using mag phase")
# print(grid_mag_phase.best_estimator_)
# print(grid_mag_phase.best_score_)
# # snr_vals = list(map(lambda a: a[1], list(filter(lambda a:a[0] == 'BPSK' and a[1] != 18, list(ds.keys()), ))))
# snr_vals = [16,14, 12, 10, 8, 6]

# bpsk = ds[("BPSK", 18)]
# qpsk = ds[("QPSK", 18)]
# for i in snr_vals:
#     bpsk = np.vstack([ds[("BPSK", i)], bpsk])
#     qpsk = np.vstack([ds[("QPSK", i)], qpsk])

# bpsk = np.add(bpsk[:, 0, :], 1j*bpsk[:, 1, :])
# qpsk = np.add(qpsk[:, 0, :], 1j*qpsk[:, 1, :])
# bpsk_features = wrapper_cyclo(bpsk)
# qpsk_features = wrapper_cyclo(qpsk)
# train_x, train_y, test_x, test_y = create_train_test(bpsk_features, qpsk_features)

# train_x_real_imag = np.hstack([train_x.real, train_x.imag])
# test_x_real_imag = np.hstack([test_x.real, test_x.imag])

# train_x_mag_phase = np.hstack([np.abs(train_x), np.angle(train_x)])
# test_x_mag_phase = np.hstack([np.abs(test_x), np.angle(test_x)])

# X_real_imag = np.vstack([train_x_real_imag, test_x_real_imag])
# X_mag_phase = np.vstack([train_x_mag_phase, test_x_mag_phase])
# Y = np.vstack([train_y, test_y])

# grid_real_imag = gamut_classification_pca(X_real_imag, Y.ravel())
# grid_mag_phase = gamut_classification_pca(X_mag_phase, Y.ravel())

# print("Using real imag")
# print(grid_real_imag.best_estimator_)
# print(grid_real_imag.best_score_)

# print("Using mag phase")
# print(grid_mag_phase.best_estimator_)
# print(grid_mag_phase.best_score_)<jupyter_output><empty_output><jupyter_text>We have that using all SNRs lowers effectiveness by 20% for BPSK vs QPSK, instead of just SNR 18. Let us now use AdaBoost with PCA onto 32/64 dimensions and generate a confusion matrix for all signals with SNR 18<jupyter_code>def create_labels(data, shuffle=False):
    """
    Takes an array of n data arrays and assign labels of i for elements in array i.
    """
    ys = [np.zeros((arr.shape[0], 1)) + i for i, arr in enumerate(data)]
    X = np.vstack(data)
    Y = np.vstack(ys)
    print(X.shape, Y.shape)

    # Time to mix the data
    if shuffle:
        together = np.hstack((X, Y))
        np.random.shuffle(together)
        X, Y = np.hsplit(together, [together.shape[1] - 1])

    return X, Y
data = [(key, value) for key, value in ds.items() if key[1] == 18]
labels, data_signal = zip(*data)
signals = list(
    map(lambda data: data[:, 0, :] + 1j*data[:, 1, :], data_signal)
) # Convert into a complex number from I and Q
# X, Y = create_labels(signals)
# X_real_imag = np.hstack([X.real, X.imag])

# X_mag_phase = np.hstack([np.abs(X), np.angle(X)])


# grid_real_imag = gamut_classification_pca(X_real_imag, Y.ravel(), use_svm=False, n_estimators=[100], n_features=[32, 64, 128, 256])
# grid_real_imag.best_estimator_
from sklearn.metrics import confusion_matrix
# clf_real_imag = grid_real_imag.best_estimator_
# true_y, pred_y = Y, clf_real_imag.predict(X)from sklearn.metrics import confusion_matrix
# clf_real_imag = grid_real_imag.best_estimator_
# true_y, pred_y = Y, clf_real_imag.predict(X)
import itertools
# import numpy as np
# import matplotlib.pyplot as plt


from sklearn.model_selection import train_test_split

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# # Compute confusion matrix
# cnf_matrix = confusion_matrix(true_y, pred_y)
# np.set_printoptions(precision=2)

# # Plot non-normalized confusion matrix
# plt.figure()
# plot_confusion_matrix(cnf_matrix, classes=labels,
#                       title='Confusion matrix, without normalization')

# # Plot normalized confusion matrix
# plt.figure()
# plot_confusion_matrix(cnf_matrix, classes=labels, normalize=True,
#                       title='Normalized confusion matrix')

# plt.show()
# cyclo_coeffs = list(map(wrapper_cyclo, signals))
# X, Y = create_labels(np.stack((signals, cyclo_coeffs)))


# X, Y = create_labels(cyclo_coeffs)
# X_real_imag = np.hstack([X.real, X.imag])



# grid_real_imag = gamut_classification_pca(X_real_imag, Y.ravel(), use_svm=False, n_estimators=[100], n_features=[32, 64, 128])

# print(grid_real_imag.best_estimator_)
# print(grid_real_imag.best_score_)

# clf = grid_real_imag.best_estimator_
# true_y, pred_y = Y.ravel(), clf.predict(X_real_imag).ravel()
# # Compute confusion matrix
# cnf_matrix = confusion_matrix(true_y, pred_y)
# np.set_printoptions(precision=2)

# # Plot non-normalized confusion matrix
# plt.figure(figsize=(10, 20))
# plot_confusion_matrix(cnf_matrix, classes=labels,
#                       title='Confusion matrix, without normalization')
# plt.show()

# # Plot normalized confusion matrix
# plt.figure(figsize=(10, 20))
# plot_confusion_matrix(cnf_matrix, classes=labels, normalize=True,
#                       title='Normalized confusion matrix')

# plt.show()
data = [(key, value) for key, value in ds.items() if key[1] >= 6]
# labels, data_signal = zip(*data)
# signals = list(
#     map(lambda data: data[:, 0, :] + 1j*data[:, 1, :], data_signal)
# ) # Convert into a complex number from I and Q

signals = []
unique_keys = []
for key in data:
    if key[0][0] not in unique_keys:
        unique_keys.append(key[0][0])
for unique_key in unique_keys:
    X = np.zeros((0,0), dtype = 'complex')        

    for s in data:
        if s[0][0] == unique_key:
#             print(s[0][0])
            if X.shape[0] == 0:
                X = s[1][:,0] + 1j*s[1][:,1]
            else:                
                Y =  s[1][:,0] + 1j*s[1][:,1]
#                 print(Y.shape)
                X = np.vstack([X, Y])
#     print(X.shape)
    signals.append(X)
        
            
        
cyclo_coeffs = list(map(wrapper_cyclo, signals))

def create_labels(data, shuffle=False):
    """
    Takes an array of n data arrays and assign labels of i for elements in array i.
    """
    ys = [np.zeros((arr.shape[0], 1)) + i for i, arr in enumerate(data)]
    X = np.vstack(data)
    Y = np.vstack(ys)
    print(X.shape, Y.shape)

    # Time to mix the data
    if shuffle:
        together = np.hstack((X, Y))
        np.random.shuffle(together)
        X, Y = np.hsplit(together, [together.shape[1] - 1])

    return X, Y

X, Y = create_labels(cyclo_coeffs)
# Xs, Y = create_labels(signals)

X_mag_phase = np.hstack([np.abs(X), np.angle(X)])

# X_mag_phase = np.hstack([np.abs(X), np.angle(X), np.abs(Xs), np.angle(Xs)])
# print(X_mag_phase.shape)

plt.plot(Y)
plt.show()

grid_mag_phase = gamut_classification_pca(X_mag_phase, Y.ravel(), use_svm=False, n_estimators=[100], n_features=[32, 64, 128])

print(grid_mag_phase.best_estimator_)
print(grid_mag_phase.best_score_)

print(grid_mag_phase.best_estimator_)
# clf = grid_mag_phase.best_estimator_
# true_y, pred_y = Y.ravel(), clf.predict(X_mag_phase).ravel()
# # Compute confusion matrix
# cnf_matrix = confusion_matrix(true_y, pred_y)
# np.set_printoptions(precision=2)

# # Plot non-normalized confusion matrix
# plt.figure(figsize=(10, 20))
# plot_confusion_matrix(cnf_matrix, classes=labels,
#                       title='Confusion matrix, without normalization')
# plt.show()

# # Plot normalized confusion matrix
# plt.figure(figsize=(10, 20))
# plot_confusion_matrix(cnf_matrix, classes=labels, normalize=True,
#                       title='Normalized confusion matrix')

# plt.show()
cntc = 0
lim =40000
for i in range(lim):
       if true_y[i] == pred_y[i]:
             cntc += 1
print(cntc/lim)

X_real_imag = np.hstack([X.real, X.imag])

grid_real_imag = gamut_classification_pca(X_real_imag, Y.ravel(), use_svm=False, n_estimators=[100], n_features=[32, 64, 128])

print(grid_real_imag.best_estimator_)
print(grid_real_imag.best_score_)

clf = grid_real_imag.best_estimator_
true_y, pred_y = Y.ravel(), clf.predict(X_real_imag).ravel()
# Compute confusion matrix
cnf_matrix = confusion_matrix(true_y, pred_y)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure(figsize=(10, 20))
plot_confusion_matrix(cnf_matrix, classes=labels,
                      title='Confusion matrix, without normalization')
plt.show()

# Plot normalized confusion matrix
plt.figure(figsize=(10, 20))
plot_confusion_matrix(cnf_matrix, classes=labels, normalize=True,
                      title='Normalized confusion matrix')

plt.show()
a = np.diag(cnf_matrix)
s = sum(a)
print(float(s)/77000)*100<jupyter_output>0.7806883116883117
{'fraction_non_alphanumeric': 0.14007532654860166, 'fraction_numerical': 0.02664476320217966, 'mean_word_length': 3.7869198312236287, 'pattern_counts': {'":': 2, '<': 28, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '171032', 'license': 'MIT', 'n_tokens_mistral': 9955, 'n_tokens_neox': 9368, 'n_words': 1841, 'path': 'test_signal_generation/test_signals_m.ipynb', 'repo_name': 'jain-nikunj/radioML', 'size': '24958'}