<jupyter_start><jupyter_text>This notebook is about dimensionality selection from a given dataset using Genetic Algorithm.<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.metrics import silhouette_score
import matplotlib.cm as cm
import os
import sys
import time
from google.colab import drive
drive.mount('/content/drive')<jupyter_output><empty_output><jupyter_text>Commonly Used Variables<jupyter_code>common_path = '/content/drive/My Drive/Colab Notebooks/spec2006-perf-simulations/'
MAX_VAL = sys.maxsize
MIN_VAL = -sys.maxsize - 1<jupyter_output><empty_output><jupyter_text>Commonly Used Functions<jupyter_code>def get_dataframe_from_directory(dir_path) :
    '''
        make one csv file from multiple csv files in the directory and
        returns a dataframe consisting of csv files of given directory and list of file names in the directory
        and a list of number of rows in each file
    '''
    dirFiles = os.listdir(dir_path)
    dirFiles.sort()
    dfList = []
    if not dir_path.endswith('/') :
        dir_path = dir_path + '/'
    fileSizes = []
    for entry in dirFiles : 
        file_path = dir_path + entry 
        if file_path.endswith('.csv') :
            dfList.append(pd.read_csv(file_path))
            for k in list(dfList[-1]) :
                dfList[-1][k] = pd.to_numeric(dfList[-1][k], errors='coerce')       # convert all strings to NaN
            dfList[-1].fillna(dfList[-1].median(), inplace=True)          # convert all NaN to median of the corresponding column
            fileSizes.append(dfList[-1].shape[0])
    if dfList :
        df = pd.concat(dfList)
    else :
        df = None
    return df, dirFiles, fileSizes

def get_dataframe_from_directories(listOfDir=[]) :
    '''
        returns a dataframe consisting of csv files of all directories and list of all the file names 
    '''
    if not listOfDir :
        return None, []
    dfList = []
    listOfFiles = []
    fileSizeList = []
    dirSizeList = []
    for directory in listOfDir :
        dirDf, files, fileSizes = get_dataframe_from_directory(directory)
        dfList.append(dirDf)
        listOfFiles += files
        fileSizeList += fileSizes
        dirSizeList.append(sum(fileSizes))
    
    df = pd.concat(dfList)
    return df, listOfFiles, dirSizeList, fileSizeList


def euclidean_dist(first, second) :
    '''
    returns euclidean distance between two data points
    '''
    if not (type(first) is np.ndarray) :
        first = np.array(first)
    if not (type(second) is np.ndarray) :
        second = np.array(second)

    return np.sqrt(np.sum((np.square(first-second))))

def divide_by_instruction(x):
    '''
        divide the pandas series 'x' by number of instructions
    '''
    return x/x['inst_retired.any']

def preprocessData(df, per_instruction_factor=1000) :
    df = df.apply(divide_by_instruction, axis=1)
    df = df.iloc[:, 1:]*1000
    return df

def getAllDirectories() :
    dirFiles = os.listdir(common_path)
    dirFiles.sort()
    allDirectories = []
    for entry in dirFiles : 
        if entry != 'listOfFiles.list' :
            file_path = common_path + entry + '/yinputs'
            allDirectories.append(file_path)
    return allDirectories

###### GENETIC ALGORITHM UTILITIES ######
def getInitialPopulation(n_features, POPULATION_SIZE=5) :
    population = []
    binList = ['0', '1']
    for i in range(POPULATION_SIZE) :
        randomList = list(np.random.choice(binList, n_features))
        population.append(''.join(randomList))
    return population
listOfDir = [ common_path + '403.gcc/yinputs' ]
# listOfDir = getAllDirectories()
df, listOfFiles, dirSizeList, fileSizeList = get_dataframe_from_directories(listOfDir)
df_model = df.iloc[:, 1:]
df_model = preprocessData(df_model)
display("Dataframe for the list of directories has been created...")
display(df_model)
display(listOfFiles)<jupyter_output><empty_output><jupyter_text>Genetic Algorithm<jupyter_code>n_features = len(df_model.columns)
POPULATION_SIZE = 5
# population = getInitialPopulation(n_features, POPULATION_SIZE)
population = [ '1010111100111010101100000100',
                        '0000011111000010001101000100',
                        '0011000100111111000100010010',
                        '1000011110000111111111100110',
                        '0111111110001010100001010010' ]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0986101919258769, 'fraction_numerical': 0.03750275755570263, 'mean_word_length': 3.2773584905660376, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '413630', 'license': 'MIT', 'n_tokens_mistral': 1548, 'n_tokens_neox': 1394, 'n_words': 342, 'path': '.ipynb_checkpoints/Dimensionality_Selection-checkpoint.ipynb', 'repo_name': 'kuldeep725/BTP', 'size': '4533'}