<commit_before>#!/usr/bin/env python

"""
Primary execution file for USP-Inhibition project
"""
import sys
sys.path.append("/home/pphilip/Tools/openbabel-install/lib")

import descriptors
import genalgo
import models
import pandas as pd
import pickle
import post_process
import sklearn
try:
    import urllib.request as urllib2
except ImportError:
    import urllib2
import utils

__author__ = "<NAME>"
__credits__ = "<NAME>"
__license__ = "BSD 3-Clause License"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__status__ = "Development"

TARGET_COLUMN = 'Activity_Score'
XY_PICKLE = '../data/xy_data.pkl'


def main():
    """
    Module to execute the entire package from data retrieval to model results
    :return: None
    """

    # Importing dataset from NCBI database to create dataframe
    """response = pd.read_csv('https://pubchem.ncbi.nlm.nih.gov/pcajax/pcget.cgi?query=download&record_type='
                           'datatable&response_type=save&aid=743255&version=1.1')"""

    # Importing inhibitor notation data
    response = urllib2.urlopen('https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/compounds_smiles.txt')
    df_smiles = utils.create_notation_dataframe(response)

    # Importing inhibitor activity data
    response = pd.read_csv('https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/AID_743255_datatable.csv')
    df_activity = utils.create_activity_dataframe(response)

    df = df_activity.merge(df_smiles)
    df.drop(df.index[[276743, 354142]], inplace=True)
    df.sort_values(by='CID', inplace=True)
    df.reset_index(drop=True, inplace=True)

    # Drop non-descriptor columns before feature space reduction
    df_x = df.drop(['Activity_Score', 'CID'], axis=1)

    # Creating target column
    df_y = df.drop(['SMILES', 'CID'], axis=1)

    # Extracting molecular descriptors for all compounds
    print("Starting descriptor calculation")
    descriptors.extract_all_descriptors(df_x, 'SMILES')
    print("Finished descriptor calculation")

    print("Joining dataframes")
    df_x = utils.join_dataframes()
    print("Joined dataframes")
    df_x.to_csv('../data/df_x_preprocessing.csv')
    df_y.to_csv('../data/df_y_preprocessing.csv')

    print("Checking dataframe for NaN and infinite values")
    df_x = utils.remove_nan_infinite(df_x)
    df_y = utils.remove_nan_infinite(df_y)
    print("Checked dataframe for NaN and infinite values")

    # Transform all column values to mean 0 and unit variance
    print("Transforming dataframe using mean and variance")
    df_x = sklearn.preprocessing.scale(df_x)
    df_y = sklearn.preprocessing.scale(df_y)
    print("Transformed dataframe using mean and variance")

    # Feature selection and space reduction
    print("Selecting best features in dataframe")
    df_x = utils.choose_features(df_x, df_y)
    print("Selected best features in dataframe")

    df_x = pd.DataFrame(df_x)
    utils.plot_features(df_x, df_y)

    df_x.to_csv('../data/df_x_postprocessing.csv')
    df_y.to_csv('../data/df_y_postprocessing.csv')
    df = df_x.join(df_y)
    # Data to training task
    # Type check inputs for sanity
    if df is None:
        raise ValueError('df is None')
    if not isinstance(df, pd.DataFrame):
        raise TypeError('df is not a dataframe')
    if TARGET_COLUMN is None:
        raise ValueError('target_column is None')
    if not isinstance(TARGET_COLUMN, basestring):
        raise TypeError('target_column is not a string')
    if TARGET_COLUMN not in df.columns:
        raise ValueError('target_column (%s) is not a valid column name' % TARGET_COLUMN)

    # Train, validation and test split
    df_train, df_test = sklearn.cross_validation.train_test_split(df, test_size=0.25)

    # Reassign target column name and index after randomized split
    df_train = df_train.rename(columns={-1: TARGET_COLUMN})
    df_test = df_test.rename(columns={-1: TARGET_COLUMN})
    df_train.reset_index(inplace=True, drop=True)
    df_test.reset_index(inplace=True, drop=True)

    # Remove the classification column from the dataframe
    x_train = np.array(df_train.drop(TARGET_COLUMN, axis=1))
    x_test = np.array(df_test.drop(TARGET_COLUMN, axis=1))
    y_train = np.array(df_train[TARGET_COLUMN])
    y_test = np.array(df_test[TARGET_COLUMN])<commit_msg>Added user-input option to choose steps to be run<commit_after>#!/usr/bin/env python

"""
Primary execution file for USP-Inhibition project
"""
import sys
sys.path.append("/home/pphilip/Tools/openbabel-install/lib")

import descriptors
import genalgo
import models
import pandas as pd
import pickle
import post_process
import sklearn
try:
    import urllib.request as urllib2
except ImportError:
    import urllib2
import utils

__author__ = "<NAME>"
__credits__ = "<NAME>"
__license__ = "BSD 3-Clause License"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__status__ = "Development"

TARGET_COLUMN = 'Activity_Score'
XY_PICKLE = '../data/xy_data.pkl'


def main():
    """
    Module to execute the entire package from data retrieval to model results
    :return: None
    """
    run = input("Type 1 to calculate files from raw data or 0 to access stored files:")
    if run:
        # Importing dataset from NCBI database to create dataframe
        """response = pd.read_csv('https://pubchem.ncbi.nlm.nih.gov/pcajax/pcget.cgi?query=download&record_type='
                               'datatable&response_type=save&aid=743255&version=1.1')"""

        # Importing inhibitor notation data
        response = urllib2.urlopen('https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/compounds_smiles.txt')
        df_smiles = utils.create_notation_dataframe(response)

        # Importing inhibitor activity data
        response = pd.read_csv('https://s3-us-west-2.amazonaws.com/pphilip-usp-inhibition/AID_743255_datatable.csv')
        df_activity = utils.create_activity_dataframe(response)

        df = df_activity.merge(df_smiles)
        df.drop(df.index[[276743, 354142]], inplace=True)
        df.sort_values(by='CID', inplace=True)
        df.reset_index(drop=True, inplace=True)

        # Drop non-descriptor columns before feature space reduction
        df_x = df.drop(['Activity_Score', 'CID'], axis=1)

        # Creating target column
        df_y = df.drop(['SMILES', 'CID'], axis=1)

        # Extracting molecular descriptors for all compounds
        print("Starting descriptor calculation")
        descriptors.extract_all_descriptors(df_x, 'SMILES')

        print("Joining dataframes")
        df_x = utils.join_dataframes()
        df_x.to_csv('../data/df_x_preprocessing.csv')
        df_y.to_csv('../data/df_y_preprocessing.csv')

        print("Checking dataframe for NaN and infinite values")
        df_x = utils.remove_nan_infinite(df_x)
        df_y = utils.remove_nan_infinite(df_y)

        # Transform all column values to mean 0 and unit variance
        print("Transforming dataframe using mean and variance")
        df_x = sklearn.preprocessing.scale(df_x)
        df_y = sklearn.preprocessing.scale(df_y)

        # Feature selection and space reduction
        print("Selecting best features in dataframe")
        df_x = utils.choose_features(df_x, df_y)
        df_x = pd.DataFrame(df_x)
        df_x.drop(df.columns[0], axis=1, inplace=True)
        df_y = pd.DataFrame(df_y)
        df_x.to_csv('../data/df_x_postprocessing.csv')
        df_y.to_csv('../data/df_y_postprocessing.csv')
    else:
        df_x = pd.read_csv('../data/df_x_postprocessing.csv')
        df_y = pd.read_csv('../data/df_y_postprocessing.csv')
        df_x.drop(df_x.columns[0], axis=1, inplace=True)

    utils.plot_features(df_x, df_y)
    df = df_x.join(df_y)
    # Data to training task
    # Type check inputs for sanity
    if df is None:
        raise ValueError('df is None')
    if not isinstance(df, pd.DataFrame):
        raise TypeError('df is not a dataframe')
    if TARGET_COLUMN is None:
        raise ValueError('target_column is None')
    if not isinstance(TARGET_COLUMN, basestring):
        raise TypeError('target_column is not a string')
    if TARGET_COLUMN not in df.columns:
        raise ValueError('target_column (%s) is not a valid column name' % TARGET_COLUMN)

    # Train, validation and test split
    df_train, df_test = sklearn.cross_validation.train_test_split(df, test_size=0.25)

    # Reassign target column name and index after randomized split
    df_train = df_train.rename(columns={-1: TARGET_COLUMN})
    df_test = df_test.rename(columns={-1: TARGET_COLUMN})
    df_train.reset_index(inplace=True, drop=True)
    df_test.reset_index(inplace=True, drop=True)

    # Remove the classification column from the dataframe
    x_train = np.array(df_train.drop(TARGET_COLUMN, axis=1))
    x_test = np.array(df_test.drop(TARGET_COLUMN, axis=1))
    y_train = np.array(df_train[TARGET_COLUMN])
    y_test = np.array(df_test[TARGET_COLUMN])
{'fraction_non_alphanumeric': 0.1077581353451188, 'fraction_numerical': 0.01058439364936381, 'mean_word_length': 3.630865484880083, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '575566', 'n_tokens_mistral': 2989, 'n_tokens_neox': 2910, 'n_words': 660}