<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import numpy as np
import os
import pandas as pd
import inflect
# next we can import some sklearn libraries to start working with stuff
## transformers and pipline
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline, FeatureUnion
## model selectors
from sklearn.model_selection import train_test_split, cross_val_score
## models
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
## feature extarctors
from sklearn.preprocessing import OneHotEncoder
from average_word_length_extractor import AverageWordLengthExtractor
from question_extractor import QuestionExtractor
from int_to_words_extractor import NumberStringExtractor
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer<jupyter_output><empty_output><jupyter_text>Reading data<jupyter_code># read data
data = pd.read_excel('../data/sample_data/EaxampleData.xlsx')
# convert yes to True(1) and no to False(0)
data = data.replace(to_replace={'yes': 1, 'no': 0})
# features over which we have to predict
prediction_columns = ['stringedQuestion',
                   'leading', 'Symptom', 'PMH', 'MEDS', 'ALLG', 'FAMHx', 'lifestyle',
                   'pysch', 'SOCHx', 'sexualHistory', 'substanceUse', 'PE', 'FORM',
                   'supportProvision', 'transition']
# extract numerical columns
numerical_columns = data.select_dtypes(np.number).columns
def add_patient_physician_interactions(data, numerical_columns):
    assert type(data) == pandas.core.frame.DataFrame
    # to combine data from two columns
    comm_df = pd.DataFrame({'original_semantics': data['semanticType'],
                            'shifted_semantics' : data['semanticType'].shift(-1),
                            'original_speaker': data['speakerID'],
                            'shifted_speaker': data['speakerID'].shift(-1)}, index=data.index)

    
    data['comparisons'] = comm_df.apply(lambda x: 1 if (x.original_semantics in ['openQuestion', 'question']) and 
                  (x.shifted_semantics == 'statement') and
                  (x.original_speaker == 'doctor') and
                  (x.shifted_speaker == 'patient') else 0, axis = 1)

#     data.head()
    indexes = data.loc[data.comparisons == 1].index 
    data.loc[data.comparisons == 1,numerical_columns] = data.loc[data.comparisons == 1, numerical_columns] + data[numerical_columns].shift(-1).iloc[indexes,:]
#     [numerical_columns] = data[numerical_columns].shift(-1)
    return data
# extract doctor interactions
complete_data = data.loc[data.speakerID == 'doctor']<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code># select columns by data
complete_data.drop(prediction_columns, inplace=True, axis=1)
unusable_columns = ['speakerID']
complete_data.drop(unusable_columns, inplace=True, axis=1)
# select x and y
X = complete_data
y = data.loc[complete_data.index, data.columns == prediction_columns[2]]
# split into train and test
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42, stratify = y)
ngram_count = CountVectorizer(ngram_range=(1, 3), analyzer='char')
avg_word = AverageWordLengthExtractor()
question_ex = QuestionExtractor()
one_hot_encoder = OneHotEncoder(handle_unknown='ignore')
numberstring= NumberStringExtractor()<jupyter_output><empty_output><jupyter_text>Modelling<jupyter_code>#convert numbers to text
num_to_str = ColumnTransformer([
    ('numberstring', numberstring, 3)
], remainder='passthrough')

# text encoders
encoders = ColumnTransformer([
    ('ngram', ngram_count, 0),
    ('avg_word', avg_word, 0),
    ('question', question_ex, 0)
], remainder = 'passthrough')

# one hot encoding
one_hot = ColumnTransformer([
    ('one_hot', one_hot_encoder, ['semanticType'])
], remainder = 'passthrough')

# text pipeline
text_features = Pipeline([
    ('num_to_str', num_to_str),
    ('encoders', encoders)
])

# preprocessing
preprocess = Pipeline([
    ('one_hot_encoder', one_hot),
    ('text_feat', text_features)
])

# ml pipeline
ml_pipeline = Pipeline([
    ('preprocessor', preprocess),
    ('model', RandomForestClassifier(n_estimators = 100))
])
cross_val_score(ml_pipeline, X_train, y_train, cv = 10).mean()<jupyter_output>/anaconda3/envs/aml/lib/python3.6/site-packages/sklearn/pipeline.py:267: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  self._final_estimator.fit(Xt, y, **fit_params)
/anaconda3/envs/aml/lib/python3.6/site-packages/sklearn/pipeline.py:267: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  self._final_estimator.fit(Xt, y, **fit_params)
/anaconda3/envs/aml/lib/python3.6/site-packages/sklearn/pipeline.py:267: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  self._final_estimator.fit(Xt, y, **fit_params)
/anaconda3/envs/aml/lib/python3.6/site-packages/sklearn/pipeline.py:267: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change [...]
{'fraction_non_alphanumeric': 0.1172465960665658, 'fraction_numerical': 0.011535552193645991, 'mean_word_length': 5.017064846416382, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '229716', 'license': 'MIT', 'n_tokens_mistral': 1729, 'n_tokens_neox': 1683, 'n_words': 398, 'path': 'code/initial_code.ipynb', 'repo_name': 'ujjwal95/tappe_project', 'size': '5288'}