<jupyter_start><jupyter_text>Lets deal withimportant dim only make new df<jupyter_code>string_dim=['MachineIdentifier']
string_df=df[string_dim]
#'SmartScreen',
#string_dim.ffill(axis = 1)

string_df.isnull().sum()
string_df.shape
sdf=string_df
num_dim=[ 'IsBeta','RtpStateBitfield','IsSxsPassiveMode', 'AVProductStatesIdentifier',
       'AVProductsInstalled', 'AVProductsEnabled','HasTpm',
       'CountryIdentifier', 'CityIdentifier',
       'GeoNameIdentifier', 'LocaleEnglishNameIdentifier', 'OsBuild', 'OsSuite', 'IsProtected', 'AutoSampleOptIn',
       'SMode','IeVerIdentifier','Firewall', 'UacLuaenable','Census_OEMNameIdentifier',
    'Census_OEMNameIdentifier','Census_OEMModelIdentifier','Census_ProcessorCoreCount',
    'Census_ProcessorManufacturerIdentifier','Census_ProcessorModelIdentifier','Census_PrimaryDiskTotalCapacity',
    'Census_SystemVolumeTotalCapacity','Census_HasOpticalDiskDrive','Census_TotalPhysicalRAM','Census_InternalPrimaryDiagonalDisplaySizeInInches',
    'Census_InternalPrimaryDisplayResolutionHorizontal','Census_InternalPrimaryDisplayResolutionVertical',
'Census_InternalBatteryNumberOfCharges',
        'Census_OSBuildNumber','Census_OSBuildRevision', 'Census_OSInstallLanguageIdentifier',
       'Census_OSUILocaleIdentifier','Census_IsPortableOperatingSystem',
'Census_IsFlightsDisabled','Census_FirmwareManufacturerIdentifier',
       'Census_FirmwareVersionIdentifier', 'Census_IsSecureBootEnabled', 'Census_IsVirtualDevice', 
    'Wdft_IsGamer','Wdft_RegionIdentifier', 'HasDetections']
num_df=df[num_dim]
num_df.head()
#num_df.head(10).to_excel('num.xlsx')#never do it always throw small size out from now.
#num_dim.info() 446074 
# num_dim['IsProtected'].dropna(how=all)
#num_dim.dropna(inplace=True)
num_df.shape,string_df.shape<jupyter_output><empty_output><jupyter_text>featire Enginneering functions and calls<jupyter_code>#numerical dim 
from sklearn.preprocessing import StandardScaler
ss=StandardScaler()
#fit
ss.fit(num_df[num_dim])
#scale
num_df[num_dim]=ss.transform(num_df[num_dim])
#view new data
print("done")
#modeling for tranformation on string_dim
from sklearn.preprocessing import LabelEncoder
model=LabelEncoder()
genre_labels = model.fit_transform(string_df['MachineIdentifier'])
dim_label = {index: label for index, label in enumerate(model.classes_)}
string_df['MachineIdentifier'] = dim_label
#print(genre_mappings.head())
print("done")

#view newly engineering features
string_df.head()
#training_features.head()
num_df.head()
#plt.figure(figsize=(16,11))
#sns.heatmap(df[string_dim].isnull(),yticklabels=False,cmap='viridis')<jupyter_output><empty_output><jupyter_text>Deal with misssing values/NAN values<jupyter_code>#dff.dropna(inplace=True)
num_df.dtypes
# plt.figure(figsize=(16,11))
# sns.heatmap(dff.isnull(),yticklabels=False,cmap='viridis')<jupyter_output><empty_output><jupyter_text>Combining back the DataFrames num and string<jupyter_code># dff=training_features+num_df
result = pd.concat([string_df, num_df], axis=1, sort=False)
#dff.isnull().sum()
# result = pd.concat(dff)
result.head()<jupyter_output><empty_output><jupyter_text>start the modeling part<jupyter_code>#result.describe()
x=result.drop('HasDetections', axis=1)
#x=dff[x]
x.head(4)
#x.to_csv("ffffffff.csv", sep=',')
y=df['HasDetections']
y.head()
x.fillna(method='ffill',inplace=True)
#y.fillna(method='ffill')<jupyter_output>/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py:3790: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  downcast=downcast, **kwargs)<jupyter_text>Building Model<jupyter_code># from sklearn.linear_model import LogisticRegression
# import numpy as np
# #fit
# lr=LogisticRegression()
# model=lr.fit(x,y)
# model
# """trying KNN for the same"""
# from sklearn.neighbors import KNeighborsClassifier
# knn = KNeighborsClassifier()
# knn.fit(x,y) 
# KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',
#            metric_params=None, n_jobs=None, n_neighbors=5, p=2,
#            weights='uniform')
#knn.predict(iris_X_test)
from sklearn.svm import SVC
svm=SVC(kernel="linear",random_state=101)
model=svm.fit(x,y)

#prediction=svm.predict(test)
from sklearn.externals import joblib
import os
if not os.path.exists('Model'):
    os.mkdir('Model')
if not os.path.exists('Scaler'):
    os.mkdir('Scaler')
joblib.dump(model, r'Model/model.pickle')
joblib.dump(ss, r'Scaler/scaler.pickle')
#feature engineer the test.csv data
import pandas as pd
dff=pd.read_csv('all-2/test.csv')
#df=pd.read_pickle("test4.pickle")
dff.head()
dff.shape
dff.isnull().sum()
num_dim=[ 'IsBeta','RtpStateBitfield','IsSxsPassiveMode', 'AVProductStatesIdentifier',
       'AVProductsInstalled', 'AVProductsEnabled','HasTpm',
       'CountryIdentifier', 'CityIdentifier',
       'GeoNameIdentifier', 'LocaleEnglishNameIdentifier', 'OsBuild', 'OsSuite', 'IsProtected', 'AutoSampleOptIn',
       'SMode','IeVerIdentifier','Firewall', 'UacLuaenable','Census_OEMNameIdentifier',
    'Census_OEMNameIdentifier','Census_OEMModelIdentifier','Census_ProcessorCoreCount',
    'Census_ProcessorManufacturerIdentifier','Census_ProcessorModelIdentifier','Census_PrimaryDiskTotalCapacity',
    'Census_SystemVolumeTotalCapacity','Census_HasOpticalDiskDrive','Census_TotalPhysicalRAM','Census_InternalPrimaryDiagonalDisplaySizeInInches',
    'Census_InternalPrimaryDisplayResolutionHorizontal','Census_InternalPrimaryDisplayResolutionVertical',
'Census_InternalBatteryNumberOfCharges',
        'Census_OSBuildNumber','Census_OSBuildRevision', 'Census_OSInstallLanguageIdentifier',
       'Census_OSUILocaleIdentifier','Census_IsPortableOperatingSystem',
'Census_IsFlightsDisabled','Census_FirmwareManufacturerIdentifier',
       'Census_FirmwareVersionIdentifier', 'Census_IsSecureBootEnabled', 'Census_IsVirtualDevice', 
    'Wdft_IsGamer','Wdft_RegionIdentifier',]
string_dim=['MachineIdentifier']
#string_df=df[string_dim]
#num_df=df[num_dim]
num_df1=dff[num_dim]
num_df1.dropna(axis=0)
string_df1=dff[string_dim]
num_df1.shape
num_df1['SMode'].fillna(method='ffill',inplace=True)
num_df1.fillna(method='ffill',inplace=True)
#num_df1.isnull().sum()
import numpy as np
m=np.mean(num_df1['SMode'])
#num_df1.fillna,inplace=True)
num_df1["SMode"]=num_df1["SMode"].fillna(m)
num_df1.isnull().sum()
num_df1.shape
from sklearn.preprocessing import StandardScaler
ss=StandardScaler()
ss.fit(num_df1[num_dim])
num_df1[num_dim]=ss.transform(num_df1[num_dim])
#view new data

print("done")
num_df1.shape,string_df1.shape
#get_dummies taking too long need to try somthing else
# training_features1=pd.get_dummies(string_df1,columns=
#                                 string_dim)
#modeling for tranformation on string_dim
from sklearn.preprocessing import LabelEncoder
model1=LabelEncoder()
genre_labels = model1.fit_transform(string_df1['MachineIdentifier'])
dim_label = {index: label for index, label in enumerate(model1.classes_)}
string_df1['MachineIdentifier'] = dim_label
#print(genre_mappings.head())
print("done")
testf = pd.concat([string_df1, num_df1], axis=1, sort=False)
#testf=testf.drop('HasDetections', axis=1)
testf.fillna(method='ffill',inplace=True)
print("done now")
testf.shape
dff.shape
#import joblib
#model=joblib.load('/Users/vishwaprabhakarsingh/Desktop/weekend_dsp_noon/Model/model.pickle')
# scaler=joblib.load('/Users/vishwaprabhakarsingh/Desktop/weekend_dsp_noon/Scaler/scaler.pickle')
predictions=model.predict(testf)
#making combine df to extrack label for submission
#sdf=df[[string_df]]
#fetchdim = pd.concat([sdf, num_df1], axis=1, sort=False)


import pandas as pd
submission = pd.DataFrame({
       "MachineIdentifier": dff["MachineIdentifier"],
       "HasDetections":predictions
   })
#submission.drop(axes=index)
#del submission['index']
submission.to_csv('super3.csv',index=False)
submission.shape
submission.head()
# import numpy as np
# import pandas as pd
# #prediction = pd.DataFrame(predictions, columns=['MachineIdentifier','HasDetections']).to_csv('prediction.csv')
# final_test['MachineIdentifier'] = test['MachineIdentifier']
# final_test['HasDetections'] =predictions

# submission = final_test[['MachineIdentifier','HasDetections']]

# submission.to_csv("submission.csv", index=False)

# submission.tail()7853253<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13346802947468506, 'fraction_numerical': 0.009032564772997385, 'mean_word_length': 8.958579881656805, 'pattern_counts': {'":': 2, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '191673', 'license': 'MIT', 'n_tokens_mistral': 2991, 'n_tokens_neox': 2917, 'n_words': 407, 'path': 'testingNewNEw_finalextremetest_feature_engineering_EDA-4.ipynb', 'repo_name': 'VishwaPrabhakar/malware_prediction_mircosoft_data', 'size': '8414'}