<jupyter_start><jupyter_text>Prepare Data<jupyter_code>#You can use your own data, data provided by Qindom is just for demonstration
data_df = pd.DataFrame(columns = ['train_X' , 'valid_X', 'train_y' , 'valid_y','train_X_y'], index=datasets)
for dataset in datasets:
    print("start data preparation for dataset: " + dataset)
    
    train_valid_X, train_valid_y = data_utils.get_data(dataset)
    # Split the data for train and validation
    train_X, valid_X, train_y, valid_y = train_test_split(train_valid_X, train_valid_y, train_size=.7)
    train_X_y = np.concatenate((train_X, train_y.reshape(train_y.size,1)), axis=1)
    
    data_df['train_X'][dataset] = train_X
    data_df['train_y'][dataset] = train_y
    data_df['valid_X'][dataset] = valid_X
    data_df['valid_y'][dataset] = valid_y
    data_df['train_X_y'][dataset] = train_X_y
    
    np.savetxt("c:\\users\\fores\\Demo\\data\\" + dataset + "_train_X_y.csv", train_X_y, delimiter=',') #change to your own directory
    np.savetxt("c:\\users\\fores\\Demo\\data\\" + dataset + "_valid_X.csv", valid_X, delimiter=',') #change to your own directory
    for data in ['train_X_y','valid_X']:
        #change to your own directory
        cmdln = "C:\\Amazon\\AWSCLI\\bin\\aws.cmd s3 cp C:\\Users\\fores\\Demo\\data\\" + dataset + "_" + data + ".csv s3://qitdata/" + username + "/" + dataset + "_" + data + ".csv"
        p = subprocess.Popen(cmdln, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        for line in p.stdout.readlines():
            print(line)
            
    print("finished data preparation for dataset:" + dataset)
    
print("Finished all data preparation")<jupyter_output><empty_output><jupyter_text>Train & Predict<jupyter_code>clf_df = pd.DataFrame(columns = ['Qit'], index=datasets)
prd_df = pd.DataFrame(columns = ['Qit'], index=datasets)
for dataset in datasets:
    
    print("start to train model for: " + dataset)

    train_X = data_df['train_X'][dataset]
    train_y = data_df['train_y'][dataset]
    train_X_y = data_df['train_X_y'][dataset]
    
    model_id = client.train_model_s3(training_data=username +'/' + dataset + '_train_X_y.csv', clf_type='QBOOST_IT')
    clf_df['Qit'][dataset] = model_id['body']
    print("model id:" + str(clf_df['Qit'][dataset]))
    
    print("start predict for dataset: " + dataset)
    
    valid_X = data_df['valid_X'][dataset]
    valid_y = data_df['valid_y'][dataset]
    
#Get prediction result from Qit    
    prediction_result = client.predict_sync(test_data=username + '/' + dataset + '_valid_X.csv', modelId=clf_df['Qit'][dataset]) 
    while 'Models are not ready yet' in prediction_result:
        print(prediction_result + ", will try after a minute")
        time.sleep(60)
        prediction_result = client.predict_sync(test_data=username + '/' + dataset + '_valid_X.csv', modelId=clf_df['Qit'][dataset])
    prd_df['Qit'][dataset] = np.array(json.loads(prediction_result)['y_test_pred'])
    
#Calculate accuracy   
    qit_accuracy = metrics.accuracy_score(valid_y, prd_df['Qit'][dataset])
  
    print("finished predict for dataset: " + dataset + " with accuracy " + qit_accuracy)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14775227915749764, 'fraction_numerical': 0.002514932411191449, 'mean_word_length': 4.115755627009646, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '415402', 'license': 'Apache-2.0', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1125, 'n_words': 213, 'path': 'Sample.ipynb', 'repo_name': 'Qindom/QIT', 'size': '3181'}