<jupyter_start><jupyter_text>What's New and Changed in version 2.12.220325---------------------------------------------New functions:    - Added isolation forest.    - Added automatic classification and regression.    - Added progress monitor for auto-ml.    - Added best pipeline report for auto-ml.    - Added to_datetime function.    - Added to_tail function in dataframe.    - Added AFL state for unified API.Enhancement:    - Added validation procedure for n_components in CATPCA.    - Improved display name in pivot_table.    - Added compression and thresholding in wavelet transform.    - Moved generate_feature to dataframe function.    - Enhanced create_dataframe_from_pandas with upsert option.    - Added ignore_scatter_matrix option in dataset report.    - Expose APL variable selection parameters.    - Enhanced text mining with German support.    - Support more loss functions in HybridGradientBoostingClassifier and HybridGradientBoostingRegressor.    - Enhanced white_noise_test() with an option: the degree of freedom, model_df.    - Enhanced Attention with the explaining of model.    - Ehhanced integer index support for TimeSeriesExplainer.explain_arima_model() for ARIMA and AutoARIMA.    - Added procomputed affinity for AgglomerateHierarchicalClustering.    - Added model compression related parameters for HybridGradientBoostingClassifier and HybridGradientBoostingRegressor.Bug Fixes:    - Fix M4 sampling with lowercase column name.    - Fix inconsistent IDs assigned to solvers between LOGR and M_LOGR.    - Fix a parameter naming error in fft(): flattop_model -> flattop_mode.    - Fix a validation error for endog parameter in Attention predict().API change:    - Added 'model_df' in the white_noise_test() for selecting the degree of freedom.    - Added 'explain_mode' in predict() of GRUAttention for selecting the mechanism for generating the reason code for inference results.<jupyter_code># Connection
import pandas as pd
from hana_ml import dataframe

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

settings = configparser.ConfigParser()
settings.read("../../config/e2edata.ini")<jupyter_output><empty_output><jupyter_text>Isolation Forest<jupyter_code>url = settings.get("hana", "url")
port = settings.getint("hana", "port")
user = settings.get("hana", "user")
pwd = settings.get("hana", "<PASSWORD>")

connection_context = dataframe.ConnectionContext(url, port, user, pwd)

# https://help.sap.com/viewer/DRAFT/319d36de4fd64ac3afbf91b1fb3ce8de/2021_4_QRC/en-US/11345d90db35416e94de653248ac2184.html
    
from hana_ml.algorithms.pal.preprocessing import IsolationForest

df = dataframe.create_dataframe_from_pandas(connection_context,
                                            pandas_df=pd.DataFrame({"ID": [0, 1, 2 ,3, 4, 5, 6, 7],
                                                                    "V000": [-2, -1, -1, 1, 1, 2, 6, -4],
                                                                    "V001": [-1, -1, -2, 1, 2, 1, 3, 7]}),
                                            table_name="#IF_DEMO",
                                            force=True)

clf = IsolationForest(random_state=2,
                      thread_ratio=0)

res = clf.fit_predict(data=df,
                      key="ID",
                      contamination=0.25)
# -1 for outliers and 1 for inliers.
print(res.collect())
connection_context<jupyter_output><empty_output><jupyter_text>automl<jupyter_code>url = settings.get("automl", "url")
port = settings.getint("automl", "port")
user = settings.get("automl", "user")
pwd = settings.get("automl", "passwd")

conn_dev3 = dataframe.ConnectionContext(url, port, user, pwd)
df = dataframe.create_dataframe_from_pandas(conn_dev3,
                                            pd.read_csv("../datasets/digits.csv"),
                                            table_name="#demo_automl_tbl",
                                            force=True).deselect("Unnamed: 0")
df = df.rename_columns({"ID": "LABEL"}).dropna().cast({"LABEL": "VARCHAR(500)"})
import uuid
from hana_ml.algorithms.pal.auto_ml import AutomaticClassification

progress_id = "automl_{}".format(uuid.uuid1())
auto_c = AutomaticClassification(generations=2, 
                                 population_size=5,
                                 offspring_size=5,
                                 elite_number=2,
                                 mutation_rate=0.1,
                                 progress_indicator_id=progress_id)
from hana_ml.visualizers.automl_progress import PipelineProgressStatusMonitor

progress_status_monitor = PipelineProgressStatusMonitor(connection_context=dataframe.ConnectionContext(url, port, user, pwd), automatic_obj=auto_c)

progress_status_monitor.start()
auto_c.enable_workload_class("PAL_AUTOML_WORKLOAD")
auto_c.fit(df)
from hana_ml.visualizers.automl_report import BestPipelineReport
BestPipelineReport(auto_c).generate_notebook_iframe()
df_test = df.head(10).deselect("LABEL")
res = auto_c.predict(df_test)
print(res.collect())
pipeline = auto_c.best_pipeline_.collect().iat[0, 1]

auto_c.fit(df, pipeline=pipeline)

res = auto_c.predict(df_test)
print(res.collect())
conn_dev3.close()<jupyter_output><empty_output><jupyter_text>create model state<jupyter_code>from hana_ml.algorithms.pal.utility import DataSets
full_set, diabetes_train, diabetes_test, _ = DataSets.load_diabetes_data(connection_context)
from hana_ml.algorithms.pal.unified_classification import UnifiedClassification
cv_values = {}
cv_values['learning_rate'] = [0.1, 0.4, 0.7, 1.0]
cv_values['n_estimators'] = [4, 6, 8, 10]
cv_values['split_threshold'] = [0.1, 0.4, 0.7, 1.0]
hgc = UnifiedClassification(func='HybridGradientBoostingTree',
                            param_search_strategy='grid',
                            resampling_method='cv',
                            evaluation_metric='error_rate',
                            ref_metric=['auc'],
                            fold_num=5,
                            random_state=1,
                            param_values=cv_values)
hgc.fit(diabetes_train, key='ID', label='CLASS',
        partition_method='stratified',
        partition_random_state=1,
        stratified_column='CLASS')
hgc.create_model_state()
print(hgc.state.collect())
features = diabetes_train.columns
features.remove('CLASS')
features.remove('ID')
pred_res = hgc.predict(diabetes_test, key='ID', features=features)

print(hgc.get_predict_execute_statement())<jupyter_output><empty_output><jupyter_text>dataframe functions  to_datetime<jupyter_code>df_datetime = dataframe.create_dataframe_from_pandas(connection_context,
                                                     pandas_df=pd.DataFrame({"datetime": ["12/01/2022 23:11:24"]}),
                                                     table_name="#to_datetime_demo",
                                                     table_structure={"datetime": "VARCHAR(50)"},
                                                     force=True)
df_datetime.get_table_structure()
print(df_datetime.to_datetime({"datetime": "MM/DD/YYYY HH24:MI:SS"}).get_table_structure())
print(df_datetime.to_datetime({"datetime": "MM/DD/YYYY HH24:MI:SS"}).collect())<jupyter_output><empty_output><jupyter_text>to_tail<jupyter_code>df_to_tail = dataframe.create_dataframe_from_pandas(connection_context,
                                                    pandas_df=pd.DataFrame({"LABEL": ["1"], "FEATURE": [2], "ID": [3]}),
                                                    table_name="#to_tail_demo",
                                                    force=True)
print(df_to_tail.collect())
print(df_to_tail.to_tail("LABEL").to_head("ID").collect())<jupyter_output><empty_output><jupyter_text>tail<jupyter_code>full_set, training_set, validation_set, test_set = DataSets.load_iris_data(connection_context)
print(full_set.tail(10, ref_col="ID").collect())<jupyter_output><empty_output><jupyter_text>generate_feature<jupyter_code>full_set.generate_feature(targets=["SEPALLENGTHCM", "SEPALWIDTHCM"],
                          group_by="SPECIES",
                          trans_func="LAG",
                          order_by="ID",
                          trans_param=[[1, 3], [1, 4]]).head(10).collect()<jupyter_output><empty_output><jupyter_text>upsert and append option in create_dataframe_from_pandas<jupyter_code>df_upsert = dataframe.create_dataframe_from_pandas(connection_context,
                                                   pandas_df=pd.DataFrame({"ID": [1, 2], "FEATURE": [2, 3], "LABEL": ["3", "4"]}),
                                                   table_name="#upsert_demo",
                                                   primary_key="ID",
                                                   force=True)
print(df_upsert.collect())
df_upsert = dataframe.create_dataframe_from_pandas(connection_context,
                                                   pandas_df=pd.DataFrame({"ID": [2], "FEATURE": [10], "LABEL": [None]}),
                                                   table_name="#upsert_demo",
                                                   primary_key="ID",
                                                   upsert=True)
print(df_upsert.collect())
df_upsert = dataframe.create_dataframe_from_pandas(connection_context,
                                                   pandas_df=pd.DataFrame({"ID": [3], "FEATURE": [10], "LABEL": [None]}),
                                                   table_name="#upsert_demo",
                                                   primary_key="ID",
                                                   append=True)
print(df_upsert.collect())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10868220251598268, 'fraction_numerical': 0.018148071767374718, 'mean_word_length': 2.3806204252352736, 'pattern_counts': {'":': 21, '<': 38, '<?xml version=': 0, '>': 39, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 59, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '373902', 'license': 'Apache-2.0', 'n_tokens_mistral': 2841, 'n_tokens_neox': 2704, 'n_words': 506, 'path': 'Python-API/pal/notebooks/22QRC01.ipynb', 'repo_name': 'xinchen510/hana-ml-samples', 'size': '9694'}