<jupyter_start><jupyter_text>%%local%matplotlib inlineimport matplotlib.pyplot as pltimport numpy as npimport pandas as pdfrom hops import hdfsfrom numpy import genfromtxtpath = hdfs.project_path() + "Logs/test.csv"local_dir = hdfs.localize(path)ts_all_plot = pd.read_csv(local_dir + 'test.csv')my_data = genfromtxt('test.csv', delimiter=',')my_data = my_data.transpose()print(my_data.shape)l_path = hdfs.project_path() + "Logs/labels.csv"l_local_dir = hdfs.localize(l_path)labels = genfromtxt('labels.csv', delimiter=',')cdict = {1.0: 'red', 0.0: 'blue'}fig, ax = plt.subplots(figsize=(50,20))for g in np.unique(labels):    ix = np.where(labels == g)    ax.plot(my_data[:, ix[0]], c = cdict[g]) , label = gplt.show()<jupyter_code>def multi_var_ano_gen(time_steps, sample_size):
    time_steps = time_steps + 1
    baseline = MultivariateDataGenerator(stream_length=time_steps, n=sample_size, k=0)    
    baseline.generate_baseline()
    # would add an extreme outlier to time series 0 at timestamp 3 and a base shift
    #           to time series 3 between timestamps 4 and 10    
    
    shift_start_index =  np.random.randint(time_steps - int(time_steps/4), size=1)[0]
    shift_end_index =  np.random.randint(shift_start_index+1, shift_start_index + int(int(time_steps/4)/2), size=1)[0]
    
    int_to_ano_type = {1: 'extreme',
                       2: 'shift',
                       3: 'variance',
                       4: 'trend'}

    ano_type_key = np.random.randint(1 , 4, size=1)[0]
    
    ano_ts = np.random.randint(low=0, high=sample_size,size=1)[0]
    
    ano_data = baseline.add_outliers(
                        {
                            int_to_ano_type[ano_type_key]:   [{'n': ano_ts, 'timestamps': [(shift_start_index,shift_end_index)]}]
                        }
                )

                    
    return [float(val) for val in list(ano_data.transpose().values[0])][1:time_steps] 
def multi_var_ben_gen(time_steps, sample_size):
    time_steps = time_steps + 1
    baseline = MultivariateDataGenerator(stream_length=time_steps, n=sample_size, k=0)    
    ben_data = baseline.generate_baseline()
                    
    return [float(val) for val in list(ben_data.transpose().values[0])][1:time_steps] 
x = multi_var_ano_gen(time_steps=24, sample_size=1)
len(x)
from pyspark.sql.functions import lit

ano_parallel_instances = sc.parallelize(range(1000))
ano_rdd = ano_parallel_instances.map(lambda row: multi_var_ano_gen(time_steps=365, sample_size=1))
ano_data = ano_rdd.toDF().withColumn("target",lit(1))
ben_parallel_instances = sc.parallelize(range(59000))
ben_rdd = ben_parallel_instances.map(lambda row: multi_var_ben_gen(time_steps=365, sample_size=1))
ben_data = ben_rdd.toDF().withColumn("target",lit(0))
data = ano_data.union(ben_data)
feature_names = data.columns
feature_names = ["f" + s  for s in feature_names]
data = data.toDF(*feature_names)
data = data.withColumnRenamed("ftarget","target")
import hsfs
# Create a connection
connection = hsfs.connection()
# Get the feature store handle for the project's feature store
fs = connection.get_feature_store()
simts_fg_meta = fs.create_feature_group(name="simts_features",
                                       version=1,
                                       time_travel_format=None,                                        
                                       statistics_config=False)
simts_fg_meta.save(data)<jupyter_output><hsfs.feature_group.FeatureGroup object at 0x7f8f901198d0>
{'fraction_non_alphanumeric': 0.11734401831711505, 'fraction_numerical': 0.021465369204350316, 'mean_word_length': 3.3254950495049505, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '588272', 'license': 'Apache-2.0', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1172, 'n_words': 208, 'path': 'adversarialaml/notebooks/feature_simulation.ipynb', 'repo_name': 'logicalclocks/AMLend2end', 'size': '3494'}