<jupyter_start><jupyter_text>Load in data<jupyter_code>cyber_df = pd.read_csv("data/CyberFLTenDays.csv").sample(10000)
cyber_df.index.name = "log_id"
cyber_df.reset_index(inplace=True, drop=False)
cyber_df['label'] = cyber_df['label'].map({'N': False, 'A': True}, na_action='ignore')<jupyter_output><empty_output><jupyter_text>Create an EntitySet with many different dataframesEach dataframe is a different definition of "connection"<jupyter_code>es = ft.EntitySet("CyberLL")
# create an index column
cyber_df["name_host_pair"] = cyber_df["src_name"].str.cat(
                                [cyber_df["dest_name"],
                                 cyber_df["src_host"],
                                 cyber_df["dest_host"]],
                                sep=' / ')
cyber_df["src_pair"] = cyber_df["src_name"].str.cat(
                                 cyber_df["src_host"],
                                 sep=' / ')
cyber_df["dest_pair"] = cyber_df["dest_name"].str.cat(
                                 cyber_df["dest_host"],
                                 sep=' / ')
es.add_dataframe(dataframe_name="log",
                 dataframe=cyber_df,
                 index="log_id",
                 time_index="secs")
es.normalize_dataframe(base_dataframe_name="log",
                       new_dataframe_name="name_host_pairs",
                       index="name_host_pair",
                       additional_columns=["src_name", "dest_name",
                                           "src_host", "dest_host",
                                           "src_pair",
                                           "dest_pair",
                                           "label"])
es.normalize_dataframe(base_dataframe_name="name_host_pairs",
                       new_dataframe_name="src_pairs",
                       index="src_pair",
                       additional_columns=["src_name", "src_host"])
es.normalize_dataframe(base_dataframe_name="src_pairs",
                       new_dataframe_name="src_names",
                       index="src_name")
es.normalize_dataframe(base_dataframe_name="src_pairs",
                       new_dataframe_name="src_hosts",
                       index="src_host")
es.normalize_dataframe(base_dataframe_name="name_host_pairs",
                       new_dataframe_name="dest_pairs",
                       index="dest_pair",
                       additional_columns=["dest_name", "dest_host"])
es.normalize_dataframe(base_dataframe_name="dest_pairs",
                       new_dataframe_name="dest_names",
                       index="dest_name")
es.normalize_dataframe(base_dataframe_name="dest_pairs",
                       new_dataframe_name="dest_hosts",
                       index="dest_host")<jupyter_output><empty_output><jupyter_text>Visualize EntitySet<jupyter_code>es.plot()<jupyter_output><empty_output><jupyter_text>Define function to generate labels and cutoff times We use [Compose](https://compose.featurelabs.com/) to define our labeling function.<jupyter_code>def malicious_connection(df, lead):
    if (len(df.index) > lead + 1): 
        return df.iloc[lead:]['label'].any()
    
def label_generator(cyber_df, index_col, after_n_obs, lead, prediction_window):
    lm = cp.LabelMaker(
        target_dataframe_name=index_col,
        time_index="secs",
        labeling_function=malicious_connection,
        window_size=window + lead
    )
    label_times = lm.search(
        cyber_df.sort_values('secs'),
        minimum_data=after_n_obs,
        gap=after_n_obs + lead + window,
        num_examples_per_instance=1,
        lead=lead,
        verbose=False
    )
    label_times['time'] = pd.to_numeric(label_times['time'])
    return label_times
# predict after 3 observations
after_n_obs = 3

# predict 2 observations out
lead = 2

# predict if any malicious attacks in a 10-observation window
window = 10<jupyter_output><empty_output><jupyter_text>Compute features for various types of connections<jupyter_code># features on src_name
cutoffs = label_generator(cyber_df, "src_name", after_n_obs, lead, window)
fm, fl = ft.dfs(entityset=es, target_dataframe_name="src_names", cutoff_time=cutoffs, verbose=True, max_depth=3)
## features on src_host
cutoffs = label_generator(cyber_df, "src_host", after_n_obs, lead, window)
fm, fl = ft.dfs(entityset=es, target_dataframe_name="src_hosts", cutoff_time=cutoffs, verbose=True, max_depth=3)
## features on dest_name
cutoffs = label_generator(cyber_df, "dest_name", after_n_obs, lead, window)
fm, fl = ft.dfs(entityset=es, target_dataframe_name="dest_names", cutoff_time=cutoffs, verbose=True, max_depth=3)
## features on dest_host
cutoffs = label_generator(cyber_df, "dest_host", after_n_obs, lead, window)
fm, fl = ft.dfs(entityset=es, target_dataframe_name="dest_hosts", cutoff_time=cutoffs, verbose=True, max_depth=3)
# features on src_name/dest_name/src_host/dest_host
cutoffs = label_generator(cyber_df, "name_host_pair", after_n_obs, lead, window)
fm, fl = ft.dfs(entityset=es, target_dataframe_name="name_host_pairs", cutoff_time=cutoffs, verbose=True, max_depth=2, trans_primitives=[Percentile])
# merge dataframes together to access the index columns created in the process of normalizing
merged = (es['log'].merge(es['name_host_pairs'])
                   .merge(es['src_pairs'])
                   .merge(es['dest_pairs']))
# features on src_name/src_host
cutoffs = label_generator(merged, 'src_pair', after_n_obs, lead, window)
fm, fl = ft.dfs(entityset=es, target_dataframe_name="src_pairs", cutoff_time=cutoffs, verbose=True, max_depth=2, trans_primitives=[Percentile])
# features on dest_name/dest_host
cutoffs = label_generator(merged, 'dest_pair', after_n_obs, lead, window)
fm, fl = ft.dfs(entityset=es, target_dataframe_name="dest_pairs", cutoff_time=cutoffs, verbose=True, max_depth=2, trans_primitives=[Percentile])<jupyter_output>Built 193 features
Elapsed: 01:00 | Progress: 100%|██████████████████████████████████████████
{'fraction_non_alphanumeric': 0.11852472757753563, 'fraction_numerical': 0.005364626990779548, 'mean_word_length': 3.1201657458563536, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '62050', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1948, 'n_tokens_neox': 1850, 'n_words': 301, 'path': 'predict-malicious-cyber-connections/Create Feature Matrices from LL Cyber Data.ipynb', 'repo_name': 'Featuretools/predict_next_purchase', 'size': '5965'}